@smthrs/notifications 0.0.0-stage → 1.0.0-rc.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +127 -0
  2. package/LICENSE +21 -0
  3. package/README.md +91 -2
  4. package/dist/cjs/AlertPolicy.d.ts +284 -0
  5. package/dist/cjs/AlertPolicy.d.ts.map +1 -0
  6. package/dist/cjs/AlertPolicy.js +124 -0
  7. package/dist/cjs/AlertPolicy.js.map +7 -0
  8. package/dist/cjs/AlertRuntime.d.ts +90 -0
  9. package/dist/cjs/AlertRuntime.d.ts.map +1 -0
  10. package/dist/cjs/AlertRuntime.js +166 -0
  11. package/dist/cjs/AlertRuntime.js.map +7 -0
  12. package/dist/cjs/AlertSink.d.ts +101 -0
  13. package/dist/cjs/AlertSink.d.ts.map +1 -0
  14. package/dist/cjs/AlertSink.js +44 -0
  15. package/dist/cjs/AlertSink.js.map +7 -0
  16. package/dist/cjs/Alerts.d.ts +57 -0
  17. package/dist/cjs/Alerts.d.ts.map +1 -0
  18. package/dist/cjs/Alerts.js +46 -0
  19. package/dist/cjs/Alerts.js.map +7 -0
  20. package/dist/cjs/Notification.d.ts +214 -0
  21. package/dist/cjs/Notification.d.ts.map +1 -0
  22. package/dist/cjs/Notification.js +62 -0
  23. package/dist/cjs/Notification.js.map +7 -0
  24. package/dist/cjs/NotificationEvent.d.ts +214 -0
  25. package/dist/cjs/NotificationEvent.d.ts.map +1 -0
  26. package/dist/cjs/NotificationEvent.js +84 -0
  27. package/dist/cjs/NotificationEvent.js.map +7 -0
  28. package/dist/cjs/NotificationQueue.d.ts +193 -0
  29. package/dist/cjs/NotificationQueue.d.ts.map +1 -0
  30. package/dist/cjs/NotificationQueue.js +478 -0
  31. package/dist/cjs/NotificationQueue.js.map +7 -0
  32. package/dist/cjs/NotificationState.d.ts +169 -0
  33. package/dist/cjs/NotificationState.d.ts.map +1 -0
  34. package/dist/cjs/NotificationState.js +110 -0
  35. package/dist/cjs/NotificationState.js.map +7 -0
  36. package/dist/cjs/Projection.d.ts +27 -0
  37. package/dist/cjs/Projection.d.ts.map +1 -0
  38. package/dist/cjs/Projection.js +54 -0
  39. package/dist/cjs/Projection.js.map +7 -0
  40. package/dist/cjs/SteerPayload.d.ts +177 -0
  41. package/dist/cjs/SteerPayload.d.ts.map +1 -0
  42. package/dist/cjs/SteerPayload.js +61 -0
  43. package/dist/cjs/SteerPayload.js.map +7 -0
  44. package/dist/cjs/index.d.ts +41 -0
  45. package/dist/cjs/index.d.ts.map +1 -0
  46. package/dist/cjs/index.js +47 -0
  47. package/dist/cjs/index.js.map +7 -0
  48. package/dist/cjs/internal/foldCache.d.ts +36 -0
  49. package/dist/cjs/internal/foldCache.d.ts.map +1 -0
  50. package/dist/cjs/internal/foldCache.js +54 -0
  51. package/dist/cjs/internal/foldCache.js.map +7 -0
  52. package/dist/cjs/layerWebhook.d.ts +53 -0
  53. package/dist/cjs/layerWebhook.d.ts.map +1 -0
  54. package/dist/cjs/layerWebhook.js +107 -0
  55. package/dist/cjs/layerWebhook.js.map +7 -0
  56. package/dist/cjs/package.json +1 -0
  57. package/dist/esm/AlertPolicy.d.ts +284 -0
  58. package/dist/esm/AlertPolicy.d.ts.map +1 -0
  59. package/dist/esm/AlertPolicy.js +284 -0
  60. package/dist/esm/AlertPolicy.js.map +1 -0
  61. package/dist/esm/AlertRuntime.d.ts +90 -0
  62. package/dist/esm/AlertRuntime.d.ts.map +1 -0
  63. package/dist/esm/AlertRuntime.js +191 -0
  64. package/dist/esm/AlertRuntime.js.map +1 -0
  65. package/dist/esm/AlertSink.d.ts +101 -0
  66. package/dist/esm/AlertSink.d.ts.map +1 -0
  67. package/dist/esm/AlertSink.js +63 -0
  68. package/dist/esm/AlertSink.js.map +1 -0
  69. package/dist/esm/Alerts.d.ts +57 -0
  70. package/dist/esm/Alerts.d.ts.map +1 -0
  71. package/dist/esm/Alerts.js +57 -0
  72. package/dist/esm/Alerts.js.map +1 -0
  73. package/dist/esm/Notification.d.ts +214 -0
  74. package/dist/esm/Notification.d.ts.map +1 -0
  75. package/dist/esm/Notification.js +121 -0
  76. package/dist/esm/Notification.js.map +1 -0
  77. package/dist/esm/NotificationEvent.d.ts +214 -0
  78. package/dist/esm/NotificationEvent.d.ts.map +1 -0
  79. package/dist/esm/NotificationEvent.js +158 -0
  80. package/dist/esm/NotificationEvent.js.map +1 -0
  81. package/dist/esm/NotificationQueue.d.ts +193 -0
  82. package/dist/esm/NotificationQueue.d.ts.map +1 -0
  83. package/dist/esm/NotificationQueue.js +626 -0
  84. package/dist/esm/NotificationQueue.js.map +1 -0
  85. package/dist/esm/NotificationState.d.ts +169 -0
  86. package/dist/esm/NotificationState.d.ts.map +1 -0
  87. package/dist/esm/NotificationState.js +191 -0
  88. package/dist/esm/NotificationState.js.map +1 -0
  89. package/dist/esm/Projection.d.ts +27 -0
  90. package/dist/esm/Projection.d.ts.map +1 -0
  91. package/dist/esm/Projection.js +43 -0
  92. package/dist/esm/Projection.js.map +1 -0
  93. package/dist/esm/SteerPayload.d.ts +177 -0
  94. package/dist/esm/SteerPayload.d.ts.map +1 -0
  95. package/dist/esm/SteerPayload.js +139 -0
  96. package/dist/esm/SteerPayload.js.map +1 -0
  97. package/dist/esm/index.d.ts +41 -0
  98. package/dist/esm/index.d.ts.map +1 -0
  99. package/dist/esm/index.js +41 -0
  100. package/dist/esm/index.js.map +1 -0
  101. package/dist/esm/internal/foldCache.d.ts +36 -0
  102. package/dist/esm/internal/foldCache.d.ts.map +1 -0
  103. package/dist/esm/internal/foldCache.js +58 -0
  104. package/dist/esm/internal/foldCache.js.map +1 -0
  105. package/dist/esm/layerWebhook.d.ts +53 -0
  106. package/dist/esm/layerWebhook.d.ts.map +1 -0
  107. package/dist/esm/layerWebhook.js +118 -0
  108. package/dist/esm/layerWebhook.js.map +1 -0
  109. package/docs/README.md +144 -0
  110. package/docs/api.md +439 -0
  111. package/docs/concepts/admission-and-promotion.md +144 -0
  112. package/docs/concepts/alerting.md +138 -0
  113. package/docs/concepts/journal-records.md +132 -0
  114. package/docs/guides/alert-on-a-stuck-run.md +140 -0
  115. package/docs/guides/drain-at-a-turn-boundary.md +95 -0
  116. package/docs/guides/handle-a-full-queue.md +102 -0
  117. package/docs/guides/report-pending-notifications.md +92 -0
  118. package/docs/guides/send-alerts-to-a-webhook.md +130 -0
  119. package/docs/guides/steer-a-run.md +127 -0
  120. package/docs/guides/testing.md +161 -0
  121. package/docs/installation.md +119 -0
  122. package/docs/quickstart.md +159 -0
  123. package/docs/troubleshooting.md +254 -0
  124. package/package.json +182 -3
  125. package/src/AlertPolicy.ts +377 -0
  126. package/src/AlertRuntime.ts +280 -0
  127. package/src/AlertSink.ts +105 -0
  128. package/src/Alerts.ts +71 -0
  129. package/src/Notification.ts +171 -0
  130. package/src/NotificationEvent.ts +213 -0
  131. package/src/NotificationQueue.ts +857 -0
  132. package/src/NotificationState.ts +295 -0
  133. package/src/Projection.ts +46 -0
  134. package/src/SteerPayload.ts +198 -0
  135. package/src/index.ts +47 -0
  136. package/src/internal/foldCache.ts +72 -0
  137. package/src/layerWebhook.ts +161 -0
@@ -0,0 +1,857 @@
1
+ /**
2
+ * Journal-backed durable notification admission and turn-boundary drain.
3
+ *
4
+ * Governing contract: `packages/smithers/notifications/docs/api.md`, published as
5
+ * `/api/notifications`.
6
+ *
7
+ * @since 0.1.0
8
+ */
9
+
10
+ import { canonicalize } from "@smthrs/canonical"
11
+ import { digestSync } from "@smthrs/crypto/Sha256"
12
+ import { Journal, JournalEvent } from "@smthrs/journal"
13
+ import { Context, Effect, HashMap, Layer, Option, Schema, SchemaIssue, Semaphore } from "effect"
14
+ import * as FoldCache from "./internal/foldCache.ts"
15
+ import * as NotificationModel from "./Notification.ts"
16
+ import * as NotificationEvent from "./NotificationEvent.ts"
17
+ import * as NotificationState from "./NotificationState.ts"
18
+
19
+ /**
20
+ * The queue could not be reached, or it refused what a caller handed it.
21
+ *
22
+ * `code` is the stable half and the one to branch on. `notification_unavailable`
23
+ * is the seam reporting that it serves nothing; `notification_id_reused` says a
24
+ * stable id was admitted once already with different content, which is a
25
+ * producer bug rather than a storage failure; `notification_invalid` says the
26
+ * value is not a notification, and `path` names the field inside it that failed.
27
+ * `notification_closed` means the receiver has finished and the caller must
28
+ * start a new request; `notification_full` means nothing was retained and the
29
+ * same notification may be retried after pending work drains. Neither the
30
+ * message nor `path` ever carries the offending value.
31
+ * `notification_refused` rejects reserved outside-change payloads before admission.
32
+ *
33
+ * @category errors
34
+ * @since 0.1.0
35
+ */
36
+ export class NotificationError extends Schema.TaggedError<NotificationError>()(
37
+ "/notifications/NotificationError",
38
+ {
39
+ code: Schema.Literals([
40
+ "notification_unavailable",
41
+ "notification_closed",
42
+ "notification_full",
43
+ "notification_id_reused",
44
+ "notification_invalid",
45
+ "notification_refused"
46
+ ]).pipe(
47
+ Schema.withConstructorDefault(Effect.succeed("notification_unavailable" as const))
48
+ ),
49
+ message: Schema.String,
50
+ /** The notification the failure is about, when one was readable. */
51
+ notificationId: Schema.optional(Schema.String),
52
+ /** Dotted path of the offending field, for `notification_invalid`. */
53
+ path: Schema.optional(Schema.String)
54
+ }
55
+ ) {}
56
+
57
+ /**
58
+ * What admitting one notification decided.
59
+ *
60
+ * `seq` is the journal sequence the admission committed at, and it is absent
61
+ * exactly when nothing was written, which today means a `rejected-full`
62
+ * refusal. `duplicate` says the id was already admitted, in which case
63
+ * `decision` and `seq` are read back from the committed record rather than
64
+ * recomputed here.
65
+ *
66
+ * A caller MUST inspect `decision`: `rejected-full` means the queue was at
67
+ * capacity and retained nothing, so the notification never reaches the model
68
+ * unless the caller admits it again once a boundary has drained.
69
+ *
70
+ * @category models
71
+ * @since 0.1.0
72
+ */
73
+ export interface AdmissionReceipt {
74
+ readonly notificationId: string
75
+ readonly decision: NotificationState.AdmissionDecision
76
+ readonly seq: number | undefined
77
+ readonly duplicate: boolean
78
+ readonly consumed?: boolean
79
+ readonly consumedNotification?: NotificationModel.Notification
80
+ }
81
+
82
+ /**
83
+ * The boundary a drain is attempted at, and whether the run would go idle
84
+ * if nothing were delivered.
85
+ *
86
+ * `(runId, targetLineageId, boundary)` is the unit of drain: two lineages
87
+ * closing a turn under the same boundary name are two drains, and each is
88
+ * recorded separately.
89
+ *
90
+ * @category models
91
+ * @since 0.1.0
92
+ */
93
+ export interface DrainInput {
94
+ readonly runId: string
95
+ readonly targetLineageId: string
96
+ readonly boundary: string
97
+ readonly wouldIdle: boolean
98
+ /**
99
+ * The journal sequence that opened this turn. A steer admitted after it is
100
+ * held for the next boundary, which is what keeps a message that arrived
101
+ * mid-turn out of the turn already in flight. Omitting it delivers
102
+ * everything pending for the lineage.
103
+ */
104
+ readonly cutoffSeq?: number | undefined
105
+ }
106
+
107
+ /**
108
+ * The notifications this boundary delivers, and whether the boundary had
109
+ * already drained.
110
+ *
111
+ * The notifications are the ones the committed promotion record names, so two
112
+ * processes draining one boundary report the same delivery rather than two
113
+ * divergent guesses.
114
+ *
115
+ * @category models
116
+ * @since 0.1.0
117
+ */
118
+ export interface DrainReceipt {
119
+ readonly notifications: ReadonlyArray<NotificationModel.Notification>
120
+ readonly boundary: string
121
+ readonly duplicate: boolean
122
+ }
123
+
124
+ /**
125
+ * The durable pending queue: admit a notification exactly once, and drain
126
+ * what a boundary is allowed to deliver.
127
+ *
128
+ * @category services
129
+ * @since 0.1.0
130
+ */
131
+ export interface Service {
132
+ readonly admit: (
133
+ runId: string,
134
+ notification: NotificationModel.Notification,
135
+ version?: number
136
+ ) => Effect.Effect<AdmissionReceipt, Journal.JournalError | NotificationError>
137
+ readonly drain: (
138
+ input: DrainInput
139
+ ) => Effect.Effect<DrainReceipt, Journal.JournalError | NotificationError>
140
+ /**
141
+ * What this run has been told that no boundary has delivered yet, in
142
+ * admission order.
143
+ *
144
+ * The queue is the owner of that fact: pending is admitted minus promoted,
145
+ * and both halves are its own journal records. A supervisor that counted
146
+ * admissions alone would report a steer as waiting forever.
147
+ */
148
+ readonly pending: (
149
+ runId: string
150
+ ) => Effect.Effect<ReadonlyArray<NotificationModel.Notification>, Journal.JournalError | NotificationError>
151
+ }
152
+
153
+ /**
154
+ * The {@link Service} tag.
155
+ *
156
+ * @category services
157
+ * @since 0.1.0
158
+ */
159
+ export class NotificationQueue extends Context.Service<NotificationQueue, Service>()(
160
+ "/notifications/NotificationQueue"
161
+ ) {}
162
+
163
+ /**
164
+ * Builds a {@link Service} from an implementation of its methods.
165
+ *
166
+ * @param implementation the methods to serve
167
+ * @category constructors
168
+ * @since 0.1.0
169
+ */
170
+ export const make = (implementation: Service): Service => NotificationQueue.of(implementation)
171
+
172
+ const unavailable = (operation: string): NotificationError =>
173
+ new NotificationError({ code: "notification_unavailable", message: `${operation} is unavailable` })
174
+
175
+ /**
176
+ * A {@link Service} that fails every method as unavailable. Overrides
177
+ * replace individual methods.
178
+ *
179
+ * @param overrides methods to serve instead of failing
180
+ * @category constructors
181
+ * @since 0.1.0
182
+ */
183
+ export const makeNoop = (overrides: Partial<Service> = {}): Service =>
184
+ make({
185
+ admit: Effect.fn("NotificationQueue.admit")(() => Effect.fail(unavailable("admit"))),
186
+ drain: Effect.fn("NotificationQueue.drain")(() => Effect.fail(unavailable("drain"))),
187
+ pending: Effect.fn("NotificationQueue.pending")(() => Effect.fail(unavailable("pending"))),
188
+ ...overrides
189
+ })
190
+
191
+ /**
192
+ * Provides {@link makeNoop}.
193
+ *
194
+ * @param overrides methods to serve instead of failing
195
+ * @category layers
196
+ * @since 0.1.0
197
+ */
198
+ export const layerNoop = (overrides: Partial<Service> = {}): Layer.Layer<NotificationQueue> =>
199
+ Layer.succeed(NotificationQueue)(makeNoop(overrides))
200
+
201
+ /**
202
+ * The deepest a notification's JSON may nest.
203
+ *
204
+ * The bound exists because the value is walked and serialized on the way to
205
+ * the journal, and an unbounded walk turns a hostile payload into an untyped
206
+ * `RangeError` instead of a refusal a caller can read.
207
+ */
208
+ const maximumPayloadDepth = 256
209
+
210
+ /**
211
+ * The largest a notification may be, in UTF-16 code units: every string and
212
+ * object key counted at its length, and every other value as one.
213
+ *
214
+ * The bound exists because an admission copies, decodes, canonicalizes,
215
+ * fingerprints, and journals the whole value, so a producer admitting a huge
216
+ * array or string spends the notifying process's CPU and memory and bloats
217
+ * every replay of the run. It is checked before any of that work starts.
218
+ */
219
+ const maximumNotificationCodeUnits = 1_048_576
220
+
221
+ /** Bounds an issue message so a refusal cannot journal or log an essay. */
222
+ const maximumIssueCodeUnits = 200
223
+
224
+ /**
225
+ * The bound a value breaks, if any, walked iteratively so the walk itself
226
+ * cannot exhaust the stack. A cycle is caught by the depth bound: it has no
227
+ * finite depth, so it trips the limit rather than looping forever. The size
228
+ * count stops the walk as soon as it passes the bound, so an oversized value
229
+ * costs the walk no more than the bound does.
230
+ */
231
+ const exceededBound = (value: unknown): "depth" | "size" | undefined => {
232
+ const pending: Array<{ readonly value: unknown; readonly depth: number }> = [{ value, depth: 0 }]
233
+ let size = 0
234
+ while (pending.length > 0) {
235
+ const next = pending.pop()!
236
+ size += typeof next.value === "string" ? next.value.length : 1
237
+ if (size > maximumNotificationCodeUnits) return "size"
238
+ if (typeof next.value !== "object" || next.value === null) continue
239
+ if (next.depth >= maximumPayloadDepth) return "depth"
240
+ if (Array.isArray(next.value)) {
241
+ for (const child of next.value) pending.push({ value: child, depth: next.depth + 1 })
242
+ continue
243
+ }
244
+ for (const [key, child] of Object.entries(next.value)) {
245
+ size += key.length
246
+ pending.push({ value: child, depth: next.depth + 1 })
247
+ }
248
+ }
249
+ return undefined
250
+ }
251
+
252
+ /**
253
+ * A structural copy of the caller's value, so nothing the queue journals or
254
+ * retains can be edited afterwards through a reference the caller kept. Values
255
+ * that are not arrays or objects pass through, which is what leaves a function
256
+ * or a symbol in place for the schema to reject.
257
+ */
258
+ const copied = (value: unknown): unknown => {
259
+ if (Array.isArray(value)) return value.map(copied)
260
+ if (typeof value === "object" && value !== null) {
261
+ return Object.fromEntries(Object.entries(value).map(([key, nested]) => [key, copied(nested)]))
262
+ }
263
+ return value
264
+ }
265
+
266
+ const formatIssue = SchemaIssue.makeFormatterStandardSchemaV1({ leafHook: (issue) => issue._tag })
267
+
268
+ const bounded = (text: string): string =>
269
+ text.length <= maximumIssueCodeUnits ? text : `${text.slice(0, maximumIssueCodeUnits - 3)}...`
270
+
271
+ const identifiedBy = (value: unknown): string | undefined => {
272
+ if (typeof value !== "object" || value === null) return undefined
273
+ const id = (value as Readonly<Record<string, unknown>>)["id"]
274
+ return typeof id === "string" ? id : undefined
275
+ }
276
+
277
+ const decodeNotification = Schema.decodeUnknownEffect(NotificationModel.Notification)
278
+
279
+ /**
280
+ * Decodes the caller's argument at the durability boundary and returns a value
281
+ * the caller cannot reach.
282
+ *
283
+ * Nothing downstream re-checks the shape: a structurally invalid notification
284
+ * that reached the journal would be acknowledged at a real sequence and then
285
+ * skipped by every replay, which is acknowledged data loss. The depth and size
286
+ * bounds run first so a hostile payload can neither exhaust the stack nor make
287
+ * the copy, the decode, and the fingerprint walk it in full.
288
+ */
289
+ const validated = (
290
+ notification: NotificationModel.Notification
291
+ ): Effect.Effect<NotificationModel.Notification, NotificationError> =>
292
+ Effect.gen(function*() {
293
+ const notificationId = identifiedBy(notification)
294
+ const exceeded = exceededBound(notification)
295
+ if (exceeded !== undefined) {
296
+ return yield* new NotificationError({
297
+ code: "notification_invalid",
298
+ message: exceeded === "depth"
299
+ ? `A notification nests deeper than the ${maximumPayloadDepth} level bound`
300
+ : `A notification is larger than the ${maximumNotificationCodeUnits} code unit bound`,
301
+ ...(notificationId === undefined ? {} : { notificationId })
302
+ })
303
+ }
304
+ return yield* decodeNotification(copied(notification)).pipe(
305
+ Effect.mapError((error) => {
306
+ // The formatter reports at least one issue for a failure and gives
307
+ // every issue a concrete path. Standard Schema's result type widens
308
+ // both, so a fallback for either would be code no input can reach.
309
+ const issue = formatIssue(error.issue).issues[0] as {
310
+ readonly message: string
311
+ readonly path: ReadonlyArray<PropertyKey>
312
+ }
313
+ const path = issue.path.map(String).join(".")
314
+ return new NotificationError({
315
+ code: "notification_invalid",
316
+ message: bounded(issue.message),
317
+ ...(notificationId === undefined ? {} : { notificationId }),
318
+ ...(path === "" ? {} : { path })
319
+ })
320
+ })
321
+ )
322
+ })
323
+
324
+ // Persisted notification JSON predates RFC 8785's rejection of lone surrogates.
325
+ const canonical = (value: unknown): string => canonicalize(value, { loneSurrogates: "escape" })
326
+
327
+ /** Fingerprint the validated input before the journal redacts its payload. */
328
+ const fingerprint = (notification: NotificationModel.Notification): Effect.Effect<string> =>
329
+ Effect.sync(() => digestSync(canonical(notification)))
330
+
331
+ /** Freeze the decoded JSON graph before any queue output can expose it. */
332
+ const freeze = (value: unknown): void => {
333
+ if (typeof value !== "object" || value === null) return
334
+ for (const nested of Object.values(value)) freeze(nested)
335
+ Object.freeze(value)
336
+ }
337
+
338
+ /**
339
+ * One committed admission, as the journal recorded it.
340
+ *
341
+ * This is an identity, not a payload: the fold keeps the fingerprint that
342
+ * settles a duplicate and the sequence the record lives at, and reads the
343
+ * notification itself back from the journal on the rare path that needs it.
344
+ * Retaining every admitted payload for a run's lifetime is what made a
345
+ * capacity-one queue grow without bound.
346
+ */
347
+ interface Committed {
348
+ readonly version: number | undefined
349
+ readonly fingerprint: string | undefined
350
+ readonly decision: NotificationState.AdmissionDecision
351
+ readonly seq: number
352
+ }
353
+
354
+ /**
355
+ * One run's folded notification history.
356
+ *
357
+ * `cursor` is the sequence of the last entry folded, absent when the run has
358
+ * no entries at all. It is what the next read pages from, so a fold costs the
359
+ * entries added since the previous call rather than the run's whole journal.
360
+ */
361
+ interface Loaded {
362
+ readonly state: NotificationState.State
363
+ readonly admissions: HashMap.HashMap<string, Committed>
364
+ /** Drain identity to the sequence its record committed at. */
365
+ readonly promotions: HashMap.HashMap<string, number>
366
+ readonly cursor: number | undefined
367
+ }
368
+
369
+ /**
370
+ * How many runs one layer keeps folded at a time. Evicting is safe: the next
371
+ * call for an evicted run folds it again from the beginning, and the retention
372
+ * policy is scan-resistant, so a supervisor sweeping more runs than this reads
373
+ * journal tails for all but a fixed handful of them.
374
+ */
375
+ const maximumCachedRuns = 64
376
+
377
+ /**
378
+ * Journal-backed production layer, with the pending capacity a composition
379
+ * chooses.
380
+ *
381
+ * Admission and drain evidence is written through `emitDurableUnfenced`, so
382
+ * both return only after the corresponding entry is durably committed and a
383
+ * dropped lifecycle event is unrepresentable. The channel is the unfenced one
384
+ * on purpose: the notifying process owns no run, so there is no ownership
385
+ * fence to hand over, and both records are first-writer-wins on their own
386
+ * identity instead.
387
+ *
388
+ * The capacity guard is a conditional write at the storage layer, not a check
389
+ * against a fold one process happened to load: the fold, the guard, and the
390
+ * admission record are decided and written inside ONE `journal.transact`
391
+ * write transaction, which the durable writer serializes against every other
392
+ * writer of the database and replays when a concurrent writer commits between
393
+ * the guard's reads and the insert. Two processes admitting to one full queue
394
+ * therefore produce one admission and one `rejected-full`, never two
395
+ * admissions.
396
+ *
397
+ * @param options the pending capacity, defaulting to `NotificationState.defaultCapacity`
398
+ * @category layers
399
+ * @since 1.0.0
400
+ */
401
+ export const layerWith = (
402
+ options: { readonly capacity?: number | undefined } = {}
403
+ ): Layer.Layer<NotificationQueue, never, Journal.Journal> =>
404
+ Layer.effect(
405
+ NotificationQueue,
406
+ Effect.gen(function*() {
407
+ const journal = yield* Journal.Journal
408
+ // Orders this layer's own fibers so one process never runs two of its
409
+ // admissions or drains against the fold cache at once. It is
410
+ // process-local by construction and is NOT what enforces capacity or
411
+ // drain identity across processes: those are the conditional writes
412
+ // inside `journal.transact`, decided and committed in one serialized
413
+ // write transaction. Always enter the journal transaction before taking
414
+ // this permit, matching callers already inside an enclosing transaction.
415
+ const operations = yield* Semaphore.make(1)
416
+ const capacity = options.capacity ?? NotificationState.defaultCapacity
417
+ // Only committed folds may be shared. A transaction reads its own
418
+ // inserts, but its read-back is still provisional until the OUTERMOST
419
+ // commit; neither a retry nor a later caller may dedupe against it.
420
+ const folded = FoldCache.make<Loaded>(maximumCachedRuns)
421
+
422
+ /**
423
+ * One entry by the sequence it committed at.
424
+ *
425
+ * The fold keeps identities and sequences rather than payloads, so the
426
+ * paths that must report an admitted notification read it back here.
427
+ */
428
+ const entryAt = (
429
+ runId: JournalEvent.RunId,
430
+ seq: number
431
+ ): Effect.Effect<Option.Option<JournalEvent.Entry>, Journal.JournalError> =>
432
+ Effect.map(
433
+ journal.entries({
434
+ runId,
435
+ ...(seq === 0 ? {} : { after: JournalEvent.Seq.make(seq - 1) }),
436
+ limit: 1
437
+ }),
438
+ (page) => {
439
+ const found = page.entries[0]
440
+ return found !== undefined && found.seq === seq ? Option.some(found) : Option.none()
441
+ }
442
+ )
443
+
444
+ const admittedAt = (
445
+ runId: JournalEvent.RunId,
446
+ seq: number
447
+ ): Effect.Effect<Option.Option<NotificationEvent.Admitted>, Journal.JournalError> =>
448
+ Effect.map(entryAt(runId, seq), (entry) => {
449
+ if (Option.isNone(entry)) return Option.none()
450
+ const decoded = NotificationEvent.fromEntry(entry.value)
451
+ if (Option.isNone(decoded) || !NotificationEvent.isAdmitted(decoded.value)) return Option.none()
452
+ freeze(decoded.value.notification)
453
+ return Option.some(decoded.value)
454
+ })
455
+
456
+ const promotedAt = (
457
+ runId: JournalEvent.RunId,
458
+ seq: number
459
+ ): Effect.Effect<Option.Option<NotificationEvent.Promoted>, Journal.JournalError> =>
460
+ Effect.map(entryAt(runId, seq), (entry) => {
461
+ if (Option.isNone(entry)) return Option.none()
462
+ const decoded = NotificationEvent.fromEntry(entry.value)
463
+ if (Option.isNone(decoded) || !NotificationEvent.isPromoted(decoded.value)) return Option.none()
464
+ return Option.some(decoded.value)
465
+ })
466
+
467
+ const load = (runId: JournalEvent.RunId): Effect.Effect<Loaded, Journal.JournalError> =>
468
+ Effect.gen(function*() {
469
+ const base: Loaded = folded.get(runId) ?? {
470
+ state: NotificationState.empty(capacity),
471
+ admissions: HashMap.empty(),
472
+ promotions: HashMap.empty(),
473
+ cursor: undefined
474
+ }
475
+ const fresh: Array<JournalEvent.Entry> = []
476
+ let after = base.cursor === undefined ? undefined : JournalEvent.Seq.make(base.cursor)
477
+ while (true) {
478
+ // `after` is an exact-optional property upstream, so an explicit
479
+ // `undefined` is not the same as an absent key.
480
+ const page = yield* journal.entries({
481
+ runId,
482
+ eventTypes: [NotificationEvent.AdmittedEventType, NotificationEvent.PromotedEventType],
483
+ ...(after === undefined ? {} : { after }),
484
+ limit: 512
485
+ })
486
+ fresh.push(...page.entries)
487
+ if (!page.hasMore || page.entries.length === 0) break
488
+ after = page.entries.at(-1)!.seq
489
+ }
490
+ if (fresh.length === 0) return base
491
+
492
+ let state = base.state
493
+ let cursor = base.cursor
494
+ // Persistent maps: one event adds a node and leaves the fold the
495
+ // previous load published untouched, where copying both maps would
496
+ // cost the run's whole history on every event and make a long run's
497
+ // folding quadratic in its own admissions.
498
+ let admissions = base.admissions
499
+ let promotions = base.promotions
500
+ for (const entry of fresh) {
501
+ cursor = entry.seq
502
+ const decoded = NotificationEvent.fromEntry(entry)
503
+ if (Option.isNone(decoded)) continue
504
+ const event = decoded.value
505
+ if (NotificationEvent.isAdmitted(event)) {
506
+ freeze(event.notification)
507
+ admissions = HashMap.set(admissions, event.notification.id, {
508
+ version: event.version,
509
+ fingerprint: event.fingerprint,
510
+ decision: event.decision,
511
+ seq: entry.seq
512
+ })
513
+ if (
514
+ event.version !== undefined &&
515
+ state.items.some((item) => item.notification.id === event.notification.id)
516
+ ) {
517
+ state = Object.freeze({
518
+ ...state,
519
+ items: Object.freeze(state.items.map((item) =>
520
+ item.notification.id === event.notification.id
521
+ ? Object.freeze({ notification: event.notification, seq: item.seq })
522
+ : item
523
+ ))
524
+ })
525
+ } else {
526
+ state = NotificationState.applyAdmission(state, event.notification, entry.seq, event.decision)
527
+ }
528
+ } else {
529
+ promotions = HashMap.set(
530
+ promotions,
531
+ NotificationEvent.drainKey(event.targetLineageId, event.boundary),
532
+ entry.seq
533
+ )
534
+ state = NotificationState.applyPromoted(state, event.ids)
535
+ }
536
+ }
537
+
538
+ const loaded: Loaded = { state, admissions, promotions, cursor }
539
+ yield* journal.whenCommitted(Effect.sync(() => folded.put(runId, loaded)))
540
+ return loaded
541
+ })
542
+
543
+ /**
544
+ * One conditional admission: the duplicate check, the capacity guard,
545
+ * and the admission record, decided and written inside ONE journal
546
+ * write transaction.
547
+ *
548
+ * This is the queue's conditional insert, the same read-guarded write
549
+ * the journal itself uses for its dedupe and fence conditions: the
550
+ * guard's reads and the insert share one serialized write transaction,
551
+ * and the durable writer replays the whole body when a concurrent
552
+ * writer commits between them. A decision a stale fold produced can
553
+ * therefore never commit — the replay re-folds against the committed
554
+ * state and a queue that filled meanwhile refuses — however many
555
+ * processes share the database. The process-local `operations`
556
+ * semaphore orders this layer's own fibers only; it is not what
557
+ * enforces capacity across processes.
558
+ */
559
+ const admitInTransaction = (
560
+ runId: JournalEvent.RunId,
561
+ admitted: NotificationModel.Notification,
562
+ admittedFingerprint: string,
563
+ version?: number
564
+ ): Effect.Effect<AdmissionReceipt, Journal.JournalError | NotificationError> =>
565
+ Effect.gen(function*() {
566
+ const loaded = yield* load(runId)
567
+ const prior = Option.getOrUndefined(HashMap.get(loaded.admissions, admitted.id))
568
+ const replacing = prior !== undefined && version !== undefined && version > (prior.version ?? 0)
569
+ if (replacing) {
570
+ const original = yield* admittedAt(runId, prior.seq)
571
+ if (Option.isNone(original)) {
572
+ return yield* new NotificationError({
573
+ code: "notification_unavailable",
574
+ notificationId: admitted.id,
575
+ message: "Original steering input is unreadable"
576
+ })
577
+ }
578
+ const identity = (input: NotificationModel.Notification) => ({ ...input, payload: null })
579
+ if (canonical(identity(original.value.notification)) !== canonical(identity(admitted))) {
580
+ return yield* new NotificationError({
581
+ code: "notification_id_reused",
582
+ notificationId: admitted.id,
583
+ message: "An edit cannot change the notification owner or delivery binding"
584
+ })
585
+ }
586
+ }
587
+ if (replacing && !loaded.state.items.some((item) => item.notification.id === admitted.id)) {
588
+ const original = yield* admittedAt(runId, prior.seq)
589
+ if (Option.isNone(original)) {
590
+ return yield* new NotificationError({
591
+ code: "notification_unavailable",
592
+ message: "Consumed input is unreadable"
593
+ })
594
+ }
595
+ return {
596
+ notificationId: admitted.id,
597
+ decision: prior.decision,
598
+ seq: prior.seq,
599
+ duplicate: true,
600
+ consumed: true,
601
+ consumedNotification: original.value.notification
602
+ }
603
+ }
604
+ if (prior !== undefined && version !== undefined && version < (prior.version ?? 0)) {
605
+ return { notificationId: admitted.id, decision: prior.decision, seq: prior.seq, duplicate: true }
606
+ }
607
+ if (prior !== undefined && !replacing) {
608
+ // Legacy rows have no fingerprint; only their persisted content is
609
+ // available for comparison, and the fold keeps identities rather
610
+ // than payloads, so the original is read back at its sequence.
611
+ // New rows settle on the fingerprint alone and read nothing.
612
+ let same = prior.fingerprint === admittedFingerprint
613
+ if (prior.fingerprint === undefined) {
614
+ const original = yield* admittedAt(runId, prior.seq)
615
+ if (Option.isNone(original)) {
616
+ return yield* new NotificationError({
617
+ code: "notification_unavailable",
618
+ notificationId: admitted.id,
619
+ message: `The admission for notification ${admitted.id} is no longer readable`
620
+ })
621
+ }
622
+ same = canonical(original.value.notification) === canonical(admitted)
623
+ }
624
+ if (!same) {
625
+ return yield* new NotificationError({
626
+ code: "notification_id_reused",
627
+ notificationId: admitted.id,
628
+ message: `Notification id ${admitted.id} was already admitted with different content`
629
+ })
630
+ }
631
+ return {
632
+ notificationId: admitted.id,
633
+ decision: prior.decision,
634
+ seq: prior.seq,
635
+ duplicate: true
636
+ }
637
+ }
638
+
639
+ const admission = replacing ? { decision: "coalesced" as const } : NotificationState.admit(
640
+ loaded.state,
641
+ admitted,
642
+ loaded.cursor === undefined ? 0 : loaded.cursor + 1
643
+ )
644
+ if (admission.decision === "rejected-full") {
645
+ // Nothing is journaled. A rejection recorded as an admission
646
+ // would match on every later attempt and burn the id forever,
647
+ // so the queue refuses in the receipt alone and the caller may
648
+ // admit again once a boundary has drained.
649
+ return {
650
+ notificationId: admitted.id,
651
+ decision: admission.decision,
652
+ seq: undefined,
653
+ duplicate: false
654
+ }
655
+ }
656
+
657
+ const receipt = yield* journal.emitDurableUnfenced(
658
+ new JournalEvent.Input({
659
+ runId,
660
+ sourceId: NotificationEvent.admissionSourceId(admitted.id),
661
+ sourceSeq: JournalEvent.SourceSeq.make(version ?? 0),
662
+ // The sequence is derived from the notification's own id, so
663
+ // a collision IS this admission observed twice. Comparing
664
+ // bytes instead would fail the second writer over `decision`,
665
+ // a field neither caller supplied and each derives from the
666
+ // fill level it happened to load at.
667
+ dedupe: "identity",
668
+ eventType: NotificationEvent.AdmittedEventType,
669
+ payload: {
670
+ notification: admitted,
671
+ decision: admission.decision,
672
+ fingerprint: admittedFingerprint,
673
+ ...(version === undefined ? {} : { version })
674
+ }
675
+ })
676
+ )
677
+ const committed = Option.getOrUndefined(HashMap.get((yield* load(runId)).admissions, admitted.id))
678
+ if (committed === undefined) {
679
+ // Only reachable when the emit deduped against an entry this
680
+ // queue did not write, so nothing was committed and there is
681
+ // no admission to report.
682
+ return yield* new NotificationError({
683
+ code: "notification_id_reused",
684
+ notificationId: admitted.id,
685
+ message: `The journal identity for notification ${admitted.id} holds an event this queue did not write`
686
+ })
687
+ }
688
+ return {
689
+ notificationId: admitted.id,
690
+ decision: committed.decision,
691
+ seq: committed.seq,
692
+ duplicate: receipt._tag === "Duplicate"
693
+ }
694
+ })
695
+
696
+ return make({
697
+ admit: Effect.fn("NotificationQueue.admit")((rawRunId, notification, version) =>
698
+ Effect.gen(function*() {
699
+ if (version !== undefined && (!Number.isSafeInteger(version) || version < 1)) {
700
+ return yield* new NotificationError({
701
+ code: "notification_invalid",
702
+ message: "version must be a positive safe integer"
703
+ })
704
+ }
705
+ const admitted = yield* validated(notification)
706
+ // T-COL-12 stays dark: nothing yet authenticates a watcher burst, pins
707
+ // its coding run or enforces fresh machine reads. Refusing here, before
708
+ // the journal, keeps co-admitted steers deliverable on every replay.
709
+ const payload = admitted.payload
710
+ if (
711
+ typeof payload === "object" && payload !== null && !Array.isArray(payload) &&
712
+ (payload as Readonly<Record<string, unknown>>)["kind"] === "outside_change"
713
+ ) {
714
+ return yield* new NotificationError({
715
+ code: "notification_refused",
716
+ notificationId: admitted.id,
717
+ message: "Outside-change delivery requires authenticated watcher facts and stale-write enforcement"
718
+ })
719
+ }
720
+ const admittedFingerprint = yield* fingerprint(admitted)
721
+ return yield* journal.transact(
722
+ operations.withPermits(1)(
723
+ admitInTransaction(JournalEvent.RunId.make(rawRunId), admitted, admittedFingerprint, version)
724
+ )
725
+ )
726
+ })
727
+ ),
728
+ drain: Effect.fn("NotificationQueue.drain")((input) =>
729
+ journal.transact(
730
+ operations.withPermits(1)(Effect.gen(function*() {
731
+ const runId = JournalEvent.RunId.make(input.runId)
732
+ const key = NotificationEvent.drainKey(input.targetLineageId, input.boundary)
733
+ const loaded = yield* load(runId)
734
+ // A replay reports what the committed record delivered, read
735
+ // back from the journal: the fold holds the identities, not the
736
+ // notifications, so a run's drained payloads are not retained.
737
+ const delivered = (
738
+ record: NotificationEvent.Promoted,
739
+ admissions: HashMap.HashMap<string, Committed>
740
+ ): Effect.Effect<
741
+ ReadonlyArray<NotificationModel.Notification>,
742
+ Journal.JournalError | NotificationError
743
+ > =>
744
+ Effect.forEach(record.ids, (id) =>
745
+ Effect.gen(function*() {
746
+ const committed = HashMap.get(admissions, id)
747
+ const event = Option.isNone(committed)
748
+ ? Option.none<NotificationEvent.Admitted>()
749
+ : yield* admittedAt(runId, committed.value.seq)
750
+ // A committed drain promises this exact delivery. Missing or
751
+ // corrupt payloads cannot become a successful partial replay.
752
+ if (Option.isNone(event)) {
753
+ return yield* new NotificationError({
754
+ code: "notification_unavailable",
755
+ notificationId: id,
756
+ message: `The delivered notification ${id} is no longer readable`
757
+ })
758
+ }
759
+ return event.value.notification
760
+ }))
761
+
762
+ const replayed = (seq: number, admissions: HashMap.HashMap<string, Committed>) =>
763
+ Effect.gen(function*() {
764
+ const record = yield* promotedAt(runId, seq)
765
+ if (Option.isNone(record)) {
766
+ return yield* new NotificationError({
767
+ code: "notification_unavailable",
768
+ message: `The drain record for boundary ${input.boundary} is no longer readable`
769
+ })
770
+ }
771
+ return {
772
+ notifications: yield* delivered(record.value, admissions),
773
+ boundary: input.boundary,
774
+ duplicate: true
775
+ }
776
+ })
777
+
778
+ const prior = HashMap.get(loaded.promotions, key)
779
+ if (Option.isSome(prior)) return yield* replayed(prior.value, loaded.admissions)
780
+
781
+ const cutoff = input.cutoffSeq ?? loaded.cursor ?? 0
782
+ // The existing coding park ladder waits for a person's answer.
783
+ // System facts must remain pending there: promoting them would
784
+ // make the harness treat an outside edit as an answered question.
785
+ // Filtering this projection changes only promoted IDs; the journal
786
+ // fold retains every held admission and its original sequence.
787
+ const promotionState = /^\d+:[a-f0-9]{64}:park:\d+$/.test(input.boundary)
788
+ ? {
789
+ ...loaded.state,
790
+ items: loaded.state.items.filter((item) => {
791
+ const payload = item.notification.payload
792
+ return !(typeof payload === "object" && payload !== null && !Array.isArray(payload) &&
793
+ (payload as Readonly<Record<string, unknown>>).kind === "outside_change")
794
+ })
795
+ }
796
+ : loaded.state
797
+ const steers = NotificationState.promoteSteers(promotionState, cutoff, input.targetLineageId)
798
+ const queued = input.wouldIdle && steers.promoted.length === 0
799
+ ? NotificationState.promoteQueued(steers.state, input.targetLineageId)
800
+ : { state: steers.state, promoted: [] }
801
+ const promoted = [...steers.promoted, ...queued.promoted].map((item) => item.notification)
802
+ const receipt = yield* journal.emitDurableUnfenced(
803
+ new JournalEvent.Input({
804
+ runId,
805
+ sourceId: NotificationEvent.drainSourceId(input.targetLineageId, input.boundary),
806
+ sourceSeq: JournalEvent.SourceSeq.make(0),
807
+ // The identity is the drain: this lineage closing this
808
+ // boundary. Two processes that both reach it have observed one
809
+ // event, and the first record committed is the delivery.
810
+ dedupe: "identity",
811
+ eventType: NotificationEvent.PromotedEventType,
812
+ payload: {
813
+ boundary: input.boundary,
814
+ targetLineageId: input.targetLineageId,
815
+ ids: promoted.map((notification) => notification.id)
816
+ }
817
+ })
818
+ )
819
+ const settled = yield* load(runId)
820
+ const committed = HashMap.get(settled.promotions, key)
821
+ if (Option.isNone(committed)) {
822
+ return yield* new NotificationError({
823
+ code: "notification_unavailable",
824
+ message: `The journal identity for boundary ${input.boundary} holds an event this queue did not write`
825
+ })
826
+ }
827
+ // An accepted emit wrote THIS drain, so the promoted set is the
828
+ // one already in hand and no read-back is owed.
829
+ if (receipt._tag === "Accepted") {
830
+ return { notifications: promoted, boundary: input.boundary, duplicate: false }
831
+ }
832
+ return yield* replayed(committed.value, settled.admissions)
833
+ }))
834
+ )
835
+ ),
836
+ pending: Effect.fn("NotificationQueue.pending")((rawRunId) =>
837
+ Effect.map(
838
+ load(JournalEvent.RunId.make(rawRunId)),
839
+ // The fold `load` already performs: admissions add, promotions
840
+ // remove. Read outside the operations semaphore, because a count
841
+ // taken while a boundary drains is a count taken at some instant
842
+ // either side of it, and blocking a supervisor behind a turn
843
+ // boundary would be the worse answer.
844
+ (loaded) => loaded.state.items.map((item) => item.notification)
845
+ )
846
+ )
847
+ })
848
+ })
849
+ )
850
+
851
+ /**
852
+ * Journal-backed production layer at the default capacity.
853
+ *
854
+ * @category layers
855
+ * @since 0.1.0
856
+ */
857
+ export const layer: Layer.Layer<NotificationQueue, never, Journal.Journal> = layerWith()