@kairos-es/core 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +33 -0
- package/dist/cjs/AppendCondition.js +26 -0
- package/dist/cjs/AppendCondition.js.map +1 -0
- package/dist/cjs/ContractStore.js +80 -0
- package/dist/cjs/ContractStore.js.map +1 -0
- package/dist/cjs/DcbEventStore.js +10 -0
- package/dist/cjs/DcbEventStore.js.map +1 -0
- package/dist/cjs/Event.js +89 -0
- package/dist/cjs/Event.js.map +1 -0
- package/dist/cjs/Projection.js +73 -0
- package/dist/cjs/Projection.js.map +1 -0
- package/dist/cjs/Query.js +221 -0
- package/dist/cjs/Query.js.map +1 -0
- package/dist/cjs/SubscribeMachine.js +235 -0
- package/dist/cjs/SubscribeMachine.js.map +1 -0
- package/dist/cjs/errors.js +22 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/inMemory.js +191 -0
- package/dist/cjs/inMemory.js.map +1 -0
- package/dist/cjs/index.js +124 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/retryOnConflict.js +95 -0
- package/dist/cjs/retryOnConflict.js.map +1 -0
- package/dist/cjs/testing.js +54 -0
- package/dist/cjs/testing.js.map +1 -0
- package/dist/dts/AppendCondition.d.ts +23 -0
- package/dist/dts/AppendCondition.d.ts.map +1 -0
- package/dist/dts/ContractStore.d.ts +56 -0
- package/dist/dts/ContractStore.d.ts.map +1 -0
- package/dist/dts/DcbEventStore.d.ts +64 -0
- package/dist/dts/DcbEventStore.d.ts.map +1 -0
- package/dist/dts/Event.d.ts +104 -0
- package/dist/dts/Event.d.ts.map +1 -0
- package/dist/dts/Projection.d.ts +108 -0
- package/dist/dts/Projection.d.ts.map +1 -0
- package/dist/dts/Query.d.ts +145 -0
- package/dist/dts/Query.d.ts.map +1 -0
- package/dist/dts/SubscribeMachine.d.ts +179 -0
- package/dist/dts/SubscribeMachine.d.ts.map +1 -0
- package/dist/dts/errors.d.ts +17 -0
- package/dist/dts/errors.d.ts.map +1 -0
- package/dist/dts/inMemory.d.ts +76 -0
- package/dist/dts/inMemory.d.ts.map +1 -0
- package/dist/dts/index.d.ts +35 -0
- package/dist/dts/index.d.ts.map +1 -0
- package/dist/dts/retryOnConflict.d.ts +70 -0
- package/dist/dts/retryOnConflict.d.ts.map +1 -0
- package/dist/dts/testing.d.ts +39 -0
- package/dist/dts/testing.d.ts.map +1 -0
- package/dist/esm/AppendCondition.js +19 -0
- package/dist/esm/AppendCondition.js.map +1 -0
- package/dist/esm/ContractStore.js +73 -0
- package/dist/esm/ContractStore.js.map +1 -0
- package/dist/esm/DcbEventStore.js +3 -0
- package/dist/esm/DcbEventStore.js.map +1 -0
- package/dist/esm/Event.js +81 -0
- package/dist/esm/Event.js.map +1 -0
- package/dist/esm/Projection.js +66 -0
- package/dist/esm/Projection.js.map +1 -0
- package/dist/esm/Query.js +209 -0
- package/dist/esm/Query.js.map +1 -0
- package/dist/esm/SubscribeMachine.js +227 -0
- package/dist/esm/SubscribeMachine.js.map +1 -0
- package/dist/esm/errors.js +14 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/inMemory.js +184 -0
- package/dist/esm/inMemory.js.map +1 -0
- package/dist/esm/index.js +38 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/package.json +4 -0
- package/dist/esm/retryOnConflict.js +87 -0
- package/dist/esm/retryOnConflict.js.map +1 -0
- package/dist/esm/testing.js +46 -0
- package/dist/esm/testing.js.map +1 -0
- package/package.json +39 -0
- package/src/AppendCondition.ts +18 -0
- package/src/ContractStore.ts +148 -0
- package/src/DcbEventStore.ts +77 -0
- package/src/Event.ts +135 -0
- package/src/Projection.ts +168 -0
- package/src/Query.ts +237 -0
- package/src/SubscribeMachine.ts +316 -0
- package/src/errors.ts +19 -0
- package/src/inMemory.ts +356 -0
- package/src/index.ts +37 -0
- package/src/retryOnConflict.ts +100 -0
- package/src/testing.ts +47 -0
- package/testing/package.json +6 -0
package/src/Query.ts
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The DCB selection language: `Query` is an OR of `QueryItem`s. Within an item,
|
|
3
|
+
* `types` is an OR-filter (empty means any type) and `tags` is an AND superset
|
|
4
|
+
* test (the event must carry all of the item's tags; extra event tags are
|
|
5
|
+
* fine). The empty query (`items: []`) matches every event.
|
|
6
|
+
*
|
|
7
|
+
* The match predicate is a single pure module function over plain data;
|
|
8
|
+
* `Query` values carry no methods. See `matchesQuery` / `matchesQueryItem`.
|
|
9
|
+
*
|
|
10
|
+
* Smart constructors enforce the indexed fast-path restriction at the
|
|
11
|
+
* constructor surface (not via a runtime throw): every query built through them
|
|
12
|
+
* is the wildcard (`query.all`), a single type-only item (`query.types`), or an
|
|
13
|
+
* OR of tagged items each carrying at least one tag (`query.item` + `query.or`,
|
|
14
|
+
* or `query.tags` for the single-item case). The `query.item` constructor
|
|
15
|
+
* requires a non-empty `tags` at the type level, so there is no public way to
|
|
16
|
+
* build an unrestricted item (empty types and empty tags). `QueryItem`/`Query`
|
|
17
|
+
* are plain data, so internal decision-model plumbing can still assemble any
|
|
18
|
+
* shape directly; the restriction binds the public surface, where consumers
|
|
19
|
+
* build their consistency boundaries.
|
|
20
|
+
*
|
|
21
|
+
* The grammar's DISPATCH lives here too, beside the grammar it dispatches over:
|
|
22
|
+
* `ServableQueryShape` / `classifyServableQuery` own the three shapes an engine
|
|
23
|
+
* serves and the routing of the degenerate empty item.
|
|
24
|
+
*/
|
|
25
|
+
import { Array as Arr, Schema } from 'effect'
|
|
26
|
+
import { type DcbEvent, EventType, type SequencedEvent, Tag } from './Event'
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* One clause of a query: a set of types (OR, empty = any) AND a set of tags
|
|
30
|
+
* (superset test, empty = no tag constraint).
|
|
31
|
+
*/
|
|
32
|
+
export const QueryItem = Schema.Struct({
|
|
33
|
+
types: Schema.Array(EventType),
|
|
34
|
+
tags: Schema.Array(Tag),
|
|
35
|
+
})
|
|
36
|
+
export type QueryItem = typeof QueryItem.Type
|
|
37
|
+
|
|
38
|
+
/** A query: an OR of items. Empty `items` matches everything. */
|
|
39
|
+
export const Query = Schema.Struct({
|
|
40
|
+
items: Schema.Array(QueryItem),
|
|
41
|
+
})
|
|
42
|
+
export type Query = typeof Query.Type
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* True when `event` satisfies `item`: its type is in `item.types` (or the item
|
|
46
|
+
* has no type constraint) AND the event's tags are a superset of `item.tags`.
|
|
47
|
+
*/
|
|
48
|
+
export const matchesQueryItem = (event: DcbEvent, item: QueryItem): boolean => {
|
|
49
|
+
const typeOk = item.types.length === 0 || item.types.includes(event.type)
|
|
50
|
+
if (!typeOk) {
|
|
51
|
+
return false
|
|
52
|
+
}
|
|
53
|
+
const eventTags = new Set<string>(event.tags)
|
|
54
|
+
return item.tags.every((tag) => eventTags.has(tag))
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The one true match predicate (ported from `popo.py:48-54`): an empty query is
|
|
59
|
+
* a wildcard; otherwise the event must match at least one item (OR across
|
|
60
|
+
* items).
|
|
61
|
+
*/
|
|
62
|
+
export const matchesQuery = (event: DcbEvent, query: Query): boolean => {
|
|
63
|
+
if (query.items.length === 0) {
|
|
64
|
+
return true
|
|
65
|
+
}
|
|
66
|
+
return query.items.some((item) => matchesQueryItem(event, item))
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Convenience predicate over a `SequencedEvent`. */
|
|
70
|
+
export const matchesSequenced = (
|
|
71
|
+
sequenced: SequencedEvent,
|
|
72
|
+
query: Query,
|
|
73
|
+
): boolean => matchesQuery(sequenced.event, query)
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Assert `query` is *servable* by the indexed fast paths, throwing a plain
|
|
77
|
+
* `Error` describing the violation otherwise. This enforces at the store
|
|
78
|
+
* boundary the restriction the smart constructors enforce at the type level
|
|
79
|
+
* (`Query.ts` module docs) but which plain-data assembly — e.g. decision-model
|
|
80
|
+
* composition, `Projection.ts:31-36` — can still bypass.
|
|
81
|
+
*
|
|
82
|
+
* The servable grammar is exactly one of:
|
|
83
|
+
* - the wildcard (`items.length === 0`), which matches every event;
|
|
84
|
+
* - a *single* item (`items.length === 1`), any types/tags, including a
|
|
85
|
+
* type-only item; or
|
|
86
|
+
* - a *multi*-item query (`items.length > 1`) in which every item carries at
|
|
87
|
+
* least one tag.
|
|
88
|
+
*
|
|
89
|
+
* A multi-item query that mixes a tagless (type-only) item with tagged items is
|
|
90
|
+
* NOT servable: the by-type path and the by-tag path are indexed differently,
|
|
91
|
+
* so an OR spanning both cannot ride a single index. This is the previously
|
|
92
|
+
* un-enforced edge (`Projection.ts:31-36`).
|
|
93
|
+
*
|
|
94
|
+
* This function is pure (no clock, no RNG, no effect). Every engine applies it at
|
|
95
|
+
* its read/subscribe/append entry points, through the one shared
|
|
96
|
+
* `makeContractStore` decorator, so the rule is inherited rather than restated per
|
|
97
|
+
* backend; a throw inside a store `Effect` becomes a *defect* — a programming
|
|
98
|
+
* error, not an error-channel value — which is the intended "defect in every
|
|
99
|
+
* engine" behaviour.
|
|
100
|
+
*/
|
|
101
|
+
export const assertServableQuery = (query: Query): void => {
|
|
102
|
+
const { items } = query
|
|
103
|
+
// Wildcard and single-item queries are always servable.
|
|
104
|
+
if (items.length <= 1) {
|
|
105
|
+
return
|
|
106
|
+
}
|
|
107
|
+
// A multi-item query is servable only if every item is tagged; a tagless
|
|
108
|
+
// (type-only) item mixed with tagged items cannot ride a single index.
|
|
109
|
+
const hasTaglessItem = items.some((item) => item.tags.length === 0)
|
|
110
|
+
if (hasTaglessItem) {
|
|
111
|
+
throw new Error(
|
|
112
|
+
'assertServableQuery: a multi-item query must have every item carry at ' +
|
|
113
|
+
'least one tag; a type-only item mixed with tagged items is not servable',
|
|
114
|
+
)
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The three shapes a servable query is SERVED by, and therefore the only three an
|
|
120
|
+
* engine's plan compiler dispatches over: the wildcard, a single type-only item,
|
|
121
|
+
* and an all-tagged query. `assertServableQuery` has already refused the mixed
|
|
122
|
+
* tagless-plus-tagged multi-item query, so these three are exhaustive over
|
|
123
|
+
* everything that reaches a store.
|
|
124
|
+
*
|
|
125
|
+
* `typeOnly` carries a NON-EMPTY list of types, and that is the whole guard
|
|
126
|
+
* against the degenerate empty item: see `classifyServableQuery`.
|
|
127
|
+
*/
|
|
128
|
+
export type ServableQueryShape =
|
|
129
|
+
| { readonly kind: 'wildcard' }
|
|
130
|
+
| {
|
|
131
|
+
readonly kind: 'typeOnly'
|
|
132
|
+
readonly types: Arr.NonEmptyReadonlyArray<EventType>
|
|
133
|
+
}
|
|
134
|
+
| { readonly kind: 'tagged' }
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Classify a servable `Query` into the shape that serves it.
|
|
138
|
+
*
|
|
139
|
+
* ## Why the dispatch lives beside the grammar
|
|
140
|
+
*
|
|
141
|
+
* The three shapes ARE the three bullets `assertServableQuery` above admits, so an
|
|
142
|
+
* engine that re-derives the dispatch is a second answer to "which shape is
|
|
143
|
+
* this?", and the shapes only stay in step while there is one answer. Core states
|
|
144
|
+
* the grammar, so stating the dispatch over it here is that one fact once rather
|
|
145
|
+
* than once per backend. It is pure — no clock, no RNG, no effect — and knows
|
|
146
|
+
* nothing of SQL: what a shape COMPILES to stays entirely the engine's.
|
|
147
|
+
*
|
|
148
|
+
* ## The degenerate empty item routes to the WILDCARD, structurally
|
|
149
|
+
*
|
|
150
|
+
* A single item with neither types nor tags — `{ items: [{ types: [], tags: [] }] }`
|
|
151
|
+
* — matches EVERY event: `matchesQueryItem` reads empty types as "any type" and
|
|
152
|
+
* empty tags as a vacuous superset test. It must therefore be served by the
|
|
153
|
+
* wildcard. Served by the type-only path it would compile to a value list with
|
|
154
|
+
* nothing in it, and a membership test against an empty list matches NOTHING — the
|
|
155
|
+
* exact inversion of what the query means. Nothing upstream stops it reaching an
|
|
156
|
+
* engine: `assertServableQuery` admits a SINGLE item of any shape, and the servable
|
|
157
|
+
* grammar forbids a tagless item only in a MULTI-item query, so this dispatch is
|
|
158
|
+
* the last place the divergence can be closed.
|
|
159
|
+
*
|
|
160
|
+
* It is closed by the SHAPE and not by a branch in front of it. `typeOnly` carries
|
|
161
|
+
* `NonEmptyReadonlyArray`, so there is no way to construct it from an empty list;
|
|
162
|
+
* the emptiness test IS the constructor, and deleting it does not silently restore
|
|
163
|
+
* the wrong routing, it stops compiling. A guard clause placed before the
|
|
164
|
+
* construction would have been deletable by anyone tidying what looks like a
|
|
165
|
+
* redundant check on a list that "obviously" has types in it.
|
|
166
|
+
*/
|
|
167
|
+
export const classifyServableQuery = (query: Query): ServableQueryShape => {
|
|
168
|
+
if (query.items.length === 0) {
|
|
169
|
+
return { kind: 'wildcard' }
|
|
170
|
+
}
|
|
171
|
+
const only = query.items.length === 1 ? query.items[0] : undefined
|
|
172
|
+
if (only !== undefined && only.tags.length === 0) {
|
|
173
|
+
return Arr.isNonEmptyReadonlyArray(only.types)
|
|
174
|
+
? { kind: 'typeOnly', types: only.types }
|
|
175
|
+
: { kind: 'wildcard' }
|
|
176
|
+
}
|
|
177
|
+
return { kind: 'tagged' }
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// --- Smart constructors (the fast-path-enforcing surface) -------------------
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* A query item carrying at least one tag (superset test), optionally narrowed
|
|
184
|
+
* to `types`. The non-empty `tags` requirement is the fast-path restriction,
|
|
185
|
+
* enforced at the type level rather than as a runtime throw. Combine several
|
|
186
|
+
* items with `or`.
|
|
187
|
+
*/
|
|
188
|
+
const item = (fields: {
|
|
189
|
+
readonly tags: readonly [Tag, ...ReadonlyArray<Tag>]
|
|
190
|
+
readonly types?: ReadonlyArray<EventType>
|
|
191
|
+
}): QueryItem => ({
|
|
192
|
+
types: fields.types ?? [],
|
|
193
|
+
tags: fields.tags,
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
/** The wildcard query: matches every event. */
|
|
197
|
+
const all = (): Query => ({ items: [] })
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* A single-item query matching events that carry all of `tags`, optionally also
|
|
201
|
+
* constrained to `types`. The common indexed fast-path shape.
|
|
202
|
+
*/
|
|
203
|
+
const tags = (
|
|
204
|
+
tags: readonly [Tag, ...ReadonlyArray<Tag>],
|
|
205
|
+
types?: ReadonlyArray<EventType>,
|
|
206
|
+
): Query => ({
|
|
207
|
+
items: [item({ tags, ...(types === undefined ? {} : { types }) })],
|
|
208
|
+
})
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* A single type-only query (the other indexed fast path): one item constrained
|
|
212
|
+
* to `types` with no tag filter. This is the only tagless item shape the
|
|
213
|
+
* surface admits, and only as the whole query.
|
|
214
|
+
*/
|
|
215
|
+
const types = (
|
|
216
|
+
types: readonly [EventType, ...ReadonlyArray<EventType>],
|
|
217
|
+
): Query => ({ items: [{ types, tags: [] }] })
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* An OR of query items. Each item should be built with `item` (which requires
|
|
221
|
+
* at least one tag); combining bare type-only items is intentionally not
|
|
222
|
+
* offered (use `types` for the single type-only query).
|
|
223
|
+
*/
|
|
224
|
+
const or = (
|
|
225
|
+
items: readonly [QueryItem, ...ReadonlyArray<QueryItem>],
|
|
226
|
+
): Query => ({
|
|
227
|
+
items,
|
|
228
|
+
})
|
|
229
|
+
|
|
230
|
+
/** Namespaced smart constructors for `Query`. */
|
|
231
|
+
export const query = {
|
|
232
|
+
all,
|
|
233
|
+
item,
|
|
234
|
+
or,
|
|
235
|
+
tags,
|
|
236
|
+
types,
|
|
237
|
+
} as const
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The subscribe machine every engine's `subscribe` is built from — the in-memory
|
|
3
|
+
* store, `@kairos-es/store-postgres` and `@kairos-es/store-sqlite`: the wake queue,
|
|
4
|
+
* the wake-source drain, the paged catch-up reconcile, the `> boundary` de-dup (via
|
|
5
|
+
* the cursor's exclusive lower bound) and the poll loop that together turn a paged
|
|
6
|
+
* read into a live subscription.
|
|
7
|
+
*
|
|
8
|
+
* It lives in core because delivery semantics are contract semantics. An engine
|
|
9
|
+
* supplies TRANSPORT through two STUBBABLE seams — `fetch`, a paged read strictly
|
|
10
|
+
* after a cursor, and `listen`, an optional wake source — and inherits the delivery
|
|
11
|
+
* behaviour whole, exactly as it supplies mechanics to `makeContractStore` and
|
|
12
|
+
* inherits the contract policy whole. Two engines that each grew their own wake
|
|
13
|
+
* loop would be identical only by inspection, and `subscribe` is the verb where
|
|
14
|
+
* that is least affordable: a divergence there loses events rather than failing.
|
|
15
|
+
* Both seams take plain in-memory values in a test, so the wake/poll/de-dup logic
|
|
16
|
+
* is exercisable with no live backend at all.
|
|
17
|
+
*
|
|
18
|
+
* All THREE are the callers, and the count matters to a reader here: it means there
|
|
19
|
+
* is exactly ONE delivery algorithm in this repo, and it is the code below. The
|
|
20
|
+
* in-memory store is a caller like the rest — it hands in a `fetch` over its log
|
|
21
|
+
* `Ref` and its append `PubSub` as a wake source and keeps no delivery logic of its
|
|
22
|
+
* own — which is what makes the shared contract suite's subscribe cases a comparison
|
|
23
|
+
* of two BACKENDS rather than of two delivery algorithms.
|
|
24
|
+
*
|
|
25
|
+
* Correctness rests on the periodic position-based reconcile — a re-read from the
|
|
26
|
+
* cursor every poll interval — and NEVER on the wake source, which only lowers
|
|
27
|
+
* delivery latency below that interval. A subscription with no wake source at all
|
|
28
|
+
* is therefore complete and correct, merely slower, which is why `listen` is
|
|
29
|
+
* optional in the TYPE rather than by the convention of passing `Stream.never`:
|
|
30
|
+
* poll-only delivery is a supported shape, not a degraded one.
|
|
31
|
+
*
|
|
32
|
+
* ## The precondition every engine inherits: ASCENDING COMMIT ORDER
|
|
33
|
+
*
|
|
34
|
+
* A position that is committed and VISIBLE must never be below one already
|
|
35
|
+
* delivered. Gaps are fine: the cursor is an ascending exclusive lower bound, so a
|
|
36
|
+
* number that is allocated and never committed, or never allocated at all, is
|
|
37
|
+
* simply a number the machine never sees. Out-of-order commits are not fine. If one
|
|
38
|
+
* writer allocates position 5 and another allocates 6, and 6 commits first, a poll
|
|
39
|
+
* landing between the two commits delivers 6 and advances the cursor past it — and
|
|
40
|
+
* 5 is then lost forever, because no later read looks below the cursor and nothing
|
|
41
|
+
* anywhere notices the hole. An engine whose concurrent writers can commit out of
|
|
42
|
+
* order MUST serialise them, or this machine will silently skip. An engine that
|
|
43
|
+
* takes an exclusive lock on both of its append paths discharges the obligation
|
|
44
|
+
* that way; an engine whose append is a single statement discharges it
|
|
45
|
+
* structurally.
|
|
46
|
+
*
|
|
47
|
+
* The obligation is written down HERE, in the algorithm that depends on it, rather
|
|
48
|
+
* than in whichever engine happens to satisfy it. It is a property of this delivery
|
|
49
|
+
* loop and of nothing else, so an engine author reading only their own source has
|
|
50
|
+
* no way to discover that anything downstream needs it — and the failure mode is
|
|
51
|
+
* the worst kind there is: no error, no defect, no failing test, just an event that
|
|
52
|
+
* a subscriber never receives. A single engine owning both its append path and its
|
|
53
|
+
* own wake loop can leave the assumption implicit; a shared machine cannot.
|
|
54
|
+
*
|
|
55
|
+
* ## `catchUpPageSize` bounds the READ, not the emission
|
|
56
|
+
*
|
|
57
|
+
* `drainFrom` below PAGES the catch-up read — `catchUpPageSize` bounds each
|
|
58
|
+
* `fetch` — but it COLLECTS every page into one array, which is emitted as a SINGLE
|
|
59
|
+
* chunk. So peak memory at subscription start is O(backlog) rather than O(page): a
|
|
60
|
+
* subscriber joining behind a large log materialises the whole backlog before its
|
|
61
|
+
* consumer sees the first element.
|
|
62
|
+
*
|
|
63
|
+
* That is a memory characteristic and nothing more, which is why ONE chunk is
|
|
64
|
+
* enough here: downstream commit boundaries do not depend on the upstream chunking
|
|
65
|
+
* at all, because the read side's `Stream.groupedWithin` re-chunks the stream into
|
|
66
|
+
* bounded micro-batches whatever arrives (ADR-0007), so a projection's transactions
|
|
67
|
+
* stay small even against a one-chunk catch-up. The cost therefore lands on backlog
|
|
68
|
+
* memory alone, and it lands hardest on the durable-log-with-in-memory-view mode,
|
|
69
|
+
* where an ephemeral view re-catches-up the entire log on every boot.
|
|
70
|
+
*/
|
|
71
|
+
import {
|
|
72
|
+
Chunk,
|
|
73
|
+
Effect,
|
|
74
|
+
Option,
|
|
75
|
+
Queue,
|
|
76
|
+
Schedule,
|
|
77
|
+
Schema,
|
|
78
|
+
type Scope,
|
|
79
|
+
Stream,
|
|
80
|
+
} from 'effect'
|
|
81
|
+
import type { Position, SequencedEvent } from './Event'
|
|
82
|
+
import { ORIGIN, ReadLimit } from './Event'
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* A short catch-up page size default for the reconcile loop. Reads page by page
|
|
86
|
+
* until a short page signals the tail is drained, so a large backlog is streamed in
|
|
87
|
+
* bounded chunks rather than one unbounded read. Every engine takes its
|
|
88
|
+
* `catchUpPageSize` from here — each SQL engine as its config's default, the
|
|
89
|
+
* in-memory store as its options' — so backends cannot drift on tuning.
|
|
90
|
+
*/
|
|
91
|
+
export const DEFAULT_CATCH_UP_PAGE_SIZE = 512
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The default upper bound a subscriber waits for a wake before re-reading from its
|
|
95
|
+
* cursor ANYWAY (milliseconds). Every engine takes its `pollInterval` from here —
|
|
96
|
+
* each SQL engine as its config's default, the in-memory store as its options' — so
|
|
97
|
+
* backends cannot drift on tuning.
|
|
98
|
+
*
|
|
99
|
+
* This periodic position-based poll — not the wake source — IS the correctness
|
|
100
|
+
* guarantee: every committed position is reconciled within this interval whether or
|
|
101
|
+
* not a wake for it was ever heard. That covers a wake fired before the source was
|
|
102
|
+
* fully established, a coalesced or dropped wake, and a wake source that died
|
|
103
|
+
* silently and never said so. A wake source only lowers delivery latency BELOW this
|
|
104
|
+
* interval; it never carries delivery, so no engine's wake mechanism can make the
|
|
105
|
+
* right figure here engine-specific. It trades worst-case delivery latency against
|
|
106
|
+
* idle read volume and nothing else. Upstream `postgres_tt.py` picks the same
|
|
107
|
+
* one-second fallback poll (`conn.wait(..., interval=1)`).
|
|
108
|
+
*/
|
|
109
|
+
export const DEFAULT_POLL_INTERVAL_MILLIS = 1000
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* The two subscribe-tuning config FIELDS, for an engine whose config is a `Schema`
|
|
113
|
+
* to spread into its own struct.
|
|
114
|
+
*
|
|
115
|
+
* These knobs are the machine's, not any engine's: they name what the loop above
|
|
116
|
+
* does, the figures they fall back to are the two constants declared beside them,
|
|
117
|
+
* and neither is derivable from a backend's own properties — which is why the
|
|
118
|
+
* defaults already live here. Shipping the FIELD rather than only the default is
|
|
119
|
+
* what closes the last gap in that arrangement. An engine that spreads these gets
|
|
120
|
+
* the key NAME, the VALIDATION and the DEFAULT from one place, so a caller who has
|
|
121
|
+
* tuned one engine's subscribe reads another's without translating, and the two
|
|
122
|
+
* cannot drift in what they ACCEPT any more than in what they assume. Two engines
|
|
123
|
+
* hand-authoring the identical `Schema.optionalWith(…)` twice were identical only
|
|
124
|
+
* by inspection.
|
|
125
|
+
*
|
|
126
|
+
* The bounds differ between the two because the quantities do. `pollInterval` is a
|
|
127
|
+
* duration, so a fraction of a millisecond is meaningful and only `positive` and
|
|
128
|
+
* `finite` matter: at zero or below the loop would spin without ever waiting for a
|
|
129
|
+
* wake, and at `Infinity` the periodic re-read — the half that carries delivery —
|
|
130
|
+
* would never come, which silently converts the wake source from an optimisation
|
|
131
|
+
* into the guarantee. `catchUpPageSize` is a COUNT, so it must be an integer
|
|
132
|
+
* (`Schema.Int` excludes `Infinity` with it): the reconcile loop compares a page's
|
|
133
|
+
* length against it to decide the tail is drained, and a page of zero would report
|
|
134
|
+
* "drained" forever without reading anything.
|
|
135
|
+
*
|
|
136
|
+
* `exact: true` on both keeps the decoded type at `key?: T` rather than
|
|
137
|
+
* `T | undefined`, matching the repo's `exactOptionalPropertyTypes` discipline — an
|
|
138
|
+
* absent key, never an explicit `undefined`.
|
|
139
|
+
*/
|
|
140
|
+
export const subscriptionTuningFields = {
|
|
141
|
+
pollInterval: Schema.optionalWith(
|
|
142
|
+
Schema.Number.pipe(Schema.positive(), Schema.finite()),
|
|
143
|
+
{ exact: true },
|
|
144
|
+
),
|
|
145
|
+
catchUpPageSize: Schema.optionalWith(Schema.Int.pipe(Schema.positive()), {
|
|
146
|
+
exact: true,
|
|
147
|
+
}),
|
|
148
|
+
} as const
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Retry a TRANSIENT poll fault (an `E` from one reconcile read) on a capped
|
|
152
|
+
* exponential backoff rather than letting `Stream.orDie` kill a subscription that
|
|
153
|
+
* may have been alive for days over a single connection blip. A fault that PERSISTS
|
|
154
|
+
* past the cap propagates (the decorator dies it) — infrastructure stays off the
|
|
155
|
+
* contract channel while the stream survives ordinary transients.
|
|
156
|
+
*/
|
|
157
|
+
const POLL_RETRY_SCHEDULE = Schedule.exponential('100 millis', 2).pipe(
|
|
158
|
+
Schedule.union(Schedule.spaced('2 seconds')),
|
|
159
|
+
Schedule.intersect(Schedule.recurs(20)),
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The two seams the subscribe machine runs over, plus the two tuning knobs. `E` is
|
|
164
|
+
* the engine's own infrastructure error — whatever its reads fail with — and it
|
|
165
|
+
* stays on the channel through to `makeContractStore`, which dies it.
|
|
166
|
+
*
|
|
167
|
+
* ONE type parameter and not two, deliberately: a wake source with an error type of
|
|
168
|
+
* its own is `Stream.orDie`d by the caller, yielding a `Stream<unknown, never>`
|
|
169
|
+
* that is assignable here under Effect v3's covariant error channel. A second
|
|
170
|
+
* parameter would buy nothing and would have to be spelled at every call site. Nor
|
|
171
|
+
* is there an `R`: neither seam contributes a requirement, and unused generality is
|
|
172
|
+
* not free.
|
|
173
|
+
*/
|
|
174
|
+
export interface SubscriptionDeps<E> {
|
|
175
|
+
/**
|
|
176
|
+
* Page-read the events strictly after `from`, up to `limit`, ascending by
|
|
177
|
+
* position. Returning FEWER than `limit` signals the tail is drained.
|
|
178
|
+
*/
|
|
179
|
+
readonly fetch: (
|
|
180
|
+
from: Position,
|
|
181
|
+
limit: ReadLimit,
|
|
182
|
+
) => Effect.Effect<ReadonlyArray<SequencedEvent>, E>
|
|
183
|
+
/**
|
|
184
|
+
* The raw wake source: each element is an OPAQUE wake signal. The machine taps it
|
|
185
|
+
* into its own wake queue and drains it on a background fibre; it never reads the
|
|
186
|
+
* elements, so anything an engine can push on a change will do.
|
|
187
|
+
*
|
|
188
|
+
* OPTIONAL, and omitting it defaults to `Stream.never` at the point of use — an
|
|
189
|
+
* engine with no wake mechanism gets poll-only delivery, which the poll loop
|
|
190
|
+
* already makes correct. Under `exactOptionalPropertyTypes` that means omitting
|
|
191
|
+
* the key entirely rather than passing `undefined`.
|
|
192
|
+
*/
|
|
193
|
+
readonly listen?: Stream.Stream<unknown, E>
|
|
194
|
+
/** The poll interval in milliseconds (see `DEFAULT_POLL_INTERVAL_MILLIS`). */
|
|
195
|
+
readonly pollInterval: number
|
|
196
|
+
/** The catch-up page size (see `DEFAULT_CATCH_UP_PAGE_SIZE`). */
|
|
197
|
+
readonly catchUpPageSize: number
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Build the subscription `Stream`: catch up from `after`, then track the live tail.
|
|
202
|
+
* The cursor threads through the `unfoldChunkEffect` unfold state, starting at
|
|
203
|
+
* `after` and advancing to the last delivered position each batch; the stream never
|
|
204
|
+
* completes on its own (always `Option.some`), so the enclosing scope interrupts it
|
|
205
|
+
* on close. An `E` mid-stream stays on the channel; `makeContractStore` dies it
|
|
206
|
+
* (subscribe is `E = never` on the contract).
|
|
207
|
+
*/
|
|
208
|
+
export const subscribeStream = <E>(
|
|
209
|
+
deps: SubscriptionDeps<E>,
|
|
210
|
+
after: Position | undefined,
|
|
211
|
+
): Stream.Stream<SequencedEvent, E, Scope.Scope> =>
|
|
212
|
+
Stream.unwrapScoped(
|
|
213
|
+
Effect.gen(function* () {
|
|
214
|
+
// The wake primitive: a sliding(1) queue holding at most one pending
|
|
215
|
+
// signal. The wake source `offer`s an opaque wake on every notification;
|
|
216
|
+
// the poll loop `take`s (with a timeout). The lost-wakeup guard is
|
|
217
|
+
// STRUCTURAL — drain the queue BEFORE reading, so a wake that lands
|
|
218
|
+
// mid-read leaves a fresh signal that makes the next `take` return at once
|
|
219
|
+
// rather than sleeping through a real change.
|
|
220
|
+
const wake = yield* Queue.sliding<void>(1)
|
|
221
|
+
|
|
222
|
+
// Tap every notification into the wake queue. If the wake source DOES
|
|
223
|
+
// surface an error, re-subscribe with capped-exponential backoff. Many wake
|
|
224
|
+
// sources do not error on a dropped connection at all — they simply go
|
|
225
|
+
// quiet — so this retry is NOT the reconnect guarantee: the poll fallback
|
|
226
|
+
// below is. The reconcile loop re-reads from the cursor on every poll, so a
|
|
227
|
+
// silently-dropped (and later re-established) wake source loses no
|
|
228
|
+
// positions. Absent a wake source entirely, `Stream.never` makes this a
|
|
229
|
+
// fibre that parks forever and delivery falls to the poll alone.
|
|
230
|
+
const listener = (deps.listen ?? Stream.never).pipe(
|
|
231
|
+
Stream.tap(() => Queue.offer(wake, undefined)),
|
|
232
|
+
Stream.retry(
|
|
233
|
+
Schedule.exponential('50 millis').pipe(
|
|
234
|
+
Schedule.union(Schedule.spaced('1 seconds')),
|
|
235
|
+
),
|
|
236
|
+
),
|
|
237
|
+
)
|
|
238
|
+
// Drain the wake source into the wake queue on a background fibre; the
|
|
239
|
+
// scope releases it. We never read its elements directly — every
|
|
240
|
+
// notification is an opaque wake signal.
|
|
241
|
+
yield* listener.pipe(Stream.runDrain, Effect.forkScoped)
|
|
242
|
+
|
|
243
|
+
// Reconcile: page-read from `from` until a short page, advancing to the last
|
|
244
|
+
// delivered position. Retries a TRANSIENT poll fault on a capped schedule so
|
|
245
|
+
// a single connection blip does not kill a days-long subscription; a
|
|
246
|
+
// persistent fault propagates and dies.
|
|
247
|
+
const drainFrom = (
|
|
248
|
+
from: Position,
|
|
249
|
+
): Effect.Effect<
|
|
250
|
+
{
|
|
251
|
+
readonly events: ReadonlyArray<SequencedEvent>
|
|
252
|
+
readonly last: Position
|
|
253
|
+
},
|
|
254
|
+
E
|
|
255
|
+
> =>
|
|
256
|
+
Effect.gen(function* () {
|
|
257
|
+
const collected: Array<SequencedEvent> = []
|
|
258
|
+
let cursor = from
|
|
259
|
+
// Loop pages until a short page signals the tail is drained.
|
|
260
|
+
while (true) {
|
|
261
|
+
const events = yield* deps.fetch(
|
|
262
|
+
cursor,
|
|
263
|
+
ReadLimit.make(deps.catchUpPageSize),
|
|
264
|
+
)
|
|
265
|
+
for (const event of events) {
|
|
266
|
+
collected.push(event)
|
|
267
|
+
}
|
|
268
|
+
const lastEvent = events.at(-1)
|
|
269
|
+
if (
|
|
270
|
+
lastEvent === undefined ||
|
|
271
|
+
events.length < deps.catchUpPageSize
|
|
272
|
+
) {
|
|
273
|
+
break
|
|
274
|
+
}
|
|
275
|
+
cursor = lastEvent.position
|
|
276
|
+
}
|
|
277
|
+
const last = collected.at(-1)?.position ?? from
|
|
278
|
+
return { events: collected, last }
|
|
279
|
+
}).pipe(Effect.retry(POLL_RETRY_SCHEDULE))
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Produce the NEXT non-empty batch from a cursor position, together with
|
|
283
|
+
* the advanced cursor — a pure `Position → Effect<[Chunk, next]>`. The
|
|
284
|
+
* `> boundary` de-dup is enforced by `fetch`'s exclusive lower bound, so a
|
|
285
|
+
* live re-read after a wake never re-emits an already-delivered position.
|
|
286
|
+
*
|
|
287
|
+
* Drain the wake queue BEFORE reading (lost-wakeup guard), then drain from
|
|
288
|
+
* the cursor. If a drain yields nothing, wait for a wake but only up to the
|
|
289
|
+
* poll interval, then loop and re-read ANYWAY — the periodic position-based
|
|
290
|
+
* poll that makes delivery correct independently of any wake. This effect
|
|
291
|
+
* only completes by returning a non-empty chunk; the scope interrupts it on
|
|
292
|
+
* close.
|
|
293
|
+
*/
|
|
294
|
+
const nextBatch = (
|
|
295
|
+
from: Position,
|
|
296
|
+
): Effect.Effect<readonly [Chunk.Chunk<SequencedEvent>, Position], E> =>
|
|
297
|
+
Effect.gen(function* () {
|
|
298
|
+
while (true) {
|
|
299
|
+
yield* Queue.takeAll(wake)
|
|
300
|
+
const { events, last } = yield* drainFrom(from)
|
|
301
|
+
if (events.length > 0) {
|
|
302
|
+
return [Chunk.fromIterable(events), last] as const
|
|
303
|
+
}
|
|
304
|
+
// Nothing new: block on a wake, but only up to the poll interval — on
|
|
305
|
+
// timeout we fall through and the loop re-reads regardless.
|
|
306
|
+
yield* Effect.ignore(
|
|
307
|
+
Effect.timeout(Queue.take(wake), deps.pollInterval),
|
|
308
|
+
)
|
|
309
|
+
}
|
|
310
|
+
})
|
|
311
|
+
|
|
312
|
+
return Stream.unfoldChunkEffect(after ?? ORIGIN, (from) =>
|
|
313
|
+
Effect.map(nextBatch(from), Option.some),
|
|
314
|
+
)
|
|
315
|
+
}),
|
|
316
|
+
)
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tagged errors for the store contract. Conflicts are values on the error
|
|
3
|
+
* channel, never defects.
|
|
4
|
+
*/
|
|
5
|
+
import { Data } from 'effect'
|
|
6
|
+
import type { AppendCondition } from './AppendCondition'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Raised by `append` when the append condition's `failIfEventsMatch` query
|
|
10
|
+
* matches at least one event after `condition.after`. Carries only the
|
|
11
|
+
* offending condition: a correct retry re-reads the store to its new head and
|
|
12
|
+
* rebuilds its condition from there, so no conflicting position is carried (it
|
|
13
|
+
* would already be stale by the time it was acted on).
|
|
14
|
+
*/
|
|
15
|
+
export class AppendConditionFailed extends Data.TaggedError(
|
|
16
|
+
'AppendConditionFailed',
|
|
17
|
+
)<{
|
|
18
|
+
readonly condition: AppendCondition
|
|
19
|
+
}> {}
|