@kairos-es/feed-postgres 0.0.0-stage → 1.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +84 -2
- package/dist/cjs/config.js +46 -0
- package/dist/cjs/config.js.map +1 -0
- package/dist/cjs/ensureFeedSchema.js +39 -0
- package/dist/cjs/ensureFeedSchema.js.map +1 -0
- package/dist/cjs/feed.js +176 -0
- package/dist/cjs/feed.js.map +1 -0
- package/dist/cjs/index.js +84 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/internal/context.js +45 -0
- package/dist/cjs/internal/context.js.map +1 -0
- package/dist/cjs/internal/ddl.js +149 -0
- package/dist/cjs/internal/ddl.js.map +1 -0
- package/dist/cjs/internal/statements.js +270 -0
- package/dist/cjs/internal/statements.js.map +1 -0
- package/dist/cjs/lease.js +342 -0
- package/dist/cjs/lease.js.map +1 -0
- package/dist/cjs/syncStore.js +179 -0
- package/dist/cjs/syncStore.js.map +1 -0
- package/dist/cjs/withFeed.js +51 -0
- package/dist/cjs/withFeed.js.map +1 -0
- package/dist/dts-cjs/config.d.ts +37 -0
- package/dist/dts-cjs/config.d.ts.map +1 -0
- package/dist/dts-cjs/ensureFeedSchema.d.ts +24 -0
- package/dist/dts-cjs/ensureFeedSchema.d.ts.map +1 -0
- package/dist/dts-cjs/feed.d.ts +81 -0
- package/dist/dts-cjs/feed.d.ts.map +1 -0
- package/dist/dts-cjs/index.d.ts +50 -0
- package/dist/dts-cjs/index.d.ts.map +1 -0
- package/dist/dts-cjs/internal/context.d.ts +28 -0
- package/dist/dts-cjs/internal/context.d.ts.map +1 -0
- package/dist/dts-cjs/internal/ddl.d.ts +99 -0
- package/dist/dts-cjs/internal/ddl.d.ts.map +1 -0
- package/dist/dts-cjs/internal/statements.d.ts +127 -0
- package/dist/dts-cjs/internal/statements.d.ts.map +1 -0
- package/dist/dts-cjs/lease.d.ts +138 -0
- package/dist/dts-cjs/lease.d.ts.map +1 -0
- package/dist/dts-cjs/package.json +3 -0
- package/dist/dts-cjs/syncStore.d.ts +57 -0
- package/dist/dts-cjs/syncStore.d.ts.map +1 -0
- package/dist/dts-cjs/withFeed.d.ts +69 -0
- package/dist/dts-cjs/withFeed.d.ts.map +1 -0
- package/dist/dts-esm/config.d.ts +37 -0
- package/dist/dts-esm/config.d.ts.map +1 -0
- package/dist/dts-esm/ensureFeedSchema.d.ts +24 -0
- package/dist/dts-esm/ensureFeedSchema.d.ts.map +1 -0
- package/dist/dts-esm/feed.d.ts +81 -0
- package/dist/dts-esm/feed.d.ts.map +1 -0
- package/dist/dts-esm/index.d.ts +50 -0
- package/dist/dts-esm/index.d.ts.map +1 -0
- package/dist/dts-esm/internal/context.d.ts +28 -0
- package/dist/dts-esm/internal/context.d.ts.map +1 -0
- package/dist/dts-esm/internal/ddl.d.ts +99 -0
- package/dist/dts-esm/internal/ddl.d.ts.map +1 -0
- package/dist/dts-esm/internal/statements.d.ts +127 -0
- package/dist/dts-esm/internal/statements.d.ts.map +1 -0
- package/dist/dts-esm/lease.d.ts +138 -0
- package/dist/dts-esm/lease.d.ts.map +1 -0
- package/dist/dts-esm/package.json +3 -0
- package/dist/dts-esm/syncStore.d.ts +57 -0
- package/dist/dts-esm/syncStore.d.ts.map +1 -0
- package/dist/dts-esm/withFeed.d.ts +69 -0
- package/dist/dts-esm/withFeed.d.ts.map +1 -0
- package/dist/esm/config.js +39 -0
- package/dist/esm/config.js.map +1 -0
- package/dist/esm/ensureFeedSchema.js +31 -0
- package/dist/esm/ensureFeedSchema.js.map +1 -0
- package/dist/esm/feed.js +166 -0
- package/dist/esm/feed.js.map +1 -0
- package/dist/esm/index.js +50 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/context.js +34 -0
- package/dist/esm/internal/context.js.map +1 -0
- package/dist/esm/internal/ddl.js +136 -0
- package/dist/esm/internal/ddl.js.map +1 -0
- package/dist/esm/internal/statements.js +262 -0
- package/dist/esm/internal/statements.js.map +1 -0
- package/dist/esm/lease.js +330 -0
- package/dist/esm/lease.js.map +1 -0
- package/dist/esm/package.json +4 -0
- package/dist/esm/syncStore.js +171 -0
- package/dist/esm/syncStore.js.map +1 -0
- package/dist/esm/withFeed.js +44 -0
- package/dist/esm/withFeed.js.map +1 -0
- package/package.json +35 -3
- package/src/config.ts +46 -0
- package/src/ensureFeedSchema.ts +32 -0
- package/src/feed.ts +284 -0
- package/src/index.ts +67 -0
- package/src/internal/context.ts +44 -0
- package/src/internal/ddl.ts +233 -0
- package/src/internal/statements.ts +387 -0
- package/src/lease.ts +503 -0
- package/src/syncStore.ts +340 -0
- package/src/withFeed.ts +148 -0
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Internal object naming and DDL for the Postgres change feed.
|
|
3
|
+
*
|
|
4
|
+
* The single source of truth for the FIVE objects this package owns — the feed
|
|
5
|
+
* rows, the sequence they share, the generations, the sync cursors and the parked
|
|
6
|
+
* list — and for the idempotent DDL that creates them. Every name is derived ONCE
|
|
7
|
+
* from `{ schema, tablePrefix }` and reused by `ensureFeedSchema` (the migration
|
|
8
|
+
* payload) and by every runtime statement in `statements.ts`, so a migration and a
|
|
9
|
+
* query can never disagree about what an object is called.
|
|
10
|
+
*
|
|
11
|
+
* The pair is the SAME one the event store and `@kairos-es/read-postgres` take, so
|
|
12
|
+
* the feed's tables land beside the log and the checkpoints they belong with, and
|
|
13
|
+
* one prefix keeps naming one complete, isolated world (ADR-0002).
|
|
14
|
+
*
|
|
15
|
+
* The identifier discipline is `@kairos-es/read-postgres`'s, duplicated rather
|
|
16
|
+
* than imported for the reason that package gives for duplicating the event
|
|
17
|
+
* store's: each package measures and quotes only the names IT materialises, and
|
|
18
|
+
* importing a sibling backend's internals would couple two packages that must
|
|
19
|
+
* stay independently publishable. Identifiers are pre-quoted here and embedded as
|
|
20
|
+
* literal text into `sql.unsafe`; values are always bound `$n` parameters. Names
|
|
21
|
+
* never pass through `sql(identifier)`, because the consumer's client may carry a
|
|
22
|
+
* query-name transform that would rewrite them.
|
|
23
|
+
*/
|
|
24
|
+
import type { SqlClient, Statement } from 'effect/sql'
|
|
25
|
+
|
|
26
|
+
/** Every object a feed config owns, as qualified (unquoted) names. */
|
|
27
|
+
export interface ResolvedFeedNames {
|
|
28
|
+
readonly schema: string
|
|
29
|
+
readonly tablePrefix: string
|
|
30
|
+
/** `<schema>.<prefix>_feed_rows`: one row per feed and key, the newest state. */
|
|
31
|
+
readonly rowsTable: string
|
|
32
|
+
/** `<prefix>_feed_rows_seq_idx`: the `(feed, seq)` index the tail reads by. */
|
|
33
|
+
readonly rowsSeqIndex: string
|
|
34
|
+
/** `<schema>.<prefix>_feed_seq`: the one sequence every feed in the prefix shares. */
|
|
35
|
+
readonly sequence: string
|
|
36
|
+
/** `<schema>.<prefix>_feed_generations`: each feed's generation and binding. */
|
|
37
|
+
readonly generationsTable: string
|
|
38
|
+
/** `<schema>.<prefix>_feed_cursors`: one cursor and lease per feed and sink. */
|
|
39
|
+
readonly cursorsTable: string
|
|
40
|
+
/** `<schema>.<prefix>_feed_parked`: the records a sink refused. */
|
|
41
|
+
readonly parkedTable: string
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** The default schema, identical to the event store's and the read side's. */
|
|
45
|
+
export const DEFAULT_SCHEMA = 'public'
|
|
46
|
+
/** The default table prefix, identical to the event store's and the read side's. */
|
|
47
|
+
export const DEFAULT_TABLE_PREFIX = 'dcb_events'
|
|
48
|
+
|
|
49
|
+
const MAX_IDENTIFIER_BYTES = 63
|
|
50
|
+
|
|
51
|
+
const utf8ByteLength = (value: string): number =>
|
|
52
|
+
new TextEncoder().encode(value).length
|
|
53
|
+
|
|
54
|
+
const suffixes = {
|
|
55
|
+
rows: '_feed_rows',
|
|
56
|
+
rowsSeqIndex: '_feed_rows_seq_idx',
|
|
57
|
+
sequence: '_feed_seq',
|
|
58
|
+
generations: '_feed_generations',
|
|
59
|
+
cursors: '_feed_cursors',
|
|
60
|
+
parked: '_feed_parked',
|
|
61
|
+
} as const
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The single-atom identifiers Postgres materialises for a config. The index is
|
|
65
|
+
* measured, unlike a primary key's implicit one, because it is named explicitly
|
|
66
|
+
* and `CREATE INDEX IF NOT EXISTS` would skip a truncated name that collides with
|
|
67
|
+
* another prefix's index instead of reporting it.
|
|
68
|
+
*/
|
|
69
|
+
const derivedAtoms = (config: {
|
|
70
|
+
readonly schema?: string
|
|
71
|
+
readonly tablePrefix?: string
|
|
72
|
+
}): ReadonlyArray<readonly [string, string]> => {
|
|
73
|
+
const schema = config.schema ?? DEFAULT_SCHEMA
|
|
74
|
+
const prefix = config.tablePrefix ?? DEFAULT_TABLE_PREFIX
|
|
75
|
+
return [
|
|
76
|
+
['schema', schema],
|
|
77
|
+
['feed rows table', `${prefix}${suffixes.rows}`],
|
|
78
|
+
['feed rows index', `${prefix}${suffixes.rowsSeqIndex}`],
|
|
79
|
+
['feed sequence', `${prefix}${suffixes.sequence}`],
|
|
80
|
+
['feed generations table', `${prefix}${suffixes.generations}`],
|
|
81
|
+
['feed cursors table', `${prefix}${suffixes.cursors}`],
|
|
82
|
+
['feed parked table', `${prefix}${suffixes.parked}`],
|
|
83
|
+
]
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The first derived identifier over Postgres's 63-byte limit, as a message, or
|
|
88
|
+
* `undefined` when every name fits. Postgres truncates an over-long identifier
|
|
89
|
+
* with only a notice, so two prefixes sharing a 63-byte stem would silently share
|
|
90
|
+
* one feed table; refusing at the boundary beats that.
|
|
91
|
+
*/
|
|
92
|
+
export const oversizedIdentifier = (config: {
|
|
93
|
+
readonly schema?: string
|
|
94
|
+
readonly tablePrefix?: string
|
|
95
|
+
}): string | undefined => {
|
|
96
|
+
for (const [label, atom] of derivedAtoms(config)) {
|
|
97
|
+
const bytes = utf8ByteLength(atom)
|
|
98
|
+
if (bytes > MAX_IDENTIFIER_BYTES) {
|
|
99
|
+
return `the ${label} identifier "${atom}" is ${bytes} bytes, over Postgres's ${MAX_IDENTIFIER_BYTES}-byte limit; it would be silently truncated, letting two prefixes share one feed object`
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return undefined
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Derive every name from the pair; throws on an over-long identifier. */
|
|
106
|
+
export const resolveFeedNames = (config: {
|
|
107
|
+
readonly schema?: string
|
|
108
|
+
readonly tablePrefix?: string
|
|
109
|
+
}): ResolvedFeedNames => {
|
|
110
|
+
const problem = oversizedIdentifier(config)
|
|
111
|
+
if (problem !== undefined) {
|
|
112
|
+
throw new Error(`resolveFeedNames: ${problem}`)
|
|
113
|
+
}
|
|
114
|
+
const schema = config.schema ?? DEFAULT_SCHEMA
|
|
115
|
+
const tablePrefix = config.tablePrefix ?? DEFAULT_TABLE_PREFIX
|
|
116
|
+
const qualified = (suffix: string) => `${schema}.${tablePrefix}${suffix}`
|
|
117
|
+
return {
|
|
118
|
+
schema,
|
|
119
|
+
tablePrefix,
|
|
120
|
+
rowsTable: qualified(suffixes.rows),
|
|
121
|
+
rowsSeqIndex: `${tablePrefix}${suffixes.rowsSeqIndex}`,
|
|
122
|
+
sequence: qualified(suffixes.sequence),
|
|
123
|
+
generationsTable: qualified(suffixes.generations),
|
|
124
|
+
cursorsTable: qualified(suffixes.cursors),
|
|
125
|
+
parkedTable: qualified(suffixes.parked),
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Quote a possibly qualified identifier, atom by atom, for literal SQL text. */
|
|
130
|
+
export const quoteQualified = (name: string): string =>
|
|
131
|
+
name
|
|
132
|
+
.split('.')
|
|
133
|
+
.map((atom) => quoteAtom(atom))
|
|
134
|
+
.join('.')
|
|
135
|
+
|
|
136
|
+
/** Quote one identifier atom, doubling any embedded `"`. */
|
|
137
|
+
export const quoteAtom = (name: string): string =>
|
|
138
|
+
`"${name.replace(/"/g, '""')}"`
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The quoted sequence name as a SQL string LITERAL, for `nextval('…')`. The
|
|
142
|
+
* literal is parsed as a `regclass`, which honours the double quotes inside it, so
|
|
143
|
+
* an awkward prefix resolves to exactly the object the DDL created; any `'` in it
|
|
144
|
+
* is doubled so the literal cannot be closed early.
|
|
145
|
+
*/
|
|
146
|
+
export const sequenceLiteral = (names: ResolvedFeedNames): string =>
|
|
147
|
+
`'${quoteQualified(names.sequence).replace(/'/g, "''")}'`
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* The ordered, idempotent DDL. Every statement is `IF NOT EXISTS`, so a second
|
|
151
|
+
* run against existing objects is a no-op that keeps every row.
|
|
152
|
+
*
|
|
153
|
+
* Shapes that are load-bearing:
|
|
154
|
+
*
|
|
155
|
+
* - `_feed_rows` is keyed `(feed, key)`: one row per record, which is what
|
|
156
|
+
* "coalesced, latest wins" means in storage, and the arbiter the writer's
|
|
157
|
+
* `ON CONFLICT` names. `doc` is NULL for a tombstone; a document that encodes to
|
|
158
|
+
* JSON `null` is the jsonb value `null`, which is not SQL NULL, so the two
|
|
159
|
+
* cannot be confused.
|
|
160
|
+
* - The `(feed, seq)` index is what `changesAfter` pages by.
|
|
161
|
+
* - ONE sequence serves every feed in the prefix. A feed's `seq` values therefore
|
|
162
|
+
* have gaps, which the cursor never minds: it compares, never counts.
|
|
163
|
+
* - `_feed_cursors` holds the cursor AND the lease, so the cursor's
|
|
164
|
+
* compare-and-set can fence on the lease holder in the same row it moves. The
|
|
165
|
+
* cursor is `(generation, seq)`, both NULL before a sink's first sync and never
|
|
166
|
+
* one without the other, which the `CHECK` holds. `last_advance_at` is when
|
|
167
|
+
* the cursor last moved, NULL until it first does; taking a lease leaves it
|
|
168
|
+
* alone, so a sink registered and never run shows as one.
|
|
169
|
+
* - `_feed_generations` holds the checkpoint key a feed is bound to, so binding
|
|
170
|
+
* one feed to two materialisations is refused across processes, not only within
|
|
171
|
+
* one.
|
|
172
|
+
*/
|
|
173
|
+
export const feedDdlStatements = (
|
|
174
|
+
names: ResolvedFeedNames,
|
|
175
|
+
): ReadonlyArray<string> => {
|
|
176
|
+
const rows = quoteQualified(names.rowsTable)
|
|
177
|
+
const generations = quoteQualified(names.generationsTable)
|
|
178
|
+
const cursors = quoteQualified(names.cursorsTable)
|
|
179
|
+
const parked = quoteQualified(names.parkedTable)
|
|
180
|
+
return [
|
|
181
|
+
`CREATE SCHEMA IF NOT EXISTS ${quoteAtom(names.schema)};`,
|
|
182
|
+
|
|
183
|
+
`CREATE SEQUENCE IF NOT EXISTS ${quoteQualified(names.sequence)};`,
|
|
184
|
+
|
|
185
|
+
`CREATE TABLE IF NOT EXISTS ${rows} (
|
|
186
|
+
"feed" text NOT NULL,
|
|
187
|
+
"key" text NOT NULL,
|
|
188
|
+
"doc" jsonb NULL,
|
|
189
|
+
"seq" bigint NOT NULL,
|
|
190
|
+
"event_position" bigint NOT NULL,
|
|
191
|
+
PRIMARY KEY ("feed", "key")
|
|
192
|
+
);`,
|
|
193
|
+
|
|
194
|
+
`CREATE INDEX IF NOT EXISTS ${quoteAtom(names.rowsSeqIndex)}
|
|
195
|
+
ON ${rows} ("feed", "seq");`,
|
|
196
|
+
|
|
197
|
+
`CREATE TABLE IF NOT EXISTS ${generations} (
|
|
198
|
+
"feed" text PRIMARY KEY,
|
|
199
|
+
"generation" bigint NOT NULL,
|
|
200
|
+
"checkpoint_projection" text NULL,
|
|
201
|
+
"checkpoint_partition" text NULL
|
|
202
|
+
);`,
|
|
203
|
+
|
|
204
|
+
`CREATE TABLE IF NOT EXISTS ${cursors} (
|
|
205
|
+
"feed" text NOT NULL,
|
|
206
|
+
"sink" text NOT NULL,
|
|
207
|
+
"generation" bigint NULL,
|
|
208
|
+
"seq" bigint NULL,
|
|
209
|
+
"lease_holder" text NULL,
|
|
210
|
+
"lease_expires_at" timestamptz NULL,
|
|
211
|
+
"last_advance_at" timestamptz NULL,
|
|
212
|
+
PRIMARY KEY ("feed", "sink"),
|
|
213
|
+
CHECK (("generation" IS NULL) = ("seq" IS NULL))
|
|
214
|
+
);`,
|
|
215
|
+
|
|
216
|
+
`CREATE TABLE IF NOT EXISTS ${parked} (
|
|
217
|
+
"feed" text NOT NULL,
|
|
218
|
+
"sink" text NOT NULL,
|
|
219
|
+
"key" text NOT NULL,
|
|
220
|
+
"seq" bigint NOT NULL,
|
|
221
|
+
"reason" text NOT NULL,
|
|
222
|
+
"parked_at" timestamptz NOT NULL,
|
|
223
|
+
PRIMARY KEY ("feed", "sink", "key")
|
|
224
|
+
);`,
|
|
225
|
+
]
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** The DDL as statements on `sql`, one `sql.unsafe` each, in order. */
|
|
229
|
+
export const runFeedDdl = (
|
|
230
|
+
sql: SqlClient.SqlClient,
|
|
231
|
+
names: ResolvedFeedNames,
|
|
232
|
+
): ReadonlyArray<Statement.Statement<unknown>> =>
|
|
233
|
+
feedDdlStatements(names).map((statement) => sql.unsafe(statement))
|
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every runtime statement text the Postgres change feed issues, compiled once per
|
|
3
|
+
* name set, and the conventions their parameters follow.
|
|
4
|
+
*
|
|
5
|
+
* One module, so what the feed sends to the database is auditable in one file.
|
|
6
|
+
* The Effect wiring that issues them lives in `feed.ts`, `syncStore.ts`, `lease.ts`
|
|
7
|
+
* and `withFeed.ts`; none of those writes SQL of its own.
|
|
8
|
+
*
|
|
9
|
+
* ## Parameter conventions
|
|
10
|
+
*
|
|
11
|
+
* - Every 64-bit value (`seq`, `generation`, `event_position`) is bound as a
|
|
12
|
+
* decimal STRING and cast (`$n::bigint`), and read back as text and turned into
|
|
13
|
+
* a `bigint`, so no value ever passes through a JS `number` and no driver's
|
|
14
|
+
* integer handling can round one.
|
|
15
|
+
* - A batch of rows is bound as ONE `jsonb` parameter and unpacked server-side
|
|
16
|
+
* (`jsonb_to_recordset`, `jsonb_array_elements_text`), so a batch is one
|
|
17
|
+
* statement whatever its size and no array-binding support is asked of the
|
|
18
|
+
* driver.
|
|
19
|
+
* - A timestamp is bound as epoch MILLISECONDS from Effect's `Clock`, and read back
|
|
20
|
+
* the same way. Lease expiry is judged against the worker's Clock rather than the
|
|
21
|
+
* database's `now()`, so a `TestClock` governs it in tests as it governs every
|
|
22
|
+
* other time in this repository; the cost is that workers on several hosts must
|
|
23
|
+
* keep their clocks within a small fraction of the lease length of each other.
|
|
24
|
+
* - A document is read back as `doc::text` and parsed here, so a consumer's JSON
|
|
25
|
+
* parsing or name transforms cannot reshape it on the way.
|
|
26
|
+
* - A column read back as text is ALIASED to its own name, so every `ORDER BY`
|
|
27
|
+
* qualifies the column with its table alias: an unqualified name in `ORDER BY`
|
|
28
|
+
* resolves to the output column first, and `seq` would then sort as text, with
|
|
29
|
+
* `10` before `4`.
|
|
30
|
+
*
|
|
31
|
+
* ## The feed lock
|
|
32
|
+
*
|
|
33
|
+
* The generations row of a feed doubles as its WRITE LOCK. A feed's `seq` is
|
|
34
|
+
* ascending in commit order only while one transaction at a time takes `seq`
|
|
35
|
+
* values for that feed, and the subscribe machine silently loses a row whose `seq`
|
|
36
|
+
* commits below one already read (ADR-0009). The materialisation is one writer, but
|
|
37
|
+
* two other things take `seq` values for a feed outside its commit: `retryParked`
|
|
38
|
+
* re-stamps a row, and a reset empties the feed. So the writer takes the row
|
|
39
|
+
* `FOR SHARE` before it takes any `seq`, `retryParked` takes it `FOR UPDATE`, and
|
|
40
|
+
* the reset UPDATEs it; each waits for the others to commit, and `seq` order is
|
|
41
|
+
* commit order for all three.
|
|
42
|
+
*/
|
|
43
|
+
import { quoteQualified, type ResolvedFeedNames, sequenceLiteral } from './ddl'
|
|
44
|
+
|
|
45
|
+
/** Every statement text, compiled for one name set. */
|
|
46
|
+
export interface FeedStatements {
|
|
47
|
+
// --- the writer ---
|
|
48
|
+
/** `$1` feed. Creates the generations row at generation 0 if absent. */
|
|
49
|
+
readonly ensureGeneration: string
|
|
50
|
+
/** `$1` feed. The writer's share of the feed lock. */
|
|
51
|
+
readonly lockFeedShared: string
|
|
52
|
+
/** `$1` feed, `$2` event position, `$3` jsonb `[{ key, doc }]`. */
|
|
53
|
+
readonly upsertRows: string
|
|
54
|
+
/** `$1` feed, `$2` event position, `$3` jsonb `[key]`. */
|
|
55
|
+
readonly tombstoneRows: string
|
|
56
|
+
// --- the source ---
|
|
57
|
+
/** `$1` feed. */
|
|
58
|
+
readonly selectGeneration: string
|
|
59
|
+
/** `$1` feed, `$2` after seq, `$3` limit. */
|
|
60
|
+
readonly changesAfter: string
|
|
61
|
+
// --- the sync store ---
|
|
62
|
+
/** `$1` feed, `$2` sink. */
|
|
63
|
+
readonly selectCursor: string
|
|
64
|
+
/**
|
|
65
|
+
* `$1` feed, `$2` sink, `$3` holder, `$4` next generation, `$5` next seq,
|
|
66
|
+
* `$6` expected generation (NULL for none), `$7` expected seq (NULL for none),
|
|
67
|
+
* `$8` now ms, recorded as the sink's last advance.
|
|
68
|
+
*/
|
|
69
|
+
readonly casCursor: string
|
|
70
|
+
/** `$1` feed, `$2` sink. */
|
|
71
|
+
readonly clearParked: string
|
|
72
|
+
/** `$1` feed, `$2` sink, `$3` jsonb `[key]`. */
|
|
73
|
+
readonly unparkKeys: string
|
|
74
|
+
/** `$1` feed, `$2` sink, `$3` jsonb `[{ key, seq, reason, parked_at_ms }]`. */
|
|
75
|
+
readonly parkKeys: string
|
|
76
|
+
/** `$1` feed, `$2` sink. */
|
|
77
|
+
readonly selectParked: string
|
|
78
|
+
/** `$1` feed. The feed lock, exclusively, for `retryParked`. */
|
|
79
|
+
readonly lockFeedExclusive: string
|
|
80
|
+
/** `$1` feed, `$2` sink, `$3` key. */
|
|
81
|
+
readonly unparkOne: string
|
|
82
|
+
/** `$1` feed, `$2` key. */
|
|
83
|
+
readonly restampRow: string
|
|
84
|
+
/** `$1` feed. */
|
|
85
|
+
readonly collectTombstones: string
|
|
86
|
+
// --- removing a sink ---
|
|
87
|
+
/** `$1` lock name. The advisory lock, held until the transaction ends. */
|
|
88
|
+
readonly tryAdvisoryXactLock: string
|
|
89
|
+
/** `$1` feed, `$2` sink, `$3` now ms. Deletes the cursor row unless a live lease row holds it. */
|
|
90
|
+
readonly deleteCursorUnlessLeased: string
|
|
91
|
+
/** `$1` feed, `$2` sink. Whether a cursor row is still there. */
|
|
92
|
+
readonly cursorExists: string
|
|
93
|
+
/** `$1` feed, `$2` sink. Every park of the sink, one row back per park. */
|
|
94
|
+
readonly deleteSinkParked: string
|
|
95
|
+
/** `$1` feed. */
|
|
96
|
+
readonly selectHead: string
|
|
97
|
+
/** `$1` feed. */
|
|
98
|
+
readonly describeSinks: string
|
|
99
|
+
/** `$1` feed. */
|
|
100
|
+
readonly describeParked: string
|
|
101
|
+
// --- the lease ---
|
|
102
|
+
/**
|
|
103
|
+
* `$1` feed, `$2` sink, `$3` holder, `$4` expiry ms, `$5` now ms, `$6` whether
|
|
104
|
+
* the caller holds the sink's advisory lock for its transaction.
|
|
105
|
+
*/
|
|
106
|
+
readonly takeLeaseRow: string
|
|
107
|
+
/** `$1` feed, `$2` sink, `$3` holder, `$4` expiry ms. */
|
|
108
|
+
readonly renewLeaseRow: string
|
|
109
|
+
/** `$1` feed, `$2` sink, `$3` holder. */
|
|
110
|
+
readonly releaseLease: string
|
|
111
|
+
/**
|
|
112
|
+
* `$1` feed, `$2` sink, `$3` holder, `$4` now ms, `$5` lock name. On the
|
|
113
|
+
* dedicated connection. Records an advisory holder unless a live lease row
|
|
114
|
+
* holds the row or this session no longer holds the lock; zero rows means one
|
|
115
|
+
* of the two.
|
|
116
|
+
*/
|
|
117
|
+
readonly claimAdvisoryHolder: string
|
|
118
|
+
/** `$1` lock name. On the dedicated connection. */
|
|
119
|
+
readonly tryAdvisoryLock: string
|
|
120
|
+
/** `$1` lock name. On the dedicated connection. */
|
|
121
|
+
readonly holdsAdvisoryLock: string
|
|
122
|
+
/** `$1` lock name. On the dedicated connection. */
|
|
123
|
+
readonly advisoryUnlock: string
|
|
124
|
+
// --- the decorated store ---
|
|
125
|
+
/** `$1` feed, `$2` projection, `$3` partition. */
|
|
126
|
+
readonly bindFeed: string
|
|
127
|
+
/** `$1` feed. */
|
|
128
|
+
readonly bumpGeneration: string
|
|
129
|
+
/** `$1` feed. */
|
|
130
|
+
readonly deleteFeedRows: string
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Compile every statement for `names`. */
|
|
134
|
+
export const compileFeedStatements = (
|
|
135
|
+
names: ResolvedFeedNames,
|
|
136
|
+
): FeedStatements => {
|
|
137
|
+
const rows = quoteQualified(names.rowsTable)
|
|
138
|
+
const generations = quoteQualified(names.generationsTable)
|
|
139
|
+
const cursors = quoteQualified(names.cursorsTable)
|
|
140
|
+
const parked = quoteQualified(names.parkedTable)
|
|
141
|
+
const nextSeq = `nextval(${sequenceLiteral(names)})`
|
|
142
|
+
const msToTimestamp = (param: string) =>
|
|
143
|
+
`to_timestamp(${param}::double precision / 1000)`
|
|
144
|
+
const timestampToMs = (column: string) =>
|
|
145
|
+
`round(extract(epoch from ${column}) * 1000)::bigint::text`
|
|
146
|
+
const advisoryKeyOf = (param: string) => `hashtextextended(${param}, 0)`
|
|
147
|
+
const advisoryKey = advisoryKeyOf('$1')
|
|
148
|
+
// Whether THIS session holds the advisory lock named by `param`. A bigint
|
|
149
|
+
// advisory key is stored in `pg_locks` split into two 32-bit halves, `classid`
|
|
150
|
+
// the high and `objid` the low, with `objsubid` 1.
|
|
151
|
+
const sessionHoldsLock = (param: string) => `EXISTS (
|
|
152
|
+
SELECT 1 FROM pg_locks l
|
|
153
|
+
WHERE l.locktype = 'advisory'
|
|
154
|
+
AND l.pid = pg_backend_pid()
|
|
155
|
+
AND l.granted
|
|
156
|
+
AND l.objsubid = 1
|
|
157
|
+
AND l.classid = ((${advisoryKeyOf(param)} >> 32) & 4294967295)::oid
|
|
158
|
+
AND l.objid = (${advisoryKeyOf(param)} & 4294967295)::oid
|
|
159
|
+
)`
|
|
160
|
+
|
|
161
|
+
return {
|
|
162
|
+
ensureGeneration: `INSERT INTO ${generations} ("feed", "generation")
|
|
163
|
+
VALUES ($1, 0)
|
|
164
|
+
ON CONFLICT ("feed") DO NOTHING`,
|
|
165
|
+
|
|
166
|
+
lockFeedShared: `SELECT 1 AS locked FROM ${generations}
|
|
167
|
+
WHERE "feed" = $1 FOR SHARE`,
|
|
168
|
+
|
|
169
|
+
// One `nextval` per INSERTED candidate, kept by the update arm through
|
|
170
|
+
// `excluded.seq`, so an unchanged document burns a value and writes nothing:
|
|
171
|
+
// the `WHERE … IS DISTINCT FROM` arm compares the normalised jsonb, so key
|
|
172
|
+
// order is no change.
|
|
173
|
+
upsertRows: `INSERT INTO ${rows} AS r ("feed", "key", "doc", "seq", "event_position")
|
|
174
|
+
SELECT $1, t."key", t."doc", ${nextSeq}, $2::bigint
|
|
175
|
+
FROM jsonb_to_recordset($3::jsonb) AS t("key" text, "doc" jsonb)
|
|
176
|
+
ON CONFLICT ("feed", "key") DO UPDATE
|
|
177
|
+
SET "doc" = excluded."doc",
|
|
178
|
+
"seq" = excluded."seq",
|
|
179
|
+
"event_position" = excluded."event_position"
|
|
180
|
+
WHERE r."doc" IS DISTINCT FROM excluded."doc"`,
|
|
181
|
+
|
|
182
|
+
// An UPDATE, never an insert: a remove of an absent key writes nothing, and
|
|
183
|
+
// neither does a remove of a key that is already a tombstone.
|
|
184
|
+
tombstoneRows: `UPDATE ${rows} AS r
|
|
185
|
+
SET "doc" = NULL, "seq" = ${nextSeq}, "event_position" = $2::bigint
|
|
186
|
+
FROM jsonb_array_elements_text($3::jsonb) AS t("key")
|
|
187
|
+
WHERE r."feed" = $1 AND r."key" = t."key" AND r."doc" IS NOT NULL`,
|
|
188
|
+
|
|
189
|
+
selectGeneration: `SELECT "generation"::text AS generation
|
|
190
|
+
FROM ${generations} WHERE "feed" = $1`,
|
|
191
|
+
|
|
192
|
+
changesAfter: `SELECT "key" AS key, "doc"::text AS doc, "seq"::text AS seq,
|
|
193
|
+
"event_position"::text AS event_position
|
|
194
|
+
FROM ${rows} AS r
|
|
195
|
+
WHERE r."feed" = $1 AND r."seq" > $2::bigint
|
|
196
|
+
ORDER BY r."seq"
|
|
197
|
+
LIMIT $3`,
|
|
198
|
+
|
|
199
|
+
selectCursor: `SELECT "generation"::text AS generation, "seq"::text AS seq
|
|
200
|
+
FROM ${cursors} WHERE "feed" = $1 AND "sink" = $2`,
|
|
201
|
+
|
|
202
|
+
// Guarded like a checkpoint AND fenced on the holder: zero rows means the
|
|
203
|
+
// stored cursor was not the expected one, or the lease is somebody else's.
|
|
204
|
+
casCursor: `UPDATE ${cursors}
|
|
205
|
+
SET "generation" = $4::bigint, "seq" = $5::bigint,
|
|
206
|
+
"last_advance_at" = ${msToTimestamp('$8')}
|
|
207
|
+
WHERE "feed" = $1 AND "sink" = $2 AND "lease_holder" = $3
|
|
208
|
+
AND "generation" IS NOT DISTINCT FROM $6::bigint
|
|
209
|
+
AND "seq" IS NOT DISTINCT FROM $7::bigint
|
|
210
|
+
RETURNING 1 AS landed`,
|
|
211
|
+
|
|
212
|
+
clearParked: `DELETE FROM ${parked} WHERE "feed" = $1 AND "sink" = $2`,
|
|
213
|
+
|
|
214
|
+
unparkKeys: `DELETE FROM ${parked}
|
|
215
|
+
WHERE "feed" = $1 AND "sink" = $2
|
|
216
|
+
AND "key" IN (SELECT jsonb_array_elements_text($3::jsonb))`,
|
|
217
|
+
|
|
218
|
+
parkKeys: `INSERT INTO ${parked} ("feed", "sink", "key", "seq", "reason", "parked_at")
|
|
219
|
+
SELECT $1, $2, t."key", t."seq"::bigint, t."reason", ${msToTimestamp('t."parked_at_ms"')}
|
|
220
|
+
FROM jsonb_to_recordset($3::jsonb)
|
|
221
|
+
AS t("key" text, "seq" text, "reason" text, "parked_at_ms" text)
|
|
222
|
+
ON CONFLICT ("feed", "sink", "key") DO UPDATE
|
|
223
|
+
SET "seq" = excluded."seq", "reason" = excluded."reason",
|
|
224
|
+
"parked_at" = excluded."parked_at"`,
|
|
225
|
+
|
|
226
|
+
selectParked: `SELECT "key" AS key, "seq"::text AS seq, "reason" AS reason,
|
|
227
|
+
${timestampToMs('"parked_at"')} AS parked_at_ms
|
|
228
|
+
FROM ${parked} AS p
|
|
229
|
+
WHERE p."feed" = $1 AND p."sink" = $2
|
|
230
|
+
ORDER BY p."seq", p."key"`,
|
|
231
|
+
|
|
232
|
+
lockFeedExclusive: `SELECT 1 AS locked FROM ${generations}
|
|
233
|
+
WHERE "feed" = $1 FOR UPDATE`,
|
|
234
|
+
|
|
235
|
+
unparkOne: `DELETE FROM ${parked}
|
|
236
|
+
WHERE "feed" = $1 AND "sink" = $2 AND "key" = $3
|
|
237
|
+
RETURNING 1 AS unparked`,
|
|
238
|
+
|
|
239
|
+
restampRow: `UPDATE ${rows} SET "seq" = ${nextSeq}
|
|
240
|
+
WHERE "feed" = $1 AND "key" = $2`,
|
|
241
|
+
|
|
242
|
+
// Every sink REGISTERED on the feed counts — every cursor row, whatever
|
|
243
|
+
// generation its cursor was saved at — and the lowest of their cursors bounds
|
|
244
|
+
// what goes. A cursor at an older generation needs no arm of its own:
|
|
245
|
+
// `FeedSyncStore.ts`'s tombstone section owns why the minimum already holds it
|
|
246
|
+
// below every row, and this store keeps that section's premise, since a reset
|
|
247
|
+
// (`withFeed.ts`) moves the generation and empties the feed in one
|
|
248
|
+
// transaction and the one sequence never goes back. A sink with no cursor is
|
|
249
|
+
// resyncing from the origin and blocks collection outright, which `bool_or`
|
|
250
|
+
// says. No cursor row at all leaves both aggregates NULL, so the guard is NULL
|
|
251
|
+
// and nothing goes. The cursors are read in the delete's own statement
|
|
252
|
+
// snapshot, so a cursor saved after it can only be one that blocked here
|
|
253
|
+
// (nothing deleted) or one that moved up. A sink whose cursor row is CREATED
|
|
254
|
+
// after the snapshot is not counted, and need not be: that row is made when
|
|
255
|
+
// the sink takes its lease, before it reads the feed, and a feed holds one row
|
|
256
|
+
// per key, so a tombstone this collects had replaced its key's live row before
|
|
257
|
+
// such a sink could read anything.
|
|
258
|
+
collectTombstones: `WITH "low" AS (
|
|
259
|
+
SELECT min(c."seq") AS "seq",
|
|
260
|
+
bool_or(c."seq" IS NULL) AS "blocked"
|
|
261
|
+
FROM ${cursors} c
|
|
262
|
+
WHERE c."feed" = $1
|
|
263
|
+
)
|
|
264
|
+
DELETE FROM ${rows} r USING "low"
|
|
265
|
+
WHERE r."feed" = $1 AND r."doc" IS NULL
|
|
266
|
+
AND NOT "low"."blocked" AND r."seq" <= "low"."seq"
|
|
267
|
+
RETURNING 1 AS collected`,
|
|
268
|
+
|
|
269
|
+
// The transaction-scoped twin of `tryAdvisoryLock`, on the same key, so it
|
|
270
|
+
// conflicts with a session lock any advisory worker holds and is let go at
|
|
271
|
+
// commit or rollback without an unlock of its own.
|
|
272
|
+
tryAdvisoryXactLock: `SELECT pg_try_advisory_xact_lock(${advisoryKey}) AS taken`,
|
|
273
|
+
|
|
274
|
+
// A row whose expiry is later than the caller's now is a live lease row and
|
|
275
|
+
// stays. A holder with no expiry is an advisory one, and is deleted only
|
|
276
|
+
// because the caller holds the advisory lock, which proves its session gone.
|
|
277
|
+
// The test is in the delete itself, so a lease row taken after an earlier
|
|
278
|
+
// read is still refused here rather than deleted from under its worker.
|
|
279
|
+
deleteCursorUnlessLeased: `DELETE FROM ${cursors}
|
|
280
|
+
WHERE "feed" = $1 AND "sink" = $2
|
|
281
|
+
AND ("lease_expires_at" IS NULL
|
|
282
|
+
OR "lease_expires_at" <= ${msToTimestamp('$3')})
|
|
283
|
+
RETURNING 1 AS removed`,
|
|
284
|
+
|
|
285
|
+
cursorExists: `SELECT 1 AS present FROM ${cursors}
|
|
286
|
+
WHERE "feed" = $1 AND "sink" = $2`,
|
|
287
|
+
|
|
288
|
+
deleteSinkParked: `DELETE FROM ${parked} WHERE "feed" = $1 AND "sink" = $2
|
|
289
|
+
RETURNING 1 AS removed`,
|
|
290
|
+
|
|
291
|
+
selectHead: `SELECT COALESCE(max("seq"), 0)::text AS head
|
|
292
|
+
FROM ${rows} WHERE "feed" = $1`,
|
|
293
|
+
|
|
294
|
+
describeSinks: `SELECT c."sink" AS sink,
|
|
295
|
+
c."generation"::text AS generation,
|
|
296
|
+
c."seq"::text AS seq,
|
|
297
|
+
c."lease_holder" AS lease_holder,
|
|
298
|
+
${timestampToMs('c."lease_expires_at"')} AS lease_expires_at_ms,
|
|
299
|
+
${timestampToMs('c."last_advance_at"')} AS last_advance_at_ms,
|
|
300
|
+
(SELECT count(*) FROM ${rows} r
|
|
301
|
+
WHERE r."feed" = c."feed"
|
|
302
|
+
AND CASE WHEN c."seq" IS NULL THEN r."doc" IS NOT NULL
|
|
303
|
+
ELSE r."seq" > c."seq" END)::text AS pending
|
|
304
|
+
FROM ${cursors} c
|
|
305
|
+
WHERE c."feed" = $1
|
|
306
|
+
ORDER BY c."sink"`,
|
|
307
|
+
|
|
308
|
+
describeParked: `SELECT "sink" AS sink, "key" AS key, "seq"::text AS seq,
|
|
309
|
+
"reason" AS reason,
|
|
310
|
+
${timestampToMs('"parked_at"')} AS parked_at_ms
|
|
311
|
+
FROM ${parked} AS p
|
|
312
|
+
WHERE p."feed" = $1
|
|
313
|
+
ORDER BY p."sink", p."seq", p."key"`,
|
|
314
|
+
|
|
315
|
+
// Taken when nobody holds it, when the holder's expiry has passed, or when the
|
|
316
|
+
// holder has no expiry — an ADVISORY holder — and the caller holds that
|
|
317
|
+
// sink's advisory lock for its own transaction (`$6`), which an advisory
|
|
318
|
+
// worker's live session would have refused, so the session that recorded the
|
|
319
|
+
// holder is gone. The caller holds that lock until this statement commits, so
|
|
320
|
+
// no advisory worker can take the lock and record itself in between.
|
|
321
|
+
takeLeaseRow: `INSERT INTO ${cursors} AS c ("feed", "sink", "lease_holder", "lease_expires_at")
|
|
322
|
+
VALUES ($1, $2, $3, ${msToTimestamp('$4')})
|
|
323
|
+
ON CONFLICT ("feed", "sink") DO UPDATE
|
|
324
|
+
SET "lease_holder" = excluded."lease_holder",
|
|
325
|
+
"lease_expires_at" = excluded."lease_expires_at"
|
|
326
|
+
WHERE c."lease_holder" IS NULL
|
|
327
|
+
OR (c."lease_expires_at" IS NOT NULL
|
|
328
|
+
AND c."lease_expires_at" <= ${msToTimestamp('$5')})
|
|
329
|
+
OR (c."lease_expires_at" IS NULL AND $6::boolean)
|
|
330
|
+
RETURNING 1 AS taken`,
|
|
331
|
+
|
|
332
|
+
// Never moves the expiry back: a renewal the worker abandoned at its timeout
|
|
333
|
+
// can still land after a later one, if its cancel came too late, and must not
|
|
334
|
+
// shorten the lease the later one recorded, which the worker's `ensureHeld`
|
|
335
|
+
// already counts on.
|
|
336
|
+
renewLeaseRow: `UPDATE ${cursors}
|
|
337
|
+
SET "lease_expires_at" = GREATEST("lease_expires_at", ${msToTimestamp('$4')})
|
|
338
|
+
WHERE "feed" = $1 AND "sink" = $2 AND "lease_holder" = $3
|
|
339
|
+
RETURNING 1 AS renewed`,
|
|
340
|
+
|
|
341
|
+
releaseLease: `UPDATE ${cursors}
|
|
342
|
+
SET "lease_holder" = NULL, "lease_expires_at" = NULL
|
|
343
|
+
WHERE "feed" = $1 AND "sink" = $2 AND "lease_holder" = $3`,
|
|
344
|
+
|
|
345
|
+
// Run on the SESSION that took the advisory lock, and proposes its row only
|
|
346
|
+
// while that session still holds the lock, so a claim from a session that has
|
|
347
|
+
// died — or let the lock go — since its try-lock records nothing, and cannot
|
|
348
|
+
// overwrite a successor that took the lock in between. Holding the lock proves
|
|
349
|
+
// that any ADVISORY holder recorded here (one with no expiry) is a dead
|
|
350
|
+
// session's, so that one is taken over. A lease-row holder whose expiry has not
|
|
351
|
+
// passed is live and is never taken over, as `takeLeaseRow` takes over an
|
|
352
|
+
// advisory holder only once its lock is free: whichever kind took the row
|
|
353
|
+
// first keeps it while it lives.
|
|
354
|
+
claimAdvisoryHolder: `INSERT INTO ${cursors} AS c ("feed", "sink", "lease_holder", "lease_expires_at")
|
|
355
|
+
SELECT $1::text, $2::text, $3::text, NULL::timestamptz
|
|
356
|
+
WHERE ${sessionHoldsLock('$5')}
|
|
357
|
+
ON CONFLICT ("feed", "sink") DO UPDATE
|
|
358
|
+
SET "lease_holder" = excluded."lease_holder",
|
|
359
|
+
"lease_expires_at" = NULL
|
|
360
|
+
WHERE c."lease_holder" IS NULL
|
|
361
|
+
OR c."lease_expires_at" IS NULL
|
|
362
|
+
OR c."lease_expires_at" <= ${msToTimestamp('$4')}
|
|
363
|
+
RETURNING 1 AS claimed`,
|
|
364
|
+
|
|
365
|
+
tryAdvisoryLock: `SELECT pg_try_advisory_lock(${advisoryKey}) AS taken`,
|
|
366
|
+
|
|
367
|
+
holdsAdvisoryLock: `SELECT ${sessionHoldsLock('$1')} AS held`,
|
|
368
|
+
|
|
369
|
+
advisoryUnlock: `SELECT pg_advisory_unlock(${advisoryKey}) AS released`,
|
|
370
|
+
|
|
371
|
+
bindFeed: `INSERT INTO ${generations} AS g
|
|
372
|
+
("feed", "generation", "checkpoint_projection", "checkpoint_partition")
|
|
373
|
+
VALUES ($1, 0, $2, $3)
|
|
374
|
+
ON CONFLICT ("feed") DO UPDATE
|
|
375
|
+
SET "checkpoint_projection" = COALESCE(g."checkpoint_projection", excluded."checkpoint_projection"),
|
|
376
|
+
"checkpoint_partition" = COALESCE(g."checkpoint_partition", excluded."checkpoint_partition")
|
|
377
|
+
RETURNING "checkpoint_projection" AS projection,
|
|
378
|
+
"checkpoint_partition" AS partition`,
|
|
379
|
+
|
|
380
|
+
bumpGeneration: `INSERT INTO ${generations} AS g ("feed", "generation")
|
|
381
|
+
VALUES ($1, 1)
|
|
382
|
+
ON CONFLICT ("feed") DO UPDATE
|
|
383
|
+
SET "generation" = g."generation" + 1`,
|
|
384
|
+
|
|
385
|
+
deleteFeedRows: `DELETE FROM ${rows} WHERE "feed" = $1`,
|
|
386
|
+
}
|
|
387
|
+
}
|