@ultimat3/realtime 22.0.0 → 22.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +17 -8
- package/README.md +52 -14
- package/package.json +4 -4
- package/src/errors.ts +22 -1
- package/src/index.ts +1 -0
- package/src/pg-identifier.ts +23 -0
- package/src/pg-preflight.ts +29 -64
- package/src/pg-publication.ts +95 -0
- package/src/pg-replication.ts +2 -1
- package/src/pg-socket.ts +139 -53
- package/src/pg-tls.ts +124 -0
- package/src/pg-wire.ts +14 -1
- package/src/replication-errors.ts +29 -16
- package/src/sync-node-contract.ts +6 -0
- package/src/sync-node.ts +1 -0
- package/src/sync-origin.ts +33 -0
- package/src/sync-upgrade.ts +33 -9
- package/src/thundering-herd.ts +9 -0
package/CLAUDE.md
CHANGED
|
@@ -149,6 +149,9 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
149
149
|
- **A hub that closed opens nothing**: `close()` sets `#closed` before the walk; `#open` closes a
|
|
150
150
|
late subscription and RAISES `X_TRANSPORT_UNAVAILABLE`. `#release` takes the reserved bridge, never
|
|
151
151
|
a name.
|
|
152
|
+
- **An upgrade from a foreign page is `403 X_SOCKET_ORIGIN_REFUSED`** before `authenticate`
|
|
153
|
+
(`sync-origin.ts`, core's `proveSameOrigin`); `AcceptBudget` is reserved before `authenticate`,
|
|
154
|
+
`refund()`ed on every exit that takes no socket.
|
|
152
155
|
- **A socket's actor comes from `createSyncNode({ authenticate })` only**, run before
|
|
153
156
|
`server.upgrade`. `null` = 401 `X_SOCKET_UNAUTHENTICATED`; a throw = 503
|
|
154
157
|
`X_SOCKET_AUTH_UNAVAILABLE`. Absent = anonymous, and `start()` warns. The actor lives only in the
|
|
@@ -188,16 +191,22 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
188
191
|
- **The pump has one way out, `#die`**, which records the failure, stops the confirm timer, closes and
|
|
189
192
|
nulls the connection. `start()` awaits the previous pump before it dials. **`stop()` releases
|
|
190
193
|
everything before it reports anything.**
|
|
191
|
-
-
|
|
192
|
-
`
|
|
193
|
-
|
|
194
|
+
- **The replicator ensures its publication** (`pg-publication.ts`, from preflight): `CREATE … FOR
|
|
195
|
+
TABLE` every entity table, or `ALTER … ADD TABLE` the missing; never a DROP, never `FOR ALL
|
|
196
|
+
TABLES`. Only a server ErrorResponse becomes the refusal. No not-live declaration exists.
|
|
197
|
+
- **A fix handing over `REPLICATION` carries `REPLICATION_GRANT_WARNING`** (cluster-wide grant).
|
|
198
|
+
- **TLS is libpq's `sslmode`** (`pg-tls.ts`): only `verify-*` verify; the runtime never rejects
|
|
199
|
+
(`rejectUnauthorized: false`), `judgeHandshake` decides; the raw socket is DEAF after
|
|
200
|
+
`upgradeTLS` (Bun feeds it ciphertext).
|
|
201
|
+
- **Only a table with NO replica identity is warned** (`NOTHING`, or keyless `DEFAULT`), before the
|
|
202
|
+
slot. A keyed DEFAULT table is correct: the window decides (`pg-identity-window.live.test.ts`).
|
|
194
203
|
- **Every session pins `datestyle=ISO`, `intervalstyle=postgres`, `extra_float_digits=3`** in the
|
|
195
|
-
startup packet (`pg-connection.ts
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
never depend on wall time or a process counter.
|
|
204
|
+
startup packet (`pg-connection.ts`), as Postgres' own walreceiver does.
|
|
205
|
+
- A change lsn is `<16 hex commit position><8 hex row position>`; never order by one half, a wall
|
|
206
|
+
time or a counter.
|
|
199
207
|
- Slot, publication and entity names match `[a-z_][a-z0-9_]*` before interpolation — a security
|
|
200
|
-
boundary. A SQLSTATE is data: `pg-wire.ts`'s
|
|
208
|
+
boundary (`pg-identifier.ts`, the one `assertIdentifier`). A SQLSTATE is data: `pg-wire.ts`'s
|
|
209
|
+
`FIXES` is read with `Object.hasOwn`.
|
|
201
210
|
- **A live row equals a repository row**: `pg-entity-row.ts` decodes through `@ultimat3/entity`'s
|
|
202
211
|
own `decodeRow` / `entityForTable`, so money (three physical columns), scale and every other kind
|
|
203
212
|
fold exactly as `postgresRepo` folds them.
|
package/README.md
CHANGED
|
@@ -316,7 +316,23 @@ never stumbled into.
|
|
|
316
316
|
On drain, `drainPlan()` gives every client its own jittered slot in a spread window and the node
|
|
317
317
|
sends a `reconnect` frame carrying that delay — clients redistribute instead of stampeding.
|
|
318
318
|
`AcceptBudget` is the receiving node's token bucket, and a refusal always carries a retry delay,
|
|
319
|
-
because refusing without one just moves the herd next door.
|
|
319
|
+
because refusing without one just moves the herd next door. **A token is reserved before `authenticate`
|
|
320
|
+
and refunded on every exit that takes no socket** (22.1.0): a 401, an authenticator that throws, a
|
|
321
|
+
shed after it, an upgrade that did not take. Reserved first, so a reconnect herd reaches the token
|
|
322
|
+
service bounded by the burst; refunded, because spent-and-kept, one client dialling with no
|
|
323
|
+
credential drained the bucket and every signed-in reconnect behind it was shed. Not per client IP:
|
|
324
|
+
behind an ingress every dial has the ingress's address, and a forwarded header is the caller's own
|
|
325
|
+
claim.
|
|
326
|
+
|
|
327
|
+
**A socket from a foreign page is refused** — `403 X_SOCKET_ORIGIN_REFUSED`, before `authenticate`
|
|
328
|
+
and before the budget. A websocket carries the session cookie and no CORS applies to it, so a page on
|
|
329
|
+
a sibling host (same-site, which `SameSite=Lax` does not stop) could open one as its visitor. The
|
|
330
|
+
rule is `@ultimat3/core`'s `proveSameOrigin`, the one `@ultimat3/http`'s CSRF check asks, with two
|
|
331
|
+
admissions of the node's own (`sync-origin.ts`): no `Origin` header (RFC 6455 has every browser send
|
|
332
|
+
one, so its absence is not a browser), and the node's own host name at any port or scheme (cookies
|
|
333
|
+
are not port-isolated, and the Compose rung serves the page on `:3000` and the node on `:3001`). A
|
|
334
|
+
page on another host is admitted by `createSyncNode({ allowedOrigins })` — the CLI passes
|
|
335
|
+
`APP_URL`'s origin.
|
|
320
336
|
|
|
321
337
|
The client dials itself back. A closed socket arms one timer — the node's delay when a `reconnect`
|
|
322
338
|
frame assigned one, otherwise `@ultimat3/core`'s `backoffDelay()` on the client's `BackoffPolicy` — and that timer calls `connect()`, which re-subscribes
|
|
@@ -506,7 +522,11 @@ wire twice by a reconnect that raced an ack.
|
|
|
506
522
|
(SCRAM-SHA-256, in-band TLS, CopyBoth), no driver dependency. It preflights `wal_level`, the
|
|
507
523
|
publication, every entity's replica identity and the slot — in that order, because the identity
|
|
508
524
|
check is worthless once the slot exists — creates the slot when there is none, and confirms the
|
|
509
|
-
slot as it goes so the WAL does not grow without bound.
|
|
525
|
+
slot as it goes so the WAL does not grow without bound. **The publication is ensured, not merely
|
|
526
|
+
checked** (`pg-publication.ts`): missing, it is created `FOR TABLE` every entity table; present,
|
|
527
|
+
it gains the entity tables it lacks (`ALTER PUBLICATION … ADD TABLE`) and loses none. `FOR TABLE`
|
|
528
|
+
because a table's owner may publish it without a superuser; a role that may not is refused with
|
|
529
|
+
`X_REPLICATION_FAILED` and the statement to run as one that may. `InMemoryChangeFeed` + `InProcessTransport` remain the
|
|
510
530
|
defaults for `x dev` and every test.
|
|
511
531
|
- **`selectChangeFeed(env, { entities })` decides which feed a boot installs** — same law
|
|
512
532
|
`selectMailDriver` follows: an unset variable means the embedded default. It returns `{ feed,
|
|
@@ -519,6 +539,25 @@ wire twice by a reconnect that raced an ack.
|
|
|
519
539
|
wrong database's WAL would be silently wrong forever. `REPLICATION_SLOT` (default `x_replicator`)
|
|
520
540
|
and `REPLICATION_PUBLICATION` (default `x_changes`) name the slot and publication, both checked
|
|
521
541
|
against `[a-z_][a-z0-9_]*` before they reach a replication command.
|
|
542
|
+
- **TLS follows libpq's `sslmode`** (`pg-tls.ts`, 22.1.0): `disable`; `allow`, `prefer` (the
|
|
543
|
+
default) and `require` encrypt and verify **nothing**; `verify-ca` checks the chain;
|
|
544
|
+
`verify-full` the chain and the host name. `sslrootcert=<path>` is the only trust anchor when
|
|
545
|
+
set (and turns `require` into `verify-ca`, as libpq does); `sslrootcert=system` means the runtime
|
|
546
|
+
store and `verify-full`. With neither, the runtime store is used — it honours
|
|
547
|
+
`NODE_EXTRA_CA_CERTS`; libpq's `~/.postgresql/root.crt` is never read. `allow` is served as
|
|
548
|
+
`prefer` (TLS offered first), never cleartext first. The runtime never rejects on its own
|
|
549
|
+
(`rejectUnauthorized: false`); the handshake's report is judged per mode, and a failure is
|
|
550
|
+
`X_REPLICATION_TLS` naming the check. Until 22.1.0 `prefer` verified — every private-CA server
|
|
551
|
+
(CNPG) failed as a refused write — and the raw socket kept feeding ciphertext to the reader
|
|
552
|
+
after the upgrade, so a trusted CA still hung the stream.
|
|
553
|
+
- **`REPLICATION` is a cluster-wide grant.** A `replication=database` session may also run
|
|
554
|
+
`BASE_BACKUP` and `START_REPLICATION PHYSICAL` with no database check, so on a shared Postgres
|
|
555
|
+
cluster the role could copy every database, `pg_authid` included, drop other slots and exhaust
|
|
556
|
+
the walsenders. Run the replicator against a cluster dedicated to the app, or give it its own role
|
|
557
|
+
through `REPLICATION_URL` with `pg_hba.conf`'s `replication` lines restricted to it — never grant
|
|
558
|
+
`REPLICATION` to an app role on a shared cluster. Every fix line that hands the grant over says
|
|
559
|
+
so (`REPLICATION_GRANT_WARNING`, `pg-wire.ts`) and links
|
|
560
|
+
[`docs/ops/01-kubernetes.md`](../../docs/ops/01-kubernetes.md#replication-is-a-cluster-wide-grant).
|
|
522
561
|
- **`PgAdvisoryLock` is the production `AdvisoryLock`** — `SELECT
|
|
523
562
|
pg_try_advisory_lock(hashtext('x:replicator:<slot>'))` on its own session. Session-scoped, so a
|
|
524
563
|
crashed replicator releases it automatically: no lease renewal, no fencing token, no split brain.
|
|
@@ -553,18 +592,17 @@ wire twice by a reconnect that raced an ack.
|
|
|
553
592
|
*transactions* in commit order, so per-record WAL positions are not monotonic across them. The
|
|
554
593
|
pair sorts in delivery order and is byte-identical on replay, which is what turns at-least-once
|
|
555
594
|
redelivery into a drop instead of a duplicate.
|
|
556
|
-
- **A
|
|
557
|
-
2026-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
follow-up.
|
|
595
|
+
- **A keyed table does not need `REPLICA IDENTITY FULL`; a table with NO identity is warned**
|
|
596
|
+
(`As of 2026-09-23`). Under DEFAULT an update carries no old tuple and a delete only the key, and
|
|
597
|
+
neither decides a live query: the shared window holds the whole row, so a row leaving the result
|
|
598
|
+
set is decided from the window and a delete by `holds(id)` — proved on real WAL by
|
|
599
|
+
`pg-identity-window.live.test.ts` (out of the filter, into it, delete, a row never held).
|
|
600
|
+
`preflight`'s fourth question, **before** `pg_create_logical_replication_slot`, names only the
|
|
601
|
+
entity tables with no identity (`NOTHING`, or `DEFAULT` with no primary key), whose `UPDATE` and
|
|
602
|
+
`DELETE` Postgres refuses once published: a **coded warning**, `X_LIVE_REPLICA_IDENTITY`, fix
|
|
603
|
+
`ALTER TABLE <t> REPLICA IDENTITY FULL;` per table — not a throw. Until 22.1.0 it named every
|
|
604
|
+
table not on FULL, on every boot. `ReplicationStreamStats.partialBefore` still counts changes off
|
|
605
|
+
a non-FULL relation — a volume figure, not a correctness one.
|
|
568
606
|
- The record store is **per page**, in memory, and it is not a query cache: it answers "what is
|
|
569
607
|
record X now", never "have I run this query before". Nothing evicts by time or size — a record
|
|
570
608
|
lives as long as something holds it. Persisting it (IndexedDB) is plan 101 slice 12.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/realtime",
|
|
3
|
-
"version": "22.
|
|
3
|
+
"version": "22.1.0",
|
|
4
4
|
"description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -38,9 +38,9 @@
|
|
|
38
38
|
"test": "bun test"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@ultimat3/core": "22.
|
|
42
|
-
"@ultimat3/entity": "22.
|
|
43
|
-
"@ultimat3/query": "22.
|
|
41
|
+
"@ultimat3/core": "22.1.0",
|
|
42
|
+
"@ultimat3/entity": "22.1.0",
|
|
43
|
+
"@ultimat3/query": "22.1.0",
|
|
44
44
|
"nats": "2.29.3"
|
|
45
45
|
}
|
|
46
46
|
}
|
package/src/errors.ts
CHANGED
|
@@ -34,6 +34,8 @@ export const REALTIME_OWNED_ERROR_CODES = [
|
|
|
34
34
|
'X_SOCKET_UNAUTHENTICATED',
|
|
35
35
|
'X_SOCKET_AUTH_UNAVAILABLE',
|
|
36
36
|
'X_REALTIME_TOPOLOGY',
|
|
37
|
+
'X_REPLICATION_TLS',
|
|
38
|
+
'X_SOCKET_ORIGIN_REFUSED',
|
|
37
39
|
] as const;
|
|
38
40
|
|
|
39
41
|
/**
|
|
@@ -138,10 +140,12 @@ export const REALTIME_ERROR_TITLES: Readonly<Record<RealtimeOwnedErrorCode, stri
|
|
|
138
140
|
X_LIVE_SERVER_RENDER: 'a browser-only live operation ran during a server render',
|
|
139
141
|
X_LIVE_ROW_UNIDENTIFIED: 'a live query returned a row with no id',
|
|
140
142
|
X_LIVE_QUERY_UNKNOWN: 'no live query is registered under the name a subscribe frame asked for',
|
|
141
|
-
X_LIVE_REPLICA_IDENTITY: 'a replicated table
|
|
143
|
+
X_LIVE_REPLICA_IDENTITY: 'a replicated table has no replica identity',
|
|
142
144
|
X_SOCKET_UNAUTHENTICATED: 'the sync upgrade carried no credential this app accepts',
|
|
143
145
|
X_SOCKET_AUTH_UNAVAILABLE: 'the sync node could not decide who a connecting socket is',
|
|
144
146
|
X_REALTIME_TOPOLOGY: 'a sync node boots on a real database with no reachable change feed',
|
|
147
|
+
X_REPLICATION_TLS: 'the replication connection failed TLS',
|
|
148
|
+
X_SOCKET_ORIGIN_REFUSED: 'the websocket upgrade came from another origin',
|
|
145
149
|
};
|
|
146
150
|
|
|
147
151
|
// One unconditional call, so a second package claiming one of realtime's codes throws
|
|
@@ -172,6 +176,7 @@ export {
|
|
|
172
176
|
ReplicaIdentityError,
|
|
173
177
|
ReplicationFailedError,
|
|
174
178
|
ReplicationProtocolError,
|
|
179
|
+
ReplicationTlsError,
|
|
175
180
|
ReplicatorSlotHeldError,
|
|
176
181
|
} from './replication-errors';
|
|
177
182
|
|
|
@@ -351,6 +356,22 @@ export class SocketUnauthenticatedError extends RealtimeError {
|
|
|
351
356
|
}
|
|
352
357
|
}
|
|
353
358
|
|
|
359
|
+
/**
|
|
360
|
+
* A browser page on another origin asked for a socket. No CORS applies to a websocket and the
|
|
361
|
+
* session cookie rides it, so admitting the upgrade would open a socket AS the visitor for a page
|
|
362
|
+
* that is not this app — cross-site websocket hijacking. Decided before `authenticate` and before
|
|
363
|
+
* the accept budget, so a hostile page costs neither.
|
|
364
|
+
*/
|
|
365
|
+
export class SocketOriginRefusedError extends RealtimeError {
|
|
366
|
+
constructor(args: { reason: string }) {
|
|
367
|
+
super({
|
|
368
|
+
code: 'X_SOCKET_ORIGIN_REFUSED',
|
|
369
|
+
cause: `the websocket upgrade was refused: ${args.reason}`,
|
|
370
|
+
fix: 'export APP_URL="https://www.example.com" # on the sync role: the origin the page is served on (or createSyncNode({ allowedOrigins }))',
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
354
375
|
/**
|
|
355
376
|
* `authenticate` raised instead of deciding. The same rule the row gate follows: a failure is not a
|
|
356
377
|
* denial, so the client is told to come back rather than told it may not connect — a token service
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Single responsibility: the identifier charset every replication statement interpolates through.
|
|
2
|
+
// Its own module because the preflight and the publication both assert through it, and either
|
|
3
|
+
// importing the other for it would close a cycle.
|
|
4
|
+
|
|
5
|
+
import { ReplicationFailedError } from './errors';
|
|
6
|
+
|
|
7
|
+
/** Identifiers reach a simple query unparameterised, so the charset is the injection boundary. */
|
|
8
|
+
const IDENTIFIER = /^[a-z_][a-z0-9_]*$/;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The one gate between a caller-supplied name and a simple query. `PgReplicationStream` checks its
|
|
12
|
+
* slot, publication and entity names in its CONSTRUCTOR — a mistyped `REPLICATION_SLOT` is a
|
|
13
|
+
* boot-time fact, and finding it at the first WAL read means a replicator that reported itself
|
|
14
|
+
* started and then never delivered a change.
|
|
15
|
+
*/
|
|
16
|
+
export const assertIdentifier = (kind: string, value: string): string => {
|
|
17
|
+
if (IDENTIFIER.test(value)) return value;
|
|
18
|
+
throw new ReplicationFailedError({
|
|
19
|
+
stage: 'preflight',
|
|
20
|
+
detail: `${kind} "${value}" is not a lower-case postgres identifier`,
|
|
21
|
+
fix: `rename the ${kind} to match [a-z_][a-z0-9_]* — it is interpolated into a replication command`,
|
|
22
|
+
});
|
|
23
|
+
};
|
package/src/pg-preflight.ts
CHANGED
|
@@ -1,35 +1,20 @@
|
|
|
1
|
-
// Single responsibility: the four questions asked of a database BEFORE `START_REPLICATION
|
|
2
|
-
// the
|
|
3
|
-
//
|
|
4
|
-
// the default replica identity from starting.
|
|
1
|
+
// Single responsibility: the four questions asked of a database BEFORE `START_REPLICATION`. Two
|
|
2
|
+
// refuse the boot with the exact statement that fixes them, one — the publication — is answered by
|
|
3
|
+
// ensuring it (`pg-publication.ts`), and the fourth warns, because refusing it would stop every app
|
|
4
|
+
// on the default replica identity from starting.
|
|
5
5
|
|
|
6
6
|
import { logger } from '@ultimat3/core';
|
|
7
7
|
import { ReplicaIdentityError, ReplicationFailedError } from './errors';
|
|
8
8
|
import type { PgConnection } from './pg-connection';
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
const IDENTIFIER = /^[a-z_][a-z0-9_]*$/;
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* The one gate between a caller-supplied name and a simple query. Exported because
|
|
15
|
-
* `PgReplicationStream` checks its slot, publication and entity names in its CONSTRUCTOR — a
|
|
16
|
-
* mistyped `REPLICATION_SLOT` is a boot-time fact, and finding it at the first WAL read means a
|
|
17
|
-
* replicator that reported itself started and then never delivered a change.
|
|
18
|
-
*/
|
|
19
|
-
export const assertIdentifier = (kind: string, value: string): string => {
|
|
20
|
-
if (IDENTIFIER.test(value)) return value;
|
|
21
|
-
throw new ReplicationFailedError({
|
|
22
|
-
stage: 'preflight',
|
|
23
|
-
detail: `${kind} "${value}" is not a lower-case postgres identifier`,
|
|
24
|
-
fix: `rename the ${kind} to match [a-z_][a-z0-9_]* — it is interpolated into a replication command`,
|
|
25
|
-
});
|
|
26
|
-
};
|
|
9
|
+
import { assertIdentifier } from './pg-identifier';
|
|
10
|
+
import { ensurePublication } from './pg-publication';
|
|
27
11
|
|
|
28
12
|
/**
|
|
29
|
-
* The four things that are always misconfigured.
|
|
30
|
-
* left to the server, so each gets its own `fix:` line; the
|
|
31
|
-
*
|
|
32
|
-
*
|
|
13
|
+
* The four things that are always misconfigured. `wal_level` and the slot produce an unreadable
|
|
14
|
+
* server message if left to the server, so each gets its own `fix:` line; the publication is
|
|
15
|
+
* created or extended here, and refused with its statement only when this role may not; the
|
|
16
|
+
* fourth is `warnPartialIdentity` and only warns. `slot` and `publication` are interpolated into
|
|
17
|
+
* simple queries, so the identifier charset is the injection boundary; re-asserted here rather than trusted, so the guarantee
|
|
33
18
|
* travels with the function instead of living only in `start()`.
|
|
34
19
|
*/
|
|
35
20
|
export async function preflight(
|
|
@@ -50,17 +35,7 @@ export async function preflight(
|
|
|
50
35
|
fix: "set wal_level=logical in the server configuration — postgresql.conf, your managed provider's database flags, or `postgres -c wal_level=logical` on a container — then restart postgres",
|
|
51
36
|
});
|
|
52
37
|
}
|
|
53
|
-
|
|
54
|
-
`SELECT 1 FROM pg_publication WHERE pubname = '${publication}'`,
|
|
55
|
-
);
|
|
56
|
-
if (publications.length === 0) {
|
|
57
|
-
const [role] = await connection.query('SELECT current_user');
|
|
58
|
-
throw new ReplicationFailedError({
|
|
59
|
-
stage: 'preflight',
|
|
60
|
-
detail: `no publication named "${publication}" exists`,
|
|
61
|
-
fix: publicationFix(publication, entities, role?.[0]),
|
|
62
|
-
});
|
|
63
|
-
}
|
|
38
|
+
await ensurePublication(connection, publication, entities);
|
|
64
39
|
await warnPartialIdentity(connection, entities);
|
|
65
40
|
const [existing] = await connection.query(
|
|
66
41
|
`SELECT plugin FROM pg_replication_slots WHERE slot_name = '${slot}'`,
|
|
@@ -81,34 +56,16 @@ export async function preflight(
|
|
|
81
56
|
}
|
|
82
57
|
|
|
83
58
|
/**
|
|
84
|
-
* The
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*/
|
|
90
|
-
function publicationFix(
|
|
91
|
-
publication: string,
|
|
92
|
-
entities: ReadonlySet<string>,
|
|
93
|
-
role: string | null | undefined,
|
|
94
|
-
): string {
|
|
95
|
-
const tables = [...entities].sort().join(', ');
|
|
96
|
-
const create = `CREATE PUBLICATION ${publication} FOR TABLE ${tables};`;
|
|
97
|
-
const who =
|
|
98
|
-
typeof role === 'string' && role !== '' ? `"${role.replaceAll('"', '""')}"` : 'CURRENT_USER';
|
|
99
|
-
return `${create} -- and, if the role cannot stream yet: ALTER ROLE ${who} WITH REPLICATION;`;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/**
|
|
103
|
-
* The fourth preflight question, and the one that does NOT refuse. A live query decides whether a
|
|
104
|
-
* row left its result set from `change.before`, and under any replica identity but FULL that tuple
|
|
105
|
-
* is the key columns alone — which `toRow` accepts, since it only requires a text `id`.
|
|
59
|
+
* The fourth preflight question, and the one that does NOT refuse: which entity tables have NO
|
|
60
|
+
* replica identity. Once such a table is in the publication, Postgres refuses its UPDATE and
|
|
61
|
+
* DELETE outright, and a change could not be keyed if it did not. A table with a primary key is
|
|
62
|
+
* not named: under DEFAULT a delete carries its key, and a live query decides against the whole
|
|
63
|
+
* row its shared window holds, never against `change.before` alone.
|
|
106
64
|
*
|
|
107
65
|
* It runs BEFORE `pg_create_logical_replication_slot`: a slot decodes with the identity the
|
|
108
66
|
* catalog held when the rows were written, so asking after the slot exists answers about a stream
|
|
109
|
-
* nobody is reading yet. It WARNS rather than throws
|
|
110
|
-
*
|
|
111
|
-
* rows it is complaining about — `ReplicationStreamStats.partialBefore` is the running half.
|
|
67
|
+
* nobody is reading yet. It WARNS rather than throws, as it always has — a replicator that will
|
|
68
|
+
* not start is worse than the tables it is naming.
|
|
112
69
|
*
|
|
113
70
|
* Entity names are the ones the constructor already put through `assertIdentifier`, which is what
|
|
114
71
|
* makes both the interpolation and the `fix:` safe; a name postgres answers with that is not in
|
|
@@ -120,9 +77,17 @@ async function warnPartialIdentity(
|
|
|
120
77
|
): Promise<void> {
|
|
121
78
|
if (entities.size === 0) return;
|
|
122
79
|
const names = [...entities].map((name) => `'${name}'`).join(', ');
|
|
80
|
+
// NO identity, not "not FULL": `n` (NOTHING), `d` (DEFAULT) with no primary key, or `i` whose
|
|
81
|
+
// index is gone.
|
|
82
|
+
// A keyed table replicates correctly under DEFAULT — a delete names its key, and the shared
|
|
83
|
+
// window holds the whole row a live query decides on — so naming it was noise on every boot.
|
|
123
84
|
const rows = await connection.query(
|
|
124
|
-
`SELECT relname FROM pg_class WHERE relkind = 'r' AND
|
|
125
|
-
`AND
|
|
85
|
+
`SELECT c.relname FROM pg_class c WHERE c.relkind = 'r' AND c.relname IN (${names}) ` +
|
|
86
|
+
`AND (c.relreplident = 'n' OR (c.relreplident = 'd' AND NOT EXISTS ` +
|
|
87
|
+
`(SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary)) ` +
|
|
88
|
+
// USING INDEX whose index was dropped: `relreplident` stays 'i' and it behaves as NOTHING.
|
|
89
|
+
`OR (c.relreplident = 'i' AND NOT EXISTS ` +
|
|
90
|
+
`(SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisreplident)))`,
|
|
126
91
|
);
|
|
127
92
|
const tables = [
|
|
128
93
|
...new Set(
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// Single responsibility: the replicator's publication, ensured at boot — created FOR every entity
|
|
2
|
+
// table when missing, extended by the entity tables it lacks when present, never shrunk. The one
|
|
3
|
+
// way an app gets it: a migration is `x db gen`'s, and `FOR ALL TABLES` needs a superuser.
|
|
4
|
+
|
|
5
|
+
import { logger, stringField } from '@ultimat3/core';
|
|
6
|
+
import { ReplicationFailedError } from './errors';
|
|
7
|
+
import type { PgConnection } from './pg-connection';
|
|
8
|
+
import { assertIdentifier } from './pg-identifier';
|
|
9
|
+
import { REPLICATION_GRANT_WARNING } from './pg-wire';
|
|
10
|
+
|
|
11
|
+
/** What `ensurePublication` did — `added` is sorted, and empty when the publication was complete. */
|
|
12
|
+
export interface PublicationOutcome {
|
|
13
|
+
readonly created: boolean;
|
|
14
|
+
readonly added: readonly string[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Ensures `publication` carries every table in `entities`.
|
|
19
|
+
*
|
|
20
|
+
* `FOR TABLE`, because a table's OWNER may publish it without a superuser — and the app role that
|
|
21
|
+
* ran the migrations owns every entity table. Membership is read through `pg_publication_tables`,
|
|
22
|
+
* which also expands `FOR ALL TABLES` and `FOR TABLES IN SCHEMA`, so an operator's broader
|
|
23
|
+
* publication is recognised as complete rather than ALTERed into an error. Cast through `regclass`
|
|
24
|
+
* so a table on the search path compares by the bare name the entity list uses.
|
|
25
|
+
*
|
|
26
|
+
* Never `DROP TABLE` from it: a table the operator added is theirs. A statement the role may not
|
|
27
|
+
* run is refused with the exact statement to run as a role that may — the refusal preflight always
|
|
28
|
+
* raised, now reached only when the boot could not fix it itself.
|
|
29
|
+
*/
|
|
30
|
+
export async function ensurePublication(
|
|
31
|
+
connection: Pick<PgConnection, 'query'>,
|
|
32
|
+
publication: string,
|
|
33
|
+
entities: ReadonlySet<string>,
|
|
34
|
+
): Promise<PublicationOutcome> {
|
|
35
|
+
assertIdentifier('publication', publication);
|
|
36
|
+
const tables = [...entities].map((name) => assertIdentifier('entity', name)).sort();
|
|
37
|
+
const exists = await connection.query(
|
|
38
|
+
`SELECT 1 FROM pg_publication WHERE pubname = '${publication}'`,
|
|
39
|
+
);
|
|
40
|
+
if (exists.length === 0) {
|
|
41
|
+
const create = `CREATE PUBLICATION ${publication} FOR TABLE ${tables.join(', ')}`;
|
|
42
|
+
await attempt(connection, create, `no publication named "${publication}" exists`);
|
|
43
|
+
logger.info('realtime.publication.created', { publication, tables });
|
|
44
|
+
return { created: true, added: tables };
|
|
45
|
+
}
|
|
46
|
+
const members = await connection.query(
|
|
47
|
+
`SELECT (quote_ident(schemaname) || '.' || quote_ident(tablename))::regclass::text ` +
|
|
48
|
+
`FROM pg_publication_tables WHERE pubname = '${publication}'`,
|
|
49
|
+
);
|
|
50
|
+
const present = new Set(members.map((row) => row[0]));
|
|
51
|
+
const missing = tables.filter((name) => !present.has(name));
|
|
52
|
+
if (missing.length === 0) return { created: false, added: [] };
|
|
53
|
+
const alter = `ALTER PUBLICATION ${publication} ADD TABLE ${missing.join(', ')}`;
|
|
54
|
+
await attempt(connection, alter, `publication "${publication}" lacks ${missing.join(', ')}`);
|
|
55
|
+
logger.info('realtime.publication.extended', { publication, tables: missing });
|
|
56
|
+
return { created: false, added: missing };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Runs `statement`, or refuses with it. Only a server's ErrorResponse is this function's to
|
|
61
|
+
* explain — a dead socket is not a privilege question, so anything else propagates unchanged.
|
|
62
|
+
*/
|
|
63
|
+
async function attempt(
|
|
64
|
+
connection: Pick<PgConnection, 'query'>,
|
|
65
|
+
statement: string,
|
|
66
|
+
why: string,
|
|
67
|
+
): Promise<void> {
|
|
68
|
+
try {
|
|
69
|
+
await connection.query(statement);
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (!(error instanceof ReplicationFailedError)) throw error;
|
|
72
|
+
const server = stringField(error, 'cause') ?? 'the server refused it';
|
|
73
|
+
const [role] = await connection.query('SELECT current_user');
|
|
74
|
+
throw new ReplicationFailedError({
|
|
75
|
+
stage: 'preflight',
|
|
76
|
+
detail: `${why}, and this role could not ${statement.split(' ')[0]?.toLowerCase()} it: ${server}`,
|
|
77
|
+
fix: statementFix(statement, role?.[0]),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The statement to run as a role that owns the tables, plus the `REPLICATION` role attribute
|
|
84
|
+
* streaming needs — which no fix line named until one did. Every name in `statement` already
|
|
85
|
+
* passed `assertIdentifier`; the role is quoted, because a role name is whatever the operator chose.
|
|
86
|
+
*/
|
|
87
|
+
function statementFix(statement: string, role: string | null | undefined): string {
|
|
88
|
+
const who =
|
|
89
|
+
typeof role === 'string' && role !== '' ? `"${role.replaceAll('"', '""')}"` : 'CURRENT_USER';
|
|
90
|
+
return (
|
|
91
|
+
`${statement}; -- as a role that owns those tables and holds CREATE on the database, then restart the replicator. ` +
|
|
92
|
+
`And, if the role cannot stream yet: ALTER ROLE ${who} WITH REPLICATION; -- ` +
|
|
93
|
+
REPLICATION_GRANT_WARNING
|
|
94
|
+
);
|
|
95
|
+
}
|
package/src/pg-replication.ts
CHANGED
|
@@ -10,7 +10,8 @@ import { isRow, type Row } from './json';
|
|
|
10
10
|
import { ByteReader, ByteWriter, epochMsToPgTimestamp, printLsn } from './pg-bytes';
|
|
11
11
|
import { PgConnection } from './pg-connection';
|
|
12
12
|
import { entityRow } from './pg-entity-row';
|
|
13
|
-
import { assertIdentifier
|
|
13
|
+
import { assertIdentifier } from './pg-identifier';
|
|
14
|
+
import { preflight } from './pg-preflight';
|
|
14
15
|
import { bunPgStream, parsePgUrl } from './pg-socket';
|
|
15
16
|
import type { PhysicalRow } from './pg-values';
|
|
16
17
|
import { keyedWrite, PgOutputDecoder, type PgOutputMessage, type PgRelation } from './pgoutput';
|
package/src/pg-socket.ts
CHANGED
|
@@ -2,15 +2,23 @@
|
|
|
2
2
|
// and the SSLRequest handshake that has to happen before the first protocol byte. Bun pushes bytes
|
|
3
3
|
// at handlers while the connection pulls messages, so a queue sits between them.
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { renderThrowable } from '@ultimat3/core';
|
|
6
|
+
import { ReplicationFailedError, ReplicationProtocolError, ReplicationTlsError } from './errors';
|
|
7
|
+
import { judgeHandshake, parseSsl, rootCertificate, type SslMode } from './pg-tls';
|
|
6
8
|
import { type PgStream, sslRequest } from './pg-wire';
|
|
7
9
|
|
|
10
|
+
export type { SslMode } from './pg-tls';
|
|
11
|
+
|
|
8
12
|
/** Structural view of Bun's socket, declared here so the contract does not need bun-types. */
|
|
9
13
|
export interface SocketLike {
|
|
10
14
|
write(data: Uint8Array): number;
|
|
11
15
|
end(): void;
|
|
12
16
|
upgradeTLS(options: {
|
|
13
|
-
readonly tls: {
|
|
17
|
+
readonly tls: {
|
|
18
|
+
readonly serverName: string;
|
|
19
|
+
readonly rejectUnauthorized: boolean;
|
|
20
|
+
readonly ca?: string;
|
|
21
|
+
};
|
|
14
22
|
readonly socket: SocketHandlers;
|
|
15
23
|
}): readonly SocketLike[];
|
|
16
24
|
}
|
|
@@ -21,6 +29,8 @@ export interface SocketHandlers {
|
|
|
21
29
|
end(): void;
|
|
22
30
|
drain(): void;
|
|
23
31
|
error(socket: SocketLike, error: Error): void;
|
|
32
|
+
/** TLS sockets only: Bun reports the verification it did instead of acting on it. */
|
|
33
|
+
handshake?(socket: SocketLike, success: boolean, authorizationError: Error | null): void;
|
|
24
34
|
}
|
|
25
35
|
|
|
26
36
|
export interface BunConnect {
|
|
@@ -31,9 +41,6 @@ export interface BunConnect {
|
|
|
31
41
|
}): Promise<SocketLike>;
|
|
32
42
|
}
|
|
33
43
|
|
|
34
|
-
/** `disable` never offers TLS, `prefer` accepts a refusal, `require` treats one as a failure. */
|
|
35
|
-
export type SslMode = 'disable' | 'prefer' | 'require';
|
|
36
|
-
|
|
37
44
|
export interface PgTarget {
|
|
38
45
|
readonly host: string;
|
|
39
46
|
readonly port: number;
|
|
@@ -41,10 +48,10 @@ export interface PgTarget {
|
|
|
41
48
|
readonly user: string;
|
|
42
49
|
readonly password: string | undefined;
|
|
43
50
|
readonly ssl: SslMode;
|
|
51
|
+
/** `sslrootcert`: a PEM path, `'system'`, or absent for the runtime's trust store. */
|
|
52
|
+
readonly rootCert?: string | undefined;
|
|
44
53
|
}
|
|
45
54
|
|
|
46
|
-
const SSL_MODES = new Set<string>(['disable', 'prefer', 'require']);
|
|
47
|
-
|
|
48
55
|
/** `postgres://user:pass@host:5432/db?sslmode=require`. The one place a connection URL is read. */
|
|
49
56
|
export function parsePgUrl(url: string): PgTarget {
|
|
50
57
|
let parsed: URL;
|
|
@@ -67,14 +74,7 @@ export function parsePgUrl(url: string): PgTarget {
|
|
|
67
74
|
fix: 'set DATABASE_URL to postgres://user:password@host:5432/database',
|
|
68
75
|
});
|
|
69
76
|
}
|
|
70
|
-
const
|
|
71
|
-
if (!SSL_MODES.has(mode)) {
|
|
72
|
-
throw new ReplicationFailedError({
|
|
73
|
-
stage: 'connect',
|
|
74
|
-
detail: `sslmode=${mode} is not one of disable, prefer, require`,
|
|
75
|
-
fix: 'use ?sslmode=require for a managed database, ?sslmode=disable for a local one',
|
|
76
|
-
});
|
|
77
|
-
}
|
|
77
|
+
const { ssl, rootCert } = parseSsl(parsed.searchParams);
|
|
78
78
|
const database = decodeURIComponent(parsed.pathname.replace(/^\//, ''));
|
|
79
79
|
return {
|
|
80
80
|
host: parsed.hostname,
|
|
@@ -82,7 +82,8 @@ export function parsePgUrl(url: string): PgTarget {
|
|
|
82
82
|
database: database === '' ? 'postgres' : database,
|
|
83
83
|
user: decodeURIComponent(parsed.username) || 'postgres',
|
|
84
84
|
password: parsed.password === '' ? undefined : decodeURIComponent(parsed.password),
|
|
85
|
-
ssl
|
|
85
|
+
ssl,
|
|
86
|
+
rootCert,
|
|
86
87
|
};
|
|
87
88
|
}
|
|
88
89
|
|
|
@@ -156,8 +157,24 @@ export const bunPgStream = (target: PgTarget): Promise<PgStream> =>
|
|
|
156
157
|
* happen here rather than in the message layer.
|
|
157
158
|
*/
|
|
158
159
|
export async function pgStreamOver(runtime: BunConnect, target: PgTarget): Promise<PgStream> {
|
|
160
|
+
// Before any socket exists: a trust anchor that cannot be read is a setting, not a connection,
|
|
161
|
+
// and refused after the connect it left the raw socket open — one descriptor per retry.
|
|
162
|
+
const ca = target.ssl === 'disable' ? undefined : await rootCertificate(target.rootCert);
|
|
159
163
|
const queue = new ChunkQueue();
|
|
160
164
|
let draining: (() => void) | undefined;
|
|
165
|
+
/**
|
|
166
|
+
* Set the moment the socket is upgraded. Bun keeps calling the RAW socket's handlers after
|
|
167
|
+
* `upgradeTLS` — with the ciphertext — so those handlers go deaf here, or the reader receives
|
|
168
|
+
* TLS records as protocol bytes: a read that never completes, or "closed with 2007 bytes of a
|
|
169
|
+
* partial message".
|
|
170
|
+
*/
|
|
171
|
+
let upgraded = false;
|
|
172
|
+
|
|
173
|
+
const resume = (): void => {
|
|
174
|
+
const parked = draining;
|
|
175
|
+
draining = undefined;
|
|
176
|
+
parked?.();
|
|
177
|
+
};
|
|
161
178
|
|
|
162
179
|
/**
|
|
163
180
|
* A write parked for `drain` can never get one from a socket that is gone, so it is released
|
|
@@ -165,31 +182,35 @@ export async function pgStreamOver(runtime: BunConnect, target: PgTarget): Promi
|
|
|
165
182
|
* side ends cleanly because an EOF that matters is already an error one layer up.
|
|
166
183
|
*/
|
|
167
184
|
const died = (): void => {
|
|
168
|
-
|
|
169
|
-
draining = undefined;
|
|
170
|
-
resume?.();
|
|
185
|
+
resume();
|
|
171
186
|
queue.end();
|
|
172
187
|
};
|
|
173
188
|
|
|
189
|
+
const connectFailure = (error: Error): ReplicationFailedError =>
|
|
190
|
+
new ReplicationFailedError({
|
|
191
|
+
stage: 'connect',
|
|
192
|
+
detail: error.message,
|
|
193
|
+
fix: `open the route to ${target.host}:${target.port}, then: x doctor db`,
|
|
194
|
+
});
|
|
195
|
+
|
|
174
196
|
const handlers: SocketHandlers = {
|
|
175
197
|
// Copied, not retained: Bun promises nothing about the chunk's contents — or that it is even
|
|
176
198
|
// the same buffer — once the handler returns, and this one outlives it in the queue.
|
|
177
|
-
data: (_socket, data) =>
|
|
178
|
-
|
|
179
|
-
|
|
199
|
+
data: (_socket, data) => {
|
|
200
|
+
if (!upgraded) queue.push(data.slice());
|
|
201
|
+
},
|
|
202
|
+
close: () => {
|
|
203
|
+
if (!upgraded) died();
|
|
204
|
+
},
|
|
205
|
+
end: () => {
|
|
206
|
+
if (!upgraded) died();
|
|
207
|
+
},
|
|
180
208
|
drain: () => {
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
209
|
+
if (!upgraded) resume();
|
|
210
|
+
},
|
|
211
|
+
error: (_socket, error) => {
|
|
212
|
+
if (!upgraded) queue.fail(connectFailure(error));
|
|
184
213
|
},
|
|
185
|
-
error: (_socket, error) =>
|
|
186
|
-
queue.fail(
|
|
187
|
-
new ReplicationFailedError({
|
|
188
|
-
stage: 'connect',
|
|
189
|
-
detail: error.message,
|
|
190
|
-
fix: `open the route to ${target.host}:${target.port}, then: x doctor db`,
|
|
191
|
-
}),
|
|
192
|
-
),
|
|
193
214
|
};
|
|
194
215
|
|
|
195
216
|
let socket = await runtime.connect({
|
|
@@ -219,7 +240,86 @@ export async function pgStreamOver(runtime: BunConnect, target: PgTarget): Promi
|
|
|
219
240
|
}
|
|
220
241
|
};
|
|
221
242
|
|
|
222
|
-
|
|
243
|
+
/**
|
|
244
|
+
* The upgrade, awaited to its HANDSHAKE rather than assumed: the runtime is told never to
|
|
245
|
+
* reject (`rejectUnauthorized: false`), because libpq's `prefer` and `require` verify nothing,
|
|
246
|
+
* and `judgeHandshake` decides what its report means under the mode that was asked for. A
|
|
247
|
+
* failure is `X_REPLICATION_TLS` here, never a refused write one step later.
|
|
248
|
+
*/
|
|
249
|
+
const upgrade = async (): Promise<SocketLike> => {
|
|
250
|
+
let settle: ((failure: ReplicationTlsError | undefined) => void) | undefined;
|
|
251
|
+
const handshake = new Promise<ReplicationTlsError | undefined>((resolve) => {
|
|
252
|
+
settle = resolve;
|
|
253
|
+
});
|
|
254
|
+
const decide = (failure: ReplicationTlsError | undefined): void => {
|
|
255
|
+
const once = settle;
|
|
256
|
+
settle = undefined;
|
|
257
|
+
once?.(failure);
|
|
258
|
+
};
|
|
259
|
+
const lost = (detail: string): ReplicationTlsError =>
|
|
260
|
+
new ReplicationTlsError({
|
|
261
|
+
detail,
|
|
262
|
+
fix: `confirm ${target.host}:${target.port} is postgres with ssl=on, or use ?sslmode=disable if it does not speak TLS`,
|
|
263
|
+
});
|
|
264
|
+
const tlsHandlers: SocketHandlers = {
|
|
265
|
+
data: (_socket, data) => queue.push(data.slice()),
|
|
266
|
+
close: () => {
|
|
267
|
+
decide(lost('the server closed the connection during the TLS handshake'));
|
|
268
|
+
died();
|
|
269
|
+
},
|
|
270
|
+
end: () => {
|
|
271
|
+
decide(lost('the server ended the connection during the TLS handshake'));
|
|
272
|
+
died();
|
|
273
|
+
},
|
|
274
|
+
drain: resume,
|
|
275
|
+
error: (_socket, error) => {
|
|
276
|
+
decide(lost(`the TLS handshake failed: ${renderThrowable(error)}`));
|
|
277
|
+
queue.fail(connectFailure(error));
|
|
278
|
+
},
|
|
279
|
+
handshake: (_socket, _success, authorizationError) =>
|
|
280
|
+
decide(judgeHandshake(target, authorizationError)),
|
|
281
|
+
};
|
|
282
|
+
upgraded = true;
|
|
283
|
+
// Bun hands back `[raw, tls]`; every later read and write goes through the second one.
|
|
284
|
+
const tls = socket.upgradeTLS({
|
|
285
|
+
tls: {
|
|
286
|
+
serverName: target.host,
|
|
287
|
+
rejectUnauthorized: false,
|
|
288
|
+
...(ca === undefined ? {} : { ca }),
|
|
289
|
+
},
|
|
290
|
+
socket: tlsHandlers,
|
|
291
|
+
})[1];
|
|
292
|
+
if (tls === undefined) {
|
|
293
|
+
throw new ReplicationFailedError({
|
|
294
|
+
stage: 'ssl',
|
|
295
|
+
detail: 'the runtime returned no TLS socket for the upgrade',
|
|
296
|
+
fix: 'bun upgrade # in-band TLS needs bun >= 1.3',
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
const failure = await handshake;
|
|
300
|
+
if (failure !== undefined) {
|
|
301
|
+
tls.end();
|
|
302
|
+
throw failure;
|
|
303
|
+
}
|
|
304
|
+
return tls;
|
|
305
|
+
};
|
|
306
|
+
|
|
307
|
+
// Every refusal below owns the socket it opened: the caller never receives a stream to close.
|
|
308
|
+
try {
|
|
309
|
+
await negotiate();
|
|
310
|
+
} catch (failure) {
|
|
311
|
+
socket.end();
|
|
312
|
+
throw failure;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
return {
|
|
316
|
+
read: () => queue.read(),
|
|
317
|
+
write: flush,
|
|
318
|
+
close: () => socket.end(),
|
|
319
|
+
};
|
|
320
|
+
|
|
321
|
+
async function negotiate(): Promise<void> {
|
|
322
|
+
if (target.ssl === 'disable') return;
|
|
223
323
|
await flush(sslRequest());
|
|
224
324
|
const answer = (await queue.read()) ?? new Uint8Array(0);
|
|
225
325
|
const verdict = answer[0];
|
|
@@ -240,29 +340,15 @@ export async function pgStreamOver(runtime: BunConnect, target: PgTarget): Promi
|
|
|
240
340
|
fix: 'point the replication URL at postgres itself — a proxy answers like this',
|
|
241
341
|
});
|
|
242
342
|
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
throw new ReplicationFailedError({
|
|
248
|
-
stage: 'ssl',
|
|
249
|
-
detail: 'the runtime returned no TLS socket for the upgrade',
|
|
250
|
-
fix: 'bun upgrade # in-band TLS needs bun >= 1.3',
|
|
251
|
-
});
|
|
252
|
-
}
|
|
253
|
-
socket = upgraded;
|
|
254
|
-
} else if (target.ssl === 'require') {
|
|
343
|
+
socket = await upgrade();
|
|
344
|
+
} else if (target.ssl !== 'allow' && target.ssl !== 'prefer') {
|
|
345
|
+
// `X_REPLICATION_FAILED`, as it always was: a server that says no is a connection outcome,
|
|
346
|
+
// and moving it to the TLS code would break whoever already matches on it.
|
|
255
347
|
throw new ReplicationFailedError({
|
|
256
348
|
stage: 'ssl',
|
|
257
|
-
detail:
|
|
349
|
+
detail: `the server refused TLS but sslmode=${target.ssl} was asked for`,
|
|
258
350
|
fix: `enable ssl on ${target.host}, or use ?sslmode=prefer to accept a cleartext session`,
|
|
259
351
|
});
|
|
260
352
|
}
|
|
261
353
|
}
|
|
262
|
-
|
|
263
|
-
return {
|
|
264
|
-
read: () => queue.read(),
|
|
265
|
-
write: flush,
|
|
266
|
-
close: () => socket.end(),
|
|
267
|
-
};
|
|
268
354
|
}
|
package/src/pg-tls.ts
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Single responsibility: libpq's `sslmode` / `sslrootcert` semantics for the replicator's own wire
|
|
2
|
+
// client — which modes exist, which of them VERIFY, what a handshake's authorization error means
|
|
3
|
+
// under each, and where the trust anchor comes from. The socket mechanics are `pg-socket.ts`'s.
|
|
4
|
+
|
|
5
|
+
import { renderFixShellArg, renderThrowable, stringField } from '@ultimat3/core';
|
|
6
|
+
import { ReplicationFailedError, ReplicationTlsError } from './errors';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* libpq's six. `disable` never offers TLS; `allow`, `prefer` and `require` encrypt and verify
|
|
10
|
+
* NOTHING; `verify-ca` checks the chain; `verify-full` checks the chain and the host name.
|
|
11
|
+
*/
|
|
12
|
+
export type SslMode = 'disable' | 'allow' | 'prefer' | 'require' | 'verify-ca' | 'verify-full';
|
|
13
|
+
|
|
14
|
+
const SSL_MODES: readonly SslMode[] = [
|
|
15
|
+
'disable',
|
|
16
|
+
'allow',
|
|
17
|
+
'prefer',
|
|
18
|
+
'require',
|
|
19
|
+
'verify-ca',
|
|
20
|
+
'verify-full',
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
const isSslMode = (value: string): value is SslMode =>
|
|
24
|
+
(SSL_MODES as readonly string[]).includes(value);
|
|
25
|
+
|
|
26
|
+
/** A path to a PEM file, `'system'` for the runtime's store, or `undefined` for its default. */
|
|
27
|
+
export interface SslSettings {
|
|
28
|
+
readonly ssl: SslMode;
|
|
29
|
+
readonly rootCert: string | undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The connection URL's TLS half, with libpq's two couplings: `require` with a root certificate
|
|
34
|
+
* FILE behaves as `verify-ca` (a CA the operator named is one they meant to be checked against),
|
|
35
|
+
* and `sslrootcert=system` defaults to `verify-full` and refuses anything weaker — the system
|
|
36
|
+
* store trusts every public CA, so only the host name makes it mean anything.
|
|
37
|
+
*/
|
|
38
|
+
export function parseSsl(params: URLSearchParams): SslSettings {
|
|
39
|
+
const given = params.get('sslmode');
|
|
40
|
+
const rootCert = params.get('sslrootcert') ?? undefined;
|
|
41
|
+
const mode = given ?? (rootCert === 'system' ? 'verify-full' : 'prefer');
|
|
42
|
+
if (!isSslMode(mode)) {
|
|
43
|
+
throw new ReplicationFailedError({
|
|
44
|
+
stage: 'connect',
|
|
45
|
+
detail: `sslmode=${mode} is not one of ${SSL_MODES.join(', ')}`,
|
|
46
|
+
fix: 'use ?sslmode=verify-full for a managed database, ?sslmode=disable for a local one',
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
if (rootCert === 'system' && mode !== 'verify-full') {
|
|
50
|
+
throw new ReplicationFailedError({
|
|
51
|
+
stage: 'connect',
|
|
52
|
+
detail: `sslrootcert=system trusts every public CA, so sslmode=${mode} would prove nothing`,
|
|
53
|
+
fix: 'use ?sslrootcert=system&sslmode=verify-full, or name your CA: ?sslrootcert=/path/ca.crt',
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
if (mode === 'require' && rootCert !== undefined && rootCert !== 'system') {
|
|
57
|
+
return { ssl: 'verify-ca', rootCert };
|
|
58
|
+
}
|
|
59
|
+
return { ssl: mode, rootCert };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Whether a mode asks the runtime for the verification at all. */
|
|
63
|
+
export const verifies = (ssl: SslMode): boolean => ssl === 'verify-ca' || ssl === 'verify-full';
|
|
64
|
+
|
|
65
|
+
/** OpenSSL's code for a certificate whose chain verified but whose names do not include the host. */
|
|
66
|
+
const HOST_MISMATCH = 'ERR_TLS_CERT_ALTNAME_INVALID';
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The verdict on a completed handshake. The runtime is always asked NOT to reject
|
|
70
|
+
* (`rejectUnauthorized: false`) and reports what it found instead, chain errors ahead of the host
|
|
71
|
+
* name; this decides what that report means under `ssl`. `undefined` is "carry on".
|
|
72
|
+
*/
|
|
73
|
+
export function judgeHandshake(
|
|
74
|
+
target: { readonly ssl: SslMode; readonly host: string; readonly port: number },
|
|
75
|
+
authorizationError: unknown,
|
|
76
|
+
): ReplicationTlsError | undefined {
|
|
77
|
+
if (!verifies(target.ssl)) return undefined;
|
|
78
|
+
if (authorizationError === null || authorizationError === undefined) return undefined;
|
|
79
|
+
const code =
|
|
80
|
+
stringField(authorizationError, 'code') ??
|
|
81
|
+
(typeof authorizationError === 'string'
|
|
82
|
+
? authorizationError
|
|
83
|
+
: renderThrowable(authorizationError));
|
|
84
|
+
if (code === HOST_MISMATCH) {
|
|
85
|
+
if (target.ssl === 'verify-ca') return undefined;
|
|
86
|
+
return new ReplicationTlsError({
|
|
87
|
+
detail: `the server certificate does not name ${target.host} (${code}) and sslmode=verify-full checks it`,
|
|
88
|
+
// The target's own host and port, so the pasted command reaches this server; a host a shell
|
|
89
|
+
// would misread becomes the libpq env pair instead, which a shell expands rather than runs.
|
|
90
|
+
fix: `openssl s_client -starttls postgres -connect ${renderFixShellArg(`${target.host}:${target.port}`, '"$PGHOST:$PGPORT"')} </dev/null | openssl x509 -noout -ext subjectAltName # connect with a name it lists, or use ?sslmode=verify-ca to check the chain alone`,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
return new ReplicationTlsError({
|
|
94
|
+
detail: `the server certificate did not verify (${code}) and sslmode=${target.ssl} checks it`,
|
|
95
|
+
fix: 'pass the server CA with ?sslrootcert=/path/to/ca.crt, or use ?sslmode=require to encrypt without verifying',
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The trust anchor `upgradeTLS` gets as `ca`: a named file REPLACES the runtime store, as libpq's
|
|
101
|
+
* `root.crt` does; `system` and absent both leave the runtime's store (which honours
|
|
102
|
+
* `NODE_EXTRA_CA_CERTS`). libpq reads `~/.postgresql/root.crt` when nothing is named — a container
|
|
103
|
+
* has no such home, so this build never looks there.
|
|
104
|
+
*/
|
|
105
|
+
export async function rootCertificate(rootCert: string | undefined): Promise<string | undefined> {
|
|
106
|
+
if (rootCert === undefined || rootCert === 'system') return undefined;
|
|
107
|
+
const file = Bun.file(rootCert);
|
|
108
|
+
if (!(await file.exists())) {
|
|
109
|
+
throw new ReplicationTlsError({
|
|
110
|
+
detail: 'sslrootcert names a file that does not exist',
|
|
111
|
+
fix: 'mount the server CA and point ?sslrootcert= at it, or use ?sslrootcert=system for a public CA',
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
// `exists()` is not readability: a secret mounted 0600 for another user exists and still refuses
|
|
115
|
+
// the read, which escaped as a raw runtime error with no code.
|
|
116
|
+
try {
|
|
117
|
+
return await file.text();
|
|
118
|
+
} catch (error) {
|
|
119
|
+
throw new ReplicationTlsError({
|
|
120
|
+
detail: `sslrootcert names a file this process cannot read: ${renderThrowable(error)}`,
|
|
121
|
+
fix: 'id -u # the replicator runs as this user: mount the CA readable by it, or use ?sslrootcert=system for a public CA',
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
}
|
package/src/pg-wire.ts
CHANGED
|
@@ -213,10 +213,23 @@ export const describeFields = (fields: Readonly<Record<string, string>>): string
|
|
|
213
213
|
* not: `changefeed-env -> changefeed -> pg-replication -> pg-wire` is already a chain, so reading
|
|
214
214
|
* the constant here would close it into a cycle. Not re-exported from `index.ts`.
|
|
215
215
|
*/
|
|
216
|
+
export const REPLICATION_EXPOSURE_DOC =
|
|
217
|
+
'https://github.com/developerz-ai/ultimate/blob/main/docs/ops/01-kubernetes.md#replication-is-a-cluster-wide-grant';
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* What granting `REPLICATION` costs, said wherever a fix hands the grant over. The attribute is
|
|
221
|
+
* CLUSTER-wide: a `replication=database` session may run `BASE_BACKUP` or `START_REPLICATION
|
|
222
|
+
* PHYSICAL` with no database check, so on a shared cluster the app role could copy every database,
|
|
223
|
+
* `pg_authid` included, drop other slots and exhaust the walsenders.
|
|
224
|
+
*/
|
|
225
|
+
export const REPLICATION_GRANT_WARNING =
|
|
226
|
+
`only on a Postgres cluster dedicated to this app — on a shared cluster REPLICATION lets this ` +
|
|
227
|
+
`role copy every database (${REPLICATION_EXPOSURE_DOC})`;
|
|
228
|
+
|
|
216
229
|
export const FIXES: Readonly<Record<string, string>> = {
|
|
217
230
|
'28P01': 'correct the password in the replication URL — the server refused the credentials',
|
|
218
231
|
'28000': 'add a `host replication <user> <cidr> scram-sha-256` line to pg_hba.conf and reload',
|
|
219
|
-
'42501':
|
|
232
|
+
'42501': `ALTER ROLE <user> WITH REPLICATION; -- ${REPLICATION_GRANT_WARNING}`,
|
|
220
233
|
'55006': 'another replicator holds the slot — exactly one replicator per database, by design',
|
|
221
234
|
// NOT `x db replication init`, which this line said until 2026-08-20 and which is not a command:
|
|
222
235
|
// `x db` takes gen, migrate, reset, seed, studio, branch and backfill. It shipped because a fix
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// The
|
|
2
|
-
// the replica identity it warns about.
|
|
1
|
+
// The five refusals the Postgres replication half raises: the wire, the connection, its TLS, the
|
|
2
|
+
// slot, and the replica identity it warns about.
|
|
3
3
|
//
|
|
4
4
|
// Split out of `errors.ts` on the one seam this package already draws — these are the only codes
|
|
5
5
|
// no browser can reach, thrown by `pg-*.ts` and the replicator and by nothing on the client half.
|
|
@@ -40,6 +40,22 @@ export class ReplicationFailedError extends RealtimeError {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
/**
|
|
44
|
+
* The replication connection failed TLS: the certificate failed the verification the `sslmode`
|
|
45
|
+
* asked for, `sslrootcert` named nothing readable, or the handshake itself failed. Its own code
|
|
46
|
+
* because the fix is a TLS setting, never the network — a certificate refusal used to surface as
|
|
47
|
+
* `X_REPLICATION_FAILED` "the socket refused a 139-byte write", one step after a silent close.
|
|
48
|
+
*/
|
|
49
|
+
export class ReplicationTlsError extends RealtimeError {
|
|
50
|
+
constructor(args: { detail: string; fix: string }) {
|
|
51
|
+
super({
|
|
52
|
+
code: 'X_REPLICATION_TLS',
|
|
53
|
+
cause: `postgres replication tls failed: ${args.detail}`,
|
|
54
|
+
fix: args.fix,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
43
59
|
/**
|
|
44
60
|
* A second replicator found the advisory lock held. Distinct from `X_REPLICATION_FAILED` because
|
|
45
61
|
* nothing is wrong with this process: the database already has its one replicator, and a second
|
|
@@ -59,29 +75,26 @@ export class ReplicatorSlotHeldError extends RealtimeError {
|
|
|
59
75
|
}
|
|
60
76
|
|
|
61
77
|
/**
|
|
62
|
-
* A table in the entity list
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
* **Raised at preflight and LOGGED, never thrown.** Every app running today on the default
|
|
68
|
-
* identity would stop booting, and the replicator refusing to start is a worse outcome than the
|
|
69
|
-
* partial rows it is warning about. The runtime half is `ReplicationStreamStats.partialBefore`,
|
|
70
|
-
* which counts the changes this actually affects. Refusing it at `x verify` time is the follow-up.
|
|
78
|
+
* A table in the entity list has NO replica identity — no primary key under DEFAULT, or
|
|
79
|
+
* `REPLICA IDENTITY NOTHING`. Once it is in the publication Postgres refuses its UPDATE and DELETE
|
|
80
|
+
* (`cannot update table … because it does not have a replica identity and publishes updates`),
|
|
81
|
+
* and a change the replicator did see could not be keyed. A keyed table under DEFAULT is correct
|
|
82
|
+
* and is never named: the shared window holds the whole row a live query decides on.
|
|
71
83
|
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
84
|
+
* **Raised at preflight and LOGGED, never thrown**, as it always was: a replicator that will not
|
|
85
|
+
* start is worse than the tables it names. The tables are the entity list's own names — every one
|
|
86
|
+
* has already passed `assertIdentifier`, so the `fix:` is SQL that can be pasted.
|
|
74
87
|
*/
|
|
75
88
|
export class ReplicaIdentityError extends RealtimeError {
|
|
76
89
|
constructor(args: { tables: readonly string[] }) {
|
|
77
90
|
super({
|
|
78
91
|
code: 'X_LIVE_REPLICA_IDENTITY',
|
|
79
92
|
cause:
|
|
80
|
-
`${args.tables.join(', ')}
|
|
81
|
-
'
|
|
93
|
+
`${args.tables.join(', ')} have no replica identity (no primary key, or REPLICA IDENTITY ` +
|
|
94
|
+
'NOTHING), so once published an UPDATE or DELETE on them fails and a change cannot be keyed',
|
|
82
95
|
fix:
|
|
83
96
|
`${args.tables.map((table) => `ALTER TABLE ${table} REPLICA IDENTITY FULL;`).join(' ')}` +
|
|
84
|
-
' -- rows already
|
|
97
|
+
' -- or give each a primary key; rows already in the WAL keep the identity they were written with',
|
|
85
98
|
});
|
|
86
99
|
}
|
|
87
100
|
}
|
|
@@ -53,6 +53,12 @@ export interface SyncNodeOptions {
|
|
|
53
53
|
* is a single-tenant node, and `start()` says so in the log.
|
|
54
54
|
*/
|
|
55
55
|
readonly authenticate?: SyncAuthenticator;
|
|
56
|
+
/**
|
|
57
|
+
* Exact origins a page may open a socket from, besides this node's own host name — the page's
|
|
58
|
+
* origin when it is served on another host (`SYNC_URL` on a separate domain). Anything else is
|
|
59
|
+
* refused `X_SOCKET_ORIGIN_REFUSED` before `authenticate` runs.
|
|
60
|
+
*/
|
|
61
|
+
readonly allowedOrigins?: readonly string[];
|
|
56
62
|
/** How often an expired grant is re-decided. The clock a socket's authority runs on. */
|
|
57
63
|
readonly reauthenticateIntervalMs?: number;
|
|
58
64
|
readonly clock?: Clock;
|
package/src/sync-node.ts
CHANGED
|
@@ -269,6 +269,7 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
269
269
|
socketCount: () => sockets.count,
|
|
270
270
|
newSocketId: () => uuid(),
|
|
271
271
|
authenticate: options.authenticate,
|
|
272
|
+
allowedOrigins: options.allowedOrigins,
|
|
272
273
|
onGranted: (socketId, grant) => grants.set(socketId, grant),
|
|
273
274
|
// The other half of recording the grant before the upgrade: an upgrade that never took
|
|
274
275
|
// gets no `close` callback, so this is the only thing that can free its entry.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Single responsibility: whether a websocket upgrade may carry this app's ambient credential. No
|
|
2
|
+
// CORS applies to a websocket and the session cookie rides it, so without this a page on a sibling
|
|
3
|
+
// host (same-site, so `SameSite=Lax` does not stop it) opened a socket as its visitor. The rule is
|
|
4
|
+
// core's `proveSameOrigin`, the one `@ultimat3/http`'s CSRF check asks too.
|
|
5
|
+
|
|
6
|
+
import { type OriginVerdict, proveSameOrigin } from '@ultimat3/core';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Admitted, in order:
|
|
10
|
+
* - **no `Origin`** — RFC 6455 has a browser send one on every handshake, so a dial without it is
|
|
11
|
+
* not a browser and no visitor's cookie can be riding it;
|
|
12
|
+
* - **the node's own host name**, whatever the port or scheme — cookies are not port-isolated, so
|
|
13
|
+
* the page on `:3000` dialling the Compose rung's `:3001` is the same credential boundary, and a
|
|
14
|
+
* TLS-terminating ingress means the node cannot know the page's scheme;
|
|
15
|
+
* - otherwise core's rule, with `allowedOrigins` as the exact list (`APP_URL`'s origin, from the
|
|
16
|
+
* CLI, when the page is served on another host than the node).
|
|
17
|
+
*/
|
|
18
|
+
export function upgradeOrigin(
|
|
19
|
+
request: Request,
|
|
20
|
+
url: URL,
|
|
21
|
+
allowedOrigins: readonly string[],
|
|
22
|
+
): OriginVerdict {
|
|
23
|
+
const origin = request.headers.get('origin');
|
|
24
|
+
if (origin === null) return { ok: true };
|
|
25
|
+
if (URL.parse(origin)?.hostname === url.hostname) return { ok: true };
|
|
26
|
+
return proveSameOrigin({
|
|
27
|
+
selfOrigins: allowedOrigins,
|
|
28
|
+
origin,
|
|
29
|
+
secFetchSite: request.headers.get('sec-fetch-site'),
|
|
30
|
+
listed: () => false,
|
|
31
|
+
listName: 'APP_URL or createSyncNode({ allowedOrigins })',
|
|
32
|
+
});
|
|
33
|
+
}
|
package/src/sync-upgrade.ts
CHANGED
|
@@ -3,8 +3,13 @@
|
|
|
3
3
|
// from what the socket then does — the same line `sync-frames.ts` and `sync-listen.ts` already draw.
|
|
4
4
|
|
|
5
5
|
import { healthzPayload, readyzPayload, reportError } from '@ultimat3/core';
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
SocketAuthUnavailableError,
|
|
8
|
+
SocketOriginRefusedError,
|
|
9
|
+
SocketUnauthenticatedError,
|
|
10
|
+
} from './errors';
|
|
7
11
|
import type { SyncAuthenticator, SyncGrant } from './sync-auth';
|
|
12
|
+
import { upgradeOrigin } from './sync-origin';
|
|
8
13
|
import { toWireError } from './sync-protocol';
|
|
9
14
|
import type { AcceptBudget, Rng } from './thundering-herd';
|
|
10
15
|
|
|
@@ -43,6 +48,11 @@ export interface UpgradeDeps {
|
|
|
43
48
|
socketCount(): number;
|
|
44
49
|
newSocketId(): string;
|
|
45
50
|
readonly authenticate?: SyncAuthenticator | undefined;
|
|
51
|
+
/**
|
|
52
|
+
* Exact origins a page may dial from besides the node's own host name — `APP_URL`'s, when the
|
|
53
|
+
* page is served on another host than the node (`sync-origin.ts`).
|
|
54
|
+
*/
|
|
55
|
+
readonly allowedOrigins?: readonly string[] | undefined;
|
|
46
56
|
/**
|
|
47
57
|
* Recorded BEFORE `server.upgrade`, because Bun runs `websocket.open` synchronously inside it
|
|
48
58
|
* (measured on bun 1.4.0) and `open` is where the node reads this grant to build the socket's
|
|
@@ -79,12 +89,19 @@ export async function handleUpgrade(
|
|
|
79
89
|
return deps.ready() ? json(payload) : json({ status: 503, body: payload.body });
|
|
80
90
|
}
|
|
81
91
|
if (url.pathname !== deps.path) return new Response('not found', { status: 404 });
|
|
82
|
-
//
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
92
|
+
// First, and before anything is spent: a foreign page is refused whatever the node's load.
|
|
93
|
+
const origin = upgradeOrigin(request, url, deps.allowedOrigins ?? []);
|
|
94
|
+
if (!origin.ok)
|
|
95
|
+
return wireErrorResponse(403, new SocketOriginRefusedError({ reason: origin.reason }));
|
|
96
|
+
// The count and readiness. Decided before `authenticate` so a full node costs no token service
|
|
97
|
+
// call.
|
|
98
|
+
if (deps.socketCount() >= deps.maxConnections || !deps.ready()) return shed(deps);
|
|
99
|
+
// The RATE is RESERVED here and REFUNDED on every exit that takes no socket. Reserved first, so a
|
|
100
|
+
// reconnect herd reaches `authenticate` bounded by the burst — the token service is the first
|
|
101
|
+
// thing a herd would otherwise flatten. Refunded, because spent-and-kept, one client dialling
|
|
102
|
+
// with no credential drained the bucket and every signed-in reconnect behind it was shed.
|
|
103
|
+
if (!deps.accept.tryAccept()) return shed(deps);
|
|
104
|
+
const refund = (): void => deps.accept.refund();
|
|
88
105
|
let grant: SyncGrant | null = null;
|
|
89
106
|
if (deps.authenticate) {
|
|
90
107
|
try {
|
|
@@ -93,6 +110,7 @@ export async function handleUpgrade(
|
|
|
93
110
|
// A failure is not a denial. The token service timing out must not read to a client as "you
|
|
94
111
|
// may not connect" — it is told to come back, and this node is the one that pages.
|
|
95
112
|
reportError(error, { source: 'realtime', scope: { operation: 'sync.authenticate' } });
|
|
113
|
+
refund();
|
|
96
114
|
return wireErrorResponse(
|
|
97
115
|
503,
|
|
98
116
|
new SocketAuthUnavailableError({ detail: 'see the node log for the cause' }),
|
|
@@ -101,6 +119,7 @@ export async function handleUpgrade(
|
|
|
101
119
|
// The decision, made before a socket exists: an upgrade is the cheapest thing to refuse and the
|
|
102
120
|
// most expensive thing to take back.
|
|
103
121
|
if (grant === null) {
|
|
122
|
+
refund();
|
|
104
123
|
return wireErrorResponse(
|
|
105
124
|
401,
|
|
106
125
|
new SocketUnauthenticatedError({ reason: 'authenticate() resolved no actor' }),
|
|
@@ -120,8 +139,11 @@ export async function handleUpgrade(
|
|
|
120
139
|
// sockets as there were parked requests, and `maxConnections` bounded nothing that a herd could
|
|
121
140
|
// reach. Sound because there is no await between this line and `server.upgrade`, and the count
|
|
122
141
|
// moves INSIDE it: Bun runs `websocket.open` synchronously there, which is where `sockets.add`
|
|
123
|
-
// runs.
|
|
124
|
-
if (!deps.ready() || deps.socketCount() >= deps.maxConnections)
|
|
142
|
+
// runs.
|
|
143
|
+
if (!deps.ready() || deps.socketCount() >= deps.maxConnections) {
|
|
144
|
+
refund();
|
|
145
|
+
return shed(deps);
|
|
146
|
+
}
|
|
125
147
|
const data: WsData = {
|
|
126
148
|
socketId: deps.newSocketId(),
|
|
127
149
|
// The node's own id is "not skewed until the hello says so", never "current forever".
|
|
@@ -144,10 +166,12 @@ export async function handleUpgrade(
|
|
|
144
166
|
// failing upgrades left 20 grants. Rethrown untouched — the throw is the operator's diagnosis,
|
|
145
167
|
// and this line owes it the release, not a verdict.
|
|
146
168
|
deps.onUngranted(data.socketId);
|
|
169
|
+
refund();
|
|
147
170
|
throw error;
|
|
148
171
|
}
|
|
149
172
|
if (!upgraded) {
|
|
150
173
|
deps.onUngranted(data.socketId);
|
|
174
|
+
refund();
|
|
151
175
|
return new Response('expected websocket', { status: 426 });
|
|
152
176
|
}
|
|
153
177
|
return undefined;
|
package/src/thundering-herd.ts
CHANGED
|
@@ -188,6 +188,15 @@ export class AcceptBudget {
|
|
|
188
188
|
return true;
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
+
/**
|
|
192
|
+
* Hands back a token `tryAccept` reserved for work that took no socket — an upgrade whose
|
|
193
|
+
* credential was refused, or whose authenticator failed. Never past `burst`.
|
|
194
|
+
*/
|
|
195
|
+
refund(): void {
|
|
196
|
+
this.#refill();
|
|
197
|
+
this.#tokens = Math.min(this.#burst, this.#tokens + 1);
|
|
198
|
+
}
|
|
199
|
+
|
|
191
200
|
/** Delay to hand a refused client, jittered so refusals do not re-synchronise the herd. */
|
|
192
201
|
retryAfterMs(rng: Rng = Math.random): number {
|
|
193
202
|
const base = Math.ceil(1000 / this.#perSecond);
|