orez 0.0.18 → 0.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/config.js +1 -1
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +46 -12
- package/dist/index.js.map +1 -1
- package/dist/mutex.d.ts +7 -0
- package/dist/mutex.d.ts.map +1 -0
- package/dist/mutex.js +24 -0
- package/dist/mutex.js.map +1 -0
- package/dist/pg-proxy.d.ts +6 -1
- package/dist/pg-proxy.d.ts.map +1 -1
- package/dist/pg-proxy.js +64 -76
- package/dist/pg-proxy.js.map +1 -1
- package/dist/pglite-manager.d.ts +14 -1
- package/dist/pglite-manager.d.ts.map +1 -1
- package/dist/pglite-manager.js +37 -18
- package/dist/pglite-manager.js.map +1 -1
- package/dist/replication/change-tracker.d.ts +7 -0
- package/dist/replication/change-tracker.d.ts.map +1 -1
- package/dist/replication/change-tracker.js +40 -3
- package/dist/replication/change-tracker.js.map +1 -1
- package/dist/replication/handler.d.ts +2 -1
- package/dist/replication/handler.d.ts.map +1 -1
- package/dist/replication/handler.js +136 -60
- package/dist/replication/handler.js.map +1 -1
- package/dist/replication/pgoutput-encoder.d.ts.map +1 -1
- package/dist/replication/pgoutput-encoder.js +11 -1
- package/dist/replication/pgoutput-encoder.js.map +1 -1
- package/package.json +3 -3
- package/src/cli.ts +1 -1
- package/src/config.ts +1 -1
- package/src/index.ts +45 -12
- package/src/integration/integration.test.ts +24 -14
- package/src/mutex.ts +24 -0
- package/src/pg-proxy.ts +78 -76
- package/src/pglite-manager.ts +48 -18
- package/src/replication/change-tracker.test.ts +4 -4
- package/src/replication/change-tracker.ts +49 -3
- package/src/replication/handler.test.ts +18 -8
- package/src/replication/handler.ts +179 -69
- package/src/replication/pgoutput-encoder.ts +9 -1
- package/src/replication/zero-compat.test.ts +7 -3
package/src/pglite-manager.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs'
|
|
1
|
+
import { readFileSync, readdirSync, existsSync, mkdirSync, renameSync } from 'node:fs'
|
|
2
2
|
import { join, resolve } from 'node:path'
|
|
3
3
|
|
|
4
4
|
import { PGlite } from '@electric-sql/pglite'
|
|
@@ -9,11 +9,22 @@ import { log } from './log.js'
|
|
|
9
9
|
|
|
10
10
|
import type { ZeroLiteConfig } from './config.js'
|
|
11
11
|
|
|
12
|
-
export
|
|
13
|
-
|
|
12
|
+
export interface PGliteInstances {
|
|
13
|
+
postgres: PGlite
|
|
14
|
+
cvr: PGlite
|
|
15
|
+
cdb: PGlite
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// create a single pglite instance with given dataDir suffix
|
|
19
|
+
async function createInstance(
|
|
20
|
+
config: ZeroLiteConfig,
|
|
21
|
+
name: string,
|
|
22
|
+
withExtensions: boolean
|
|
23
|
+
): Promise<PGlite> {
|
|
24
|
+
const dataPath = resolve(config.dataDir, `pgdata-${name}`)
|
|
14
25
|
mkdirSync(dataPath, { recursive: true })
|
|
15
26
|
|
|
16
|
-
log.debug.pglite(`creating instance at ${dataPath}`)
|
|
27
|
+
log.debug.pglite(`creating ${name} instance at ${dataPath}`)
|
|
17
28
|
const {
|
|
18
29
|
dataDir: _d,
|
|
19
30
|
debug: _dbg,
|
|
@@ -24,34 +35,53 @@ export async function createPGliteInstance(config: ZeroLiteConfig): Promise<PGli
|
|
|
24
35
|
debug: config.logLevel === 'debug' ? 1 : 0,
|
|
25
36
|
relaxedDurability: true,
|
|
26
37
|
...userOpts,
|
|
27
|
-
extensions: userOpts.extensions || { vector, pg_trgm },
|
|
38
|
+
extensions: withExtensions ? (userOpts.extensions || { vector, pg_trgm }) : {},
|
|
28
39
|
})
|
|
29
40
|
|
|
30
41
|
await db.waitReady
|
|
31
|
-
log.debug.pglite(
|
|
42
|
+
log.debug.pglite(`${name} ready`)
|
|
43
|
+
return db
|
|
44
|
+
}
|
|
32
45
|
|
|
33
|
-
|
|
34
|
-
|
|
46
|
+
/**
|
|
47
|
+
* create separate pglite instances for each "database".
|
|
48
|
+
*
|
|
49
|
+
* this mirrors real postgresql where postgres, zero_cvr, and zero_cdb are
|
|
50
|
+
* independent databases with separate transaction contexts. each instance
|
|
51
|
+
* has its own session state, so transactions on one database can't be
|
|
52
|
+
* corrupted by queries on another.
|
|
53
|
+
*/
|
|
54
|
+
export async function createPGliteInstances(config: ZeroLiteConfig): Promise<PGliteInstances> {
|
|
55
|
+
// migrate from old single-instance layout (pgdata → pgdata-postgres)
|
|
56
|
+
const oldDataPath = resolve(config.dataDir, 'pgdata')
|
|
57
|
+
const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
|
|
58
|
+
if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
|
|
59
|
+
renameSync(oldDataPath, newDataPath)
|
|
60
|
+
log.debug.pglite('migrated pgdata → pgdata-postgres')
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// create all 3 instances in parallel (only postgres needs app extensions)
|
|
64
|
+
const [postgres, cvr, cdb] = await Promise.all([
|
|
65
|
+
createInstance(config, 'postgres', true),
|
|
66
|
+
createInstance(config, 'cvr', false),
|
|
67
|
+
createInstance(config, 'cdb', false),
|
|
68
|
+
])
|
|
35
69
|
|
|
36
|
-
//
|
|
37
|
-
await
|
|
38
|
-
await db.exec('CREATE SCHEMA IF NOT EXISTS zero_cdb')
|
|
70
|
+
// postgres-specific setup
|
|
71
|
+
await postgres.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
39
72
|
|
|
40
|
-
// create empty publication for zero-cache
|
|
41
|
-
// the app's migrations (via --on-db-ready) add specific tables to it,
|
|
42
|
-
// excluding private tables like user/session/account.
|
|
43
|
-
// DO NOT use FOR ALL TABLES - it auto-includes every future table.
|
|
73
|
+
// create empty publication for zero-cache on postgres instance
|
|
44
74
|
const pubName = process.env.ZERO_APP_PUBLICATIONS || 'zero_pub'
|
|
45
|
-
const pubs = await
|
|
75
|
+
const pubs = await postgres.query<{ count: string }>(
|
|
46
76
|
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
47
77
|
[pubName]
|
|
48
78
|
)
|
|
49
79
|
if (Number(pubs.rows[0].count) === 0) {
|
|
50
80
|
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
51
|
-
await
|
|
81
|
+
await postgres.exec(`CREATE PUBLICATION ${quoted}`)
|
|
52
82
|
}
|
|
53
83
|
|
|
54
|
-
return
|
|
84
|
+
return { postgres, cvr, cdb }
|
|
55
85
|
}
|
|
56
86
|
|
|
57
87
|
export async function runMigrations(db: PGlite, config: ZeroLiteConfig): Promise<void> {
|
|
@@ -33,7 +33,7 @@ describe('change-tracker', () => {
|
|
|
33
33
|
const changes = await getChangesSince(db, 0)
|
|
34
34
|
expect(changes).toHaveLength(1)
|
|
35
35
|
expect(changes[0].op).toBe('INSERT')
|
|
36
|
-
expect(changes[0].table_name).toBe('items')
|
|
36
|
+
expect(changes[0].table_name).toBe('public.items')
|
|
37
37
|
expect(changes[0].row_data).toMatchObject({ name: 'a', value: 1 })
|
|
38
38
|
expect(changes[0].old_data).toBeNull()
|
|
39
39
|
})
|
|
@@ -116,8 +116,8 @@ describe('change-tracker', () => {
|
|
|
116
116
|
|
|
117
117
|
const changes = await getChangesSince(db, 0)
|
|
118
118
|
const tables = new Set(changes.map((c) => c.table_name))
|
|
119
|
-
expect(tables).toContain('items')
|
|
120
|
-
expect(tables).toContain('other')
|
|
119
|
+
expect(tables).toContain('public.items')
|
|
120
|
+
expect(tables).toContain('public.other')
|
|
121
121
|
})
|
|
122
122
|
|
|
123
123
|
it('handles rapid inserts (50 rows)', async () => {
|
|
@@ -178,7 +178,7 @@ describe('change-tracker', () => {
|
|
|
178
178
|
await db.exec(`INSERT INTO public."my""table" (val) VALUES ('works')`)
|
|
179
179
|
|
|
180
180
|
const changes = await getChangesSince(db, 0)
|
|
181
|
-
const special = changes.filter((c) => c.table_name === 'my"table')
|
|
181
|
+
const special = changes.filter((c) => c.table_name === 'public.my"table')
|
|
182
182
|
expect(special).toHaveLength(1)
|
|
183
183
|
expect(special[0].op).toBe('INSERT')
|
|
184
184
|
expect(special[0].row_data).toMatchObject({ val: 'works' })
|
|
@@ -45,18 +45,21 @@ export async function installChangeTracking(db: PGlite): Promise<void> {
|
|
|
45
45
|
// create trigger function
|
|
46
46
|
await db.exec(`
|
|
47
47
|
CREATE OR REPLACE FUNCTION public._zero_track_change() RETURNS TRIGGER AS $$
|
|
48
|
+
DECLARE
|
|
49
|
+
qualified_name TEXT;
|
|
48
50
|
BEGIN
|
|
51
|
+
qualified_name := TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME;
|
|
49
52
|
IF TG_OP = 'DELETE' THEN
|
|
50
53
|
INSERT INTO public._zero_changes (table_name, op, old_data)
|
|
51
|
-
VALUES (
|
|
54
|
+
VALUES (qualified_name, 'DELETE', row_to_json(OLD)::jsonb);
|
|
52
55
|
RETURN OLD;
|
|
53
56
|
ELSIF TG_OP = 'UPDATE' THEN
|
|
54
57
|
INSERT INTO public._zero_changes (table_name, op, row_data, old_data)
|
|
55
|
-
VALUES (
|
|
58
|
+
VALUES (qualified_name, 'UPDATE', row_to_json(NEW)::jsonb, row_to_json(OLD)::jsonb);
|
|
56
59
|
RETURN NEW;
|
|
57
60
|
ELSIF TG_OP = 'INSERT' THEN
|
|
58
61
|
INSERT INTO public._zero_changes (table_name, op, row_data)
|
|
59
|
-
VALUES (
|
|
62
|
+
VALUES (qualified_name, 'INSERT', row_to_json(NEW)::jsonb);
|
|
60
63
|
RETURN NEW;
|
|
61
64
|
END IF;
|
|
62
65
|
RETURN NULL;
|
|
@@ -137,6 +140,49 @@ async function installTriggersOnAllTables(db: PGlite): Promise<void> {
|
|
|
137
140
|
log.debug.pglite(`installed change tracking triggers on ${count} tables`)
|
|
138
141
|
}
|
|
139
142
|
|
|
143
|
+
/**
|
|
144
|
+
* install change tracking triggers on tables in shard schemas.
|
|
145
|
+
* zero-cache creates shard schemas (e.g. chat_0) with clients/mutations
|
|
146
|
+
* tables that track mutation confirmations. these must be replicated
|
|
147
|
+
* for .server promises to resolve.
|
|
148
|
+
*/
|
|
149
|
+
export async function installTriggersOnShardTables(db: PGlite): Promise<void> {
|
|
150
|
+
const result = await db.query<{ nspname: string }>(
|
|
151
|
+
`SELECT nspname FROM pg_namespace
|
|
152
|
+
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'public')
|
|
153
|
+
AND nspname NOT LIKE 'pg_%'
|
|
154
|
+
AND nspname NOT LIKE 'zero_%'
|
|
155
|
+
AND nspname NOT LIKE '_zero_%'
|
|
156
|
+
AND nspname NOT LIKE '%/%'`
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
if (result.rows.length === 0) return
|
|
160
|
+
|
|
161
|
+
let count = 0
|
|
162
|
+
for (const { nspname } of result.rows) {
|
|
163
|
+
const tables = await db.query<{ tablename: string }>(
|
|
164
|
+
`SELECT tablename FROM pg_tables WHERE schemaname = $1`,
|
|
165
|
+
[nspname]
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
for (const { tablename } of tables.rows) {
|
|
169
|
+
const quotedSchema = quoteIdent(nspname)
|
|
170
|
+
const quotedTable = quoteIdent(tablename)
|
|
171
|
+
await db.exec(`
|
|
172
|
+
DROP TRIGGER IF EXISTS _zero_change_trigger ON ${quotedSchema}.${quotedTable};
|
|
173
|
+
CREATE TRIGGER _zero_change_trigger
|
|
174
|
+
AFTER INSERT OR UPDATE OR DELETE ON ${quotedSchema}.${quotedTable}
|
|
175
|
+
FOR EACH ROW EXECUTE FUNCTION public._zero_track_change();
|
|
176
|
+
`)
|
|
177
|
+
count++
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (count > 0) {
|
|
182
|
+
log.debug.pglite(`installed change tracking on ${count} shard tables`)
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
140
186
|
export async function getChangesSince(
|
|
141
187
|
db: PGlite,
|
|
142
188
|
watermark: number,
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
handleStartReplication,
|
|
8
8
|
type ReplicationWriter,
|
|
9
9
|
} from './handler'
|
|
10
|
+
import { Mutex } from '../mutex'
|
|
10
11
|
|
|
11
12
|
// parse wire protocol RowDescription+DataRow response into columns/values
|
|
12
13
|
function parseResponse(buf: Uint8Array): { columns: string[]; values: string[] } | null {
|
|
@@ -116,6 +117,7 @@ describe('handleReplicationQuery', () => {
|
|
|
116
117
|
describe('handleStartReplication', () => {
|
|
117
118
|
let db: PGlite
|
|
118
119
|
let replicationPromise: Promise<void>
|
|
120
|
+
const testMutex = new Mutex()
|
|
119
121
|
|
|
120
122
|
beforeEach(async () => {
|
|
121
123
|
db = new PGlite()
|
|
@@ -160,7 +162,8 @@ describe('handleStartReplication', () => {
|
|
|
160
162
|
replicationPromise = handleStartReplication(
|
|
161
163
|
'START_REPLICATION SLOT "s" LOGICAL 0/0',
|
|
162
164
|
writer,
|
|
163
|
-
db
|
|
165
|
+
db,
|
|
166
|
+
testMutex
|
|
164
167
|
)
|
|
165
168
|
|
|
166
169
|
await new Promise((r) => setTimeout(r, 200))
|
|
@@ -175,7 +178,8 @@ describe('handleStartReplication', () => {
|
|
|
175
178
|
replicationPromise = handleStartReplication(
|
|
176
179
|
'START_REPLICATION SLOT "s" LOGICAL 0/0',
|
|
177
180
|
writer,
|
|
178
|
-
db
|
|
181
|
+
db,
|
|
182
|
+
testMutex
|
|
179
183
|
)
|
|
180
184
|
|
|
181
185
|
await new Promise((r) => setTimeout(r, 700))
|
|
@@ -190,7 +194,8 @@ describe('handleStartReplication', () => {
|
|
|
190
194
|
replicationPromise = handleStartReplication(
|
|
191
195
|
'START_REPLICATION SLOT "s" LOGICAL 0/0',
|
|
192
196
|
writer,
|
|
193
|
-
db
|
|
197
|
+
db,
|
|
198
|
+
testMutex
|
|
194
199
|
)
|
|
195
200
|
|
|
196
201
|
await new Promise((r) => setTimeout(r, 100))
|
|
@@ -220,7 +225,8 @@ describe('handleStartReplication', () => {
|
|
|
220
225
|
replicationPromise = handleStartReplication(
|
|
221
226
|
'START_REPLICATION SLOT "s" LOGICAL 0/0',
|
|
222
227
|
writer,
|
|
223
|
-
db
|
|
228
|
+
db,
|
|
229
|
+
testMutex
|
|
224
230
|
)
|
|
225
231
|
|
|
226
232
|
await new Promise((r) => setTimeout(r, 100))
|
|
@@ -246,7 +252,8 @@ describe('handleStartReplication', () => {
|
|
|
246
252
|
replicationPromise = handleStartReplication(
|
|
247
253
|
'START_REPLICATION SLOT "s" LOGICAL 0/0',
|
|
248
254
|
writer,
|
|
249
|
-
db
|
|
255
|
+
db,
|
|
256
|
+
testMutex
|
|
250
257
|
)
|
|
251
258
|
|
|
252
259
|
await new Promise((r) => setTimeout(r, 100))
|
|
@@ -271,7 +278,8 @@ describe('handleStartReplication', () => {
|
|
|
271
278
|
replicationPromise = handleStartReplication(
|
|
272
279
|
'START_REPLICATION SLOT "s" LOGICAL 0/0',
|
|
273
280
|
writer,
|
|
274
|
-
db
|
|
281
|
+
db,
|
|
282
|
+
testMutex
|
|
275
283
|
)
|
|
276
284
|
|
|
277
285
|
await new Promise((r) => setTimeout(r, 100))
|
|
@@ -291,7 +299,8 @@ describe('handleStartReplication', () => {
|
|
|
291
299
|
replicationPromise = handleStartReplication(
|
|
292
300
|
'START_REPLICATION SLOT "s" LOGICAL 0/0',
|
|
293
301
|
writer,
|
|
294
|
-
db
|
|
302
|
+
db,
|
|
303
|
+
testMutex
|
|
295
304
|
)
|
|
296
305
|
|
|
297
306
|
await new Promise((r) => setTimeout(r, 100))
|
|
@@ -313,7 +322,8 @@ describe('handleStartReplication', () => {
|
|
|
313
322
|
replicationPromise = handleStartReplication(
|
|
314
323
|
'START_REPLICATION SLOT "s" LOGICAL 0/0',
|
|
315
324
|
writer,
|
|
316
|
-
db
|
|
325
|
+
db,
|
|
326
|
+
testMutex
|
|
317
327
|
)
|
|
318
328
|
|
|
319
329
|
await new Promise((r) => setTimeout(r, 100))
|
|
@@ -7,9 +7,12 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { log } from '../log.js'
|
|
10
|
+
|
|
11
|
+
import type { Mutex } from '../mutex.js'
|
|
10
12
|
import {
|
|
11
13
|
getChangesSince,
|
|
12
14
|
getCurrentWatermark,
|
|
15
|
+
installTriggersOnShardTables,
|
|
13
16
|
type ChangeRecord,
|
|
14
17
|
} from './change-tracker.js'
|
|
15
18
|
import {
|
|
@@ -198,7 +201,9 @@ export async function handleReplicationQuery(
|
|
|
198
201
|
}
|
|
199
202
|
|
|
200
203
|
if (upper.startsWith('CREATE_REPLICATION_SLOT')) {
|
|
201
|
-
const match = trimmed.match(
|
|
204
|
+
const match = trimmed.match(
|
|
205
|
+
/CREATE_REPLICATION_SLOT\s+(?:"([^"]+)"|'([^']+)'|(\S+))/i
|
|
206
|
+
)
|
|
202
207
|
const slotName = match?.[1] || match?.[2] || match?.[3] || 'zero_slot'
|
|
203
208
|
const lsn = lsnToString(nextLsn())
|
|
204
209
|
const snapshotName = `00000003-00000001-1`
|
|
@@ -238,6 +243,12 @@ export async function handleReplicationQuery(
|
|
|
238
243
|
return buildCommandComplete('ALTER ROLE')
|
|
239
244
|
}
|
|
240
245
|
|
|
246
|
+
// SET TRANSACTION - pglite rejects this if any query ran first (e.g. SET search_path).
|
|
247
|
+
// return synthetic response since pglite is single-connection and doesn't need isolation levels.
|
|
248
|
+
if (upper.startsWith('SET TRANSACTION') || upper.startsWith('SET SESSION')) {
|
|
249
|
+
return buildCommandComplete('SET')
|
|
250
|
+
}
|
|
251
|
+
|
|
241
252
|
return null
|
|
242
253
|
}
|
|
243
254
|
|
|
@@ -248,7 +259,8 @@ export async function handleReplicationQuery(
|
|
|
248
259
|
export async function handleStartReplication(
|
|
249
260
|
query: string,
|
|
250
261
|
writer: ReplicationWriter,
|
|
251
|
-
db: PGlite
|
|
262
|
+
db: PGlite,
|
|
263
|
+
mutex: Mutex
|
|
252
264
|
): Promise<void> {
|
|
253
265
|
log.debug.proxy('replication: entering streaming mode')
|
|
254
266
|
|
|
@@ -262,8 +274,23 @@ export async function handleStartReplication(
|
|
|
262
274
|
|
|
263
275
|
let lastWatermark = 0
|
|
264
276
|
|
|
265
|
-
//
|
|
266
|
-
|
|
277
|
+
// declared outside mutex block so they're accessible in the poll loop
|
|
278
|
+
const tableKeyColumns = new Map<string, Set<string>>()
|
|
279
|
+
const excludedColumns = new Map<string, Set<string>>()
|
|
280
|
+
const booleanColumns = new Map<string, Set<string>>()
|
|
281
|
+
|
|
282
|
+
// acquire mutex for all setup queries to avoid conflicting with proxy connections.
|
|
283
|
+
// the change-streamer's initial copy also queries PGlite via the proxy, and
|
|
284
|
+
// direct db.query()/db.exec() calls here bypass the proxy's mutex, causing
|
|
285
|
+
// "already in transaction" errors when they interleave.
|
|
286
|
+
await mutex.acquire()
|
|
287
|
+
try {
|
|
288
|
+
// install change tracking triggers on shard schema tables (e.g. chat_0.clients)
|
|
289
|
+
// these track zero-cache's lastMutationID for .server promise resolution
|
|
290
|
+
await installTriggersOnShardTables(db)
|
|
291
|
+
|
|
292
|
+
// set up LISTEN for real-time change notifications
|
|
293
|
+
await db.exec(`
|
|
267
294
|
CREATE OR REPLACE FUNCTION public._zero_notify_change() RETURNS TRIGGER AS $$
|
|
268
295
|
BEGIN
|
|
269
296
|
PERFORM pg_notify('_zero_changes', TG_TABLE_NAME);
|
|
@@ -272,77 +299,135 @@ export async function handleStartReplication(
|
|
|
272
299
|
$$ LANGUAGE plpgsql;
|
|
273
300
|
`)
|
|
274
301
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
302
|
+
// install notify trigger on tracked tables (use configured publication if available)
|
|
303
|
+
const pubName = process.env.ZERO_APP_PUBLICATIONS
|
|
304
|
+
let tables: { tablename: string }[]
|
|
305
|
+
if (pubName) {
|
|
306
|
+
const result = await db.query<{ tablename: string }>(
|
|
307
|
+
`SELECT tablename FROM pg_publication_tables
|
|
281
308
|
WHERE pubname = $1 AND schemaname = 'public' AND tablename NOT LIKE '_zero_%'`,
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
309
|
+
[pubName]
|
|
310
|
+
)
|
|
311
|
+
tables = result.rows
|
|
312
|
+
} else {
|
|
313
|
+
const result = await db.query<{ tablename: string }>(
|
|
314
|
+
`SELECT tablename FROM pg_tables
|
|
288
315
|
WHERE schemaname = 'public'
|
|
289
316
|
AND tablename NOT IN ('migrations', '_zero_changes')
|
|
290
317
|
AND tablename NOT LIKE '_zero_%'`
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
318
|
+
)
|
|
319
|
+
tables = result.rows
|
|
320
|
+
}
|
|
294
321
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
322
|
+
for (const { tablename } of tables) {
|
|
323
|
+
const quoted = '"' + tablename.replace(/"/g, '""') + '"'
|
|
324
|
+
await db.exec(`
|
|
298
325
|
DROP TRIGGER IF EXISTS _zero_notify_trigger ON public.${quoted};
|
|
299
326
|
CREATE TRIGGER _zero_notify_trigger
|
|
300
327
|
AFTER INSERT OR UPDATE OR DELETE ON public.${quoted}
|
|
301
328
|
FOR EACH STATEMENT EXECUTE FUNCTION public._zero_notify_change();
|
|
302
329
|
`)
|
|
303
|
-
|
|
330
|
+
}
|
|
304
331
|
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
AND
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
332
|
+
// discover shard schemas (e.g. chat_0) and install NOTIFY triggers
|
|
333
|
+
const shardSchemas = await db.query<{ nspname: string }>(
|
|
334
|
+
`SELECT nspname FROM pg_namespace
|
|
335
|
+
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'public')
|
|
336
|
+
AND nspname NOT LIKE 'pg_%'
|
|
337
|
+
AND nspname NOT LIKE 'zero_%'
|
|
338
|
+
AND nspname NOT LIKE '_zero_%'
|
|
339
|
+
AND nspname NOT LIKE '%/%'`
|
|
340
|
+
)
|
|
341
|
+
const relevantSchemas = ['public', ...shardSchemas.rows.map((r) => r.nspname)]
|
|
342
|
+
|
|
343
|
+
for (const schema of relevantSchemas) {
|
|
344
|
+
if (schema === 'public') continue
|
|
345
|
+
const shardTables = await db.query<{ tablename: string }>(
|
|
346
|
+
`SELECT tablename FROM pg_tables WHERE schemaname = $1`,
|
|
347
|
+
[schema]
|
|
348
|
+
)
|
|
349
|
+
for (const { tablename } of shardTables.rows) {
|
|
350
|
+
const quotedSchema = '"' + schema.replace(/"/g, '""') + '"'
|
|
351
|
+
const quotedTable = '"' + tablename.replace(/"/g, '""') + '"'
|
|
352
|
+
await db.exec(`
|
|
353
|
+
DROP TRIGGER IF EXISTS _zero_notify_trigger ON ${quotedSchema}.${quotedTable};
|
|
354
|
+
CREATE TRIGGER _zero_notify_trigger
|
|
355
|
+
AFTER INSERT OR UPDATE OR DELETE ON ${quotedSchema}.${quotedTable}
|
|
356
|
+
FOR EACH STATEMENT EXECUTE FUNCTION public._zero_notify_change();
|
|
357
|
+
`)
|
|
358
|
+
}
|
|
359
|
+
if (shardTables.rows.length > 0) {
|
|
360
|
+
log.debug.proxy(
|
|
361
|
+
`installed notify triggers on ${shardTables.rows.length} tables in schema "${schema}"`
|
|
362
|
+
)
|
|
363
|
+
}
|
|
321
364
|
}
|
|
322
|
-
keys.add(column_name)
|
|
323
|
-
}
|
|
324
|
-
log.debug.proxy(`loaded primary keys for ${tableKeyColumns.size} tables`)
|
|
325
365
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
366
|
+
// build primary key lookup for all relevant schemas
|
|
367
|
+
for (const schema of relevantSchemas) {
|
|
368
|
+
const pkResult = await db.query<{ table_name: string; column_name: string }>(
|
|
369
|
+
`SELECT tc.table_name, kcu.column_name
|
|
370
|
+
FROM information_schema.table_constraints tc
|
|
371
|
+
JOIN information_schema.key_column_usage kcu
|
|
372
|
+
ON tc.constraint_name = kcu.constraint_name
|
|
373
|
+
AND tc.table_schema = kcu.table_schema
|
|
374
|
+
WHERE tc.constraint_type = 'PRIMARY KEY'
|
|
375
|
+
AND tc.table_schema = $1`,
|
|
376
|
+
[schema]
|
|
377
|
+
)
|
|
378
|
+
for (const { table_name, column_name } of pkResult.rows) {
|
|
379
|
+
const key = `${schema}.${table_name}`
|
|
380
|
+
let keys = tableKeyColumns.get(key)
|
|
381
|
+
if (!keys) {
|
|
382
|
+
keys = new Set()
|
|
383
|
+
tableKeyColumns.set(key, keys)
|
|
384
|
+
}
|
|
385
|
+
keys.add(column_name)
|
|
340
386
|
}
|
|
341
|
-
cols.add(column_name)
|
|
342
387
|
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
388
|
+
log.debug.proxy(`loaded primary keys for ${tableKeyColumns.size} tables`)
|
|
389
|
+
|
|
390
|
+
// build excluded columns lookup (types zero-cache can't handle)
|
|
391
|
+
// also track boolean columns so we can send correct typeOid (16) in RELATION messages.
|
|
392
|
+
const UNSUPPORTED_TYPES = new Set(['tsvector', 'tsquery', 'USER-DEFINED'])
|
|
393
|
+
for (const schema of relevantSchemas) {
|
|
394
|
+
const colResult = await db.query<{
|
|
395
|
+
table_name: string
|
|
396
|
+
column_name: string
|
|
397
|
+
data_type: string
|
|
398
|
+
}>(
|
|
399
|
+
`SELECT table_name, column_name, data_type
|
|
400
|
+
FROM information_schema.columns
|
|
401
|
+
WHERE table_schema = $1`,
|
|
402
|
+
[schema]
|
|
403
|
+
)
|
|
404
|
+
for (const { table_name, column_name, data_type } of colResult.rows) {
|
|
405
|
+
const key = `${schema}.${table_name}`
|
|
406
|
+
if (UNSUPPORTED_TYPES.has(data_type)) {
|
|
407
|
+
let cols = excludedColumns.get(key)
|
|
408
|
+
if (!cols) {
|
|
409
|
+
cols = new Set()
|
|
410
|
+
excludedColumns.set(key, cols)
|
|
411
|
+
}
|
|
412
|
+
cols.add(column_name)
|
|
413
|
+
}
|
|
414
|
+
if (data_type === 'boolean') {
|
|
415
|
+
let cols = booleanColumns.get(key)
|
|
416
|
+
if (!cols) {
|
|
417
|
+
cols = new Set()
|
|
418
|
+
booleanColumns.set(key, cols)
|
|
419
|
+
}
|
|
420
|
+
cols.add(column_name)
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
if (excludedColumns.size > 0) {
|
|
425
|
+
log.debug.proxy(
|
|
426
|
+
`excluding unsupported columns: ${[...excludedColumns.entries()].map(([t, c]) => `${t}(${[...c].join(',')})`).join(', ')}`
|
|
427
|
+
)
|
|
428
|
+
}
|
|
429
|
+
} finally {
|
|
430
|
+
mutex.release()
|
|
346
431
|
}
|
|
347
432
|
|
|
348
433
|
// track which tables we've sent RELATION messages for
|
|
@@ -356,10 +441,25 @@ export async function handleStartReplication(
|
|
|
356
441
|
const poll = async () => {
|
|
357
442
|
while (running) {
|
|
358
443
|
try {
|
|
359
|
-
|
|
444
|
+
// acquire mutex to avoid conflicting with proxy connections
|
|
445
|
+
await mutex.acquire()
|
|
446
|
+
let changes: Awaited<ReturnType<typeof getChangesSince>>
|
|
447
|
+
try {
|
|
448
|
+
changes = await getChangesSince(db, lastWatermark, 100)
|
|
449
|
+
} finally {
|
|
450
|
+
mutex.release()
|
|
451
|
+
}
|
|
360
452
|
|
|
361
453
|
if (changes.length > 0) {
|
|
362
|
-
await streamChanges(
|
|
454
|
+
await streamChanges(
|
|
455
|
+
changes,
|
|
456
|
+
writer,
|
|
457
|
+
sentRelations,
|
|
458
|
+
txCounter++,
|
|
459
|
+
tableKeyColumns,
|
|
460
|
+
excludedColumns,
|
|
461
|
+
booleanColumns
|
|
462
|
+
)
|
|
363
463
|
lastWatermark = changes[changes.length - 1].watermark
|
|
364
464
|
}
|
|
365
465
|
|
|
@@ -391,7 +491,8 @@ async function streamChanges(
|
|
|
391
491
|
sentRelations: Set<string>,
|
|
392
492
|
txId: number,
|
|
393
493
|
tableKeyColumns: Map<string, Set<string>>,
|
|
394
|
-
excludedColumns: Map<string, Set<string
|
|
494
|
+
excludedColumns: Map<string, Set<string>>,
|
|
495
|
+
booleanColumns: Map<string, Set<string>>
|
|
395
496
|
): Promise<void> {
|
|
396
497
|
const ts = nowMicros()
|
|
397
498
|
const lsn = nextLsn()
|
|
@@ -401,8 +502,15 @@ async function streamChanges(
|
|
|
401
502
|
writer.write(wrapCopyData(beginMsg))
|
|
402
503
|
|
|
403
504
|
for (const change of changes) {
|
|
404
|
-
|
|
405
|
-
const
|
|
505
|
+
// parse schema-qualified name (schema.table or bare table)
|
|
506
|
+
const dot = change.table_name.indexOf('.')
|
|
507
|
+
const schema = dot !== -1 ? change.table_name.substring(0, dot) : 'public'
|
|
508
|
+
const tableName =
|
|
509
|
+
dot !== -1 ? change.table_name.substring(dot + 1) : change.table_name
|
|
510
|
+
const qualifiedKey = `${schema}.${tableName}`
|
|
511
|
+
|
|
512
|
+
const tableOid = getTableOid(qualifiedKey)
|
|
513
|
+
const excluded = excludedColumns.get(qualifiedKey)
|
|
406
514
|
|
|
407
515
|
// filter out unsupported columns from row data
|
|
408
516
|
let rowData = change.row_data
|
|
@@ -423,17 +531,19 @@ async function streamChanges(
|
|
|
423
531
|
const row = rowData || oldData
|
|
424
532
|
if (!row) continue
|
|
425
533
|
|
|
426
|
-
const keySet = tableKeyColumns.get(
|
|
534
|
+
const keySet = tableKeyColumns.get(qualifiedKey)
|
|
535
|
+
const boolSet = booleanColumns.get(qualifiedKey)
|
|
427
536
|
const columns = inferColumns(row).map((col) => ({
|
|
428
537
|
...col,
|
|
538
|
+
typeOid: boolSet?.has(col.name) ? 16 : col.typeOid,
|
|
429
539
|
isKey: keySet?.has(col.name) ?? false,
|
|
430
540
|
}))
|
|
431
541
|
|
|
432
542
|
// send RELATION if not yet sent
|
|
433
|
-
if (!sentRelations.has(
|
|
434
|
-
const relMsg = encodeRelation(tableOid,
|
|
543
|
+
if (!sentRelations.has(qualifiedKey)) {
|
|
544
|
+
const relMsg = encodeRelation(tableOid, schema, tableName, 0x64, columns)
|
|
435
545
|
writer.write(wrapCopyData(wrapXLogData(lsn, lsn, ts, relMsg)))
|
|
436
|
-
sentRelations.add(
|
|
546
|
+
sentRelations.add(qualifiedKey)
|
|
437
547
|
}
|
|
438
548
|
|
|
439
549
|
// send the change
|
|
@@ -147,7 +147,15 @@ function encodeTupleData(
|
|
|
147
147
|
values.push(null)
|
|
148
148
|
totalSize += 1 // 'n' byte
|
|
149
149
|
} else {
|
|
150
|
-
|
|
150
|
+
// convert to postgresql text format
|
|
151
|
+
let strVal: string
|
|
152
|
+
if (typeof val === 'boolean') {
|
|
153
|
+
strVal = val ? 't' : 'f'
|
|
154
|
+
} else if (typeof val === 'object') {
|
|
155
|
+
strVal = JSON.stringify(val)
|
|
156
|
+
} else {
|
|
157
|
+
strVal = String(val)
|
|
158
|
+
}
|
|
151
159
|
const bytes = encodeString(strVal)
|
|
152
160
|
values.push(bytes)
|
|
153
161
|
totalSize += 1 + 4 + bytes.length // 't' + len + data
|