orez 0.0.18 → 0.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/cli.js +1 -1
  2. package/dist/cli.js.map +1 -1
  3. package/dist/config.js +1 -1
  4. package/dist/config.js.map +1 -1
  5. package/dist/index.d.ts +1 -0
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +46 -12
  8. package/dist/index.js.map +1 -1
  9. package/dist/mutex.d.ts +7 -0
  10. package/dist/mutex.d.ts.map +1 -0
  11. package/dist/mutex.js +24 -0
  12. package/dist/mutex.js.map +1 -0
  13. package/dist/pg-proxy.d.ts +6 -1
  14. package/dist/pg-proxy.d.ts.map +1 -1
  15. package/dist/pg-proxy.js +64 -76
  16. package/dist/pg-proxy.js.map +1 -1
  17. package/dist/pglite-manager.d.ts +14 -1
  18. package/dist/pglite-manager.d.ts.map +1 -1
  19. package/dist/pglite-manager.js +37 -18
  20. package/dist/pglite-manager.js.map +1 -1
  21. package/dist/replication/change-tracker.d.ts +7 -0
  22. package/dist/replication/change-tracker.d.ts.map +1 -1
  23. package/dist/replication/change-tracker.js +40 -3
  24. package/dist/replication/change-tracker.js.map +1 -1
  25. package/dist/replication/handler.d.ts +2 -1
  26. package/dist/replication/handler.d.ts.map +1 -1
  27. package/dist/replication/handler.js +136 -60
  28. package/dist/replication/handler.js.map +1 -1
  29. package/dist/replication/pgoutput-encoder.d.ts.map +1 -1
  30. package/dist/replication/pgoutput-encoder.js +11 -1
  31. package/dist/replication/pgoutput-encoder.js.map +1 -1
  32. package/package.json +3 -3
  33. package/src/cli.ts +1 -1
  34. package/src/config.ts +1 -1
  35. package/src/index.ts +45 -12
  36. package/src/integration/integration.test.ts +24 -14
  37. package/src/mutex.ts +24 -0
  38. package/src/pg-proxy.ts +78 -76
  39. package/src/pglite-manager.ts +48 -18
  40. package/src/replication/change-tracker.test.ts +4 -4
  41. package/src/replication/change-tracker.ts +49 -3
  42. package/src/replication/handler.test.ts +18 -8
  43. package/src/replication/handler.ts +179 -69
  44. package/src/replication/pgoutput-encoder.ts +9 -1
  45. package/src/replication/zero-compat.test.ts +7 -3
@@ -1,4 +1,4 @@
1
- import { readFileSync, readdirSync, existsSync, mkdirSync } from 'node:fs'
1
+ import { readFileSync, readdirSync, existsSync, mkdirSync, renameSync } from 'node:fs'
2
2
  import { join, resolve } from 'node:path'
3
3
 
4
4
  import { PGlite } from '@electric-sql/pglite'
@@ -9,11 +9,22 @@ import { log } from './log.js'
9
9
 
10
10
  import type { ZeroLiteConfig } from './config.js'
11
11
 
12
- export async function createPGliteInstance(config: ZeroLiteConfig): Promise<PGlite> {
13
- const dataPath = resolve(config.dataDir, 'pgdata')
12
+ export interface PGliteInstances {
13
+ postgres: PGlite
14
+ cvr: PGlite
15
+ cdb: PGlite
16
+ }
17
+
18
+ // create a single pglite instance with given dataDir suffix
19
+ async function createInstance(
20
+ config: ZeroLiteConfig,
21
+ name: string,
22
+ withExtensions: boolean
23
+ ): Promise<PGlite> {
24
+ const dataPath = resolve(config.dataDir, `pgdata-${name}`)
14
25
  mkdirSync(dataPath, { recursive: true })
15
26
 
16
- log.debug.pglite(`creating instance at ${dataPath}`)
27
+ log.debug.pglite(`creating ${name} instance at ${dataPath}`)
17
28
  const {
18
29
  dataDir: _d,
19
30
  debug: _dbg,
@@ -24,34 +35,53 @@ export async function createPGliteInstance(config: ZeroLiteConfig): Promise<PGli
24
35
  debug: config.logLevel === 'debug' ? 1 : 0,
25
36
  relaxedDurability: true,
26
37
  ...userOpts,
27
- extensions: userOpts.extensions || { vector, pg_trgm },
38
+ extensions: withExtensions ? (userOpts.extensions || { vector, pg_trgm }) : {},
28
39
  })
29
40
 
30
41
  await db.waitReady
31
- log.debug.pglite('ready')
42
+ log.debug.pglite(`${name} ready`)
43
+ return db
44
+ }
32
45
 
33
- // ensure plpgsql is available (needed by migrations and trigger functions)
34
- await db.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
46
+ /**
47
+ * create separate pglite instances for each "database".
48
+ *
49
+ * this mirrors real postgresql where postgres, zero_cvr, and zero_cdb are
50
+ * independent databases with separate transaction contexts. each instance
51
+ * has its own session state, so transactions on one database can't be
52
+ * corrupted by queries on another.
53
+ */
54
+ export async function createPGliteInstances(config: ZeroLiteConfig): Promise<PGliteInstances> {
55
+ // migrate from old single-instance layout (pgdata → pgdata-postgres)
56
+ const oldDataPath = resolve(config.dataDir, 'pgdata')
57
+ const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
58
+ if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
59
+ renameSync(oldDataPath, newDataPath)
60
+ log.debug.pglite('migrated pgdata → pgdata-postgres')
61
+ }
62
+
63
+ // create all 3 instances in parallel (only postgres needs app extensions)
64
+ const [postgres, cvr, cdb] = await Promise.all([
65
+ createInstance(config, 'postgres', true),
66
+ createInstance(config, 'cvr', false),
67
+ createInstance(config, 'cdb', false),
68
+ ])
35
69
 
36
- // create schemas for multi-db simulation
37
- await db.exec('CREATE SCHEMA IF NOT EXISTS zero_cvr')
38
- await db.exec('CREATE SCHEMA IF NOT EXISTS zero_cdb')
70
+ // postgres-specific setup
71
+ await postgres.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
39
72
 
40
- // create empty publication for zero-cache
41
- // the app's migrations (via --on-db-ready) add specific tables to it,
42
- // excluding private tables like user/session/account.
43
- // DO NOT use FOR ALL TABLES - it auto-includes every future table.
73
+ // create empty publication for zero-cache on postgres instance
44
74
  const pubName = process.env.ZERO_APP_PUBLICATIONS || 'zero_pub'
45
- const pubs = await db.query<{ count: string }>(
75
+ const pubs = await postgres.query<{ count: string }>(
46
76
  `SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
47
77
  [pubName]
48
78
  )
49
79
  if (Number(pubs.rows[0].count) === 0) {
50
80
  const quoted = '"' + pubName.replace(/"/g, '""') + '"'
51
- await db.exec(`CREATE PUBLICATION ${quoted}`)
81
+ await postgres.exec(`CREATE PUBLICATION ${quoted}`)
52
82
  }
53
83
 
54
- return db
84
+ return { postgres, cvr, cdb }
55
85
  }
56
86
 
57
87
  export async function runMigrations(db: PGlite, config: ZeroLiteConfig): Promise<void> {
@@ -33,7 +33,7 @@ describe('change-tracker', () => {
33
33
  const changes = await getChangesSince(db, 0)
34
34
  expect(changes).toHaveLength(1)
35
35
  expect(changes[0].op).toBe('INSERT')
36
- expect(changes[0].table_name).toBe('items')
36
+ expect(changes[0].table_name).toBe('public.items')
37
37
  expect(changes[0].row_data).toMatchObject({ name: 'a', value: 1 })
38
38
  expect(changes[0].old_data).toBeNull()
39
39
  })
@@ -116,8 +116,8 @@ describe('change-tracker', () => {
116
116
 
117
117
  const changes = await getChangesSince(db, 0)
118
118
  const tables = new Set(changes.map((c) => c.table_name))
119
- expect(tables).toContain('items')
120
- expect(tables).toContain('other')
119
+ expect(tables).toContain('public.items')
120
+ expect(tables).toContain('public.other')
121
121
  })
122
122
 
123
123
  it('handles rapid inserts (50 rows)', async () => {
@@ -178,7 +178,7 @@ describe('change-tracker', () => {
178
178
  await db.exec(`INSERT INTO public."my""table" (val) VALUES ('works')`)
179
179
 
180
180
  const changes = await getChangesSince(db, 0)
181
- const special = changes.filter((c) => c.table_name === 'my"table')
181
+ const special = changes.filter((c) => c.table_name === 'public.my"table')
182
182
  expect(special).toHaveLength(1)
183
183
  expect(special[0].op).toBe('INSERT')
184
184
  expect(special[0].row_data).toMatchObject({ val: 'works' })
@@ -45,18 +45,21 @@ export async function installChangeTracking(db: PGlite): Promise<void> {
45
45
  // create trigger function
46
46
  await db.exec(`
47
47
  CREATE OR REPLACE FUNCTION public._zero_track_change() RETURNS TRIGGER AS $$
48
+ DECLARE
49
+ qualified_name TEXT;
48
50
  BEGIN
51
+ qualified_name := TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME;
49
52
  IF TG_OP = 'DELETE' THEN
50
53
  INSERT INTO public._zero_changes (table_name, op, old_data)
51
- VALUES (TG_TABLE_NAME, 'DELETE', row_to_json(OLD)::jsonb);
54
+ VALUES (qualified_name, 'DELETE', row_to_json(OLD)::jsonb);
52
55
  RETURN OLD;
53
56
  ELSIF TG_OP = 'UPDATE' THEN
54
57
  INSERT INTO public._zero_changes (table_name, op, row_data, old_data)
55
- VALUES (TG_TABLE_NAME, 'UPDATE', row_to_json(NEW)::jsonb, row_to_json(OLD)::jsonb);
58
+ VALUES (qualified_name, 'UPDATE', row_to_json(NEW)::jsonb, row_to_json(OLD)::jsonb);
56
59
  RETURN NEW;
57
60
  ELSIF TG_OP = 'INSERT' THEN
58
61
  INSERT INTO public._zero_changes (table_name, op, row_data)
59
- VALUES (TG_TABLE_NAME, 'INSERT', row_to_json(NEW)::jsonb);
62
+ VALUES (qualified_name, 'INSERT', row_to_json(NEW)::jsonb);
60
63
  RETURN NEW;
61
64
  END IF;
62
65
  RETURN NULL;
@@ -137,6 +140,49 @@ async function installTriggersOnAllTables(db: PGlite): Promise<void> {
137
140
  log.debug.pglite(`installed change tracking triggers on ${count} tables`)
138
141
  }
139
142
 
143
+ /**
144
+ * install change tracking triggers on tables in shard schemas.
145
+ * zero-cache creates shard schemas (e.g. chat_0) with clients/mutations
146
+ * tables that track mutation confirmations. these must be replicated
147
+ * for .server promises to resolve.
148
+ */
149
+ export async function installTriggersOnShardTables(db: PGlite): Promise<void> {
150
+ const result = await db.query<{ nspname: string }>(
151
+ `SELECT nspname FROM pg_namespace
152
+ WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'public')
153
+ AND nspname NOT LIKE 'pg_%'
154
+ AND nspname NOT LIKE 'zero_%'
155
+ AND nspname NOT LIKE '_zero_%'
156
+ AND nspname NOT LIKE '%/%'`
157
+ )
158
+
159
+ if (result.rows.length === 0) return
160
+
161
+ let count = 0
162
+ for (const { nspname } of result.rows) {
163
+ const tables = await db.query<{ tablename: string }>(
164
+ `SELECT tablename FROM pg_tables WHERE schemaname = $1`,
165
+ [nspname]
166
+ )
167
+
168
+ for (const { tablename } of tables.rows) {
169
+ const quotedSchema = quoteIdent(nspname)
170
+ const quotedTable = quoteIdent(tablename)
171
+ await db.exec(`
172
+ DROP TRIGGER IF EXISTS _zero_change_trigger ON ${quotedSchema}.${quotedTable};
173
+ CREATE TRIGGER _zero_change_trigger
174
+ AFTER INSERT OR UPDATE OR DELETE ON ${quotedSchema}.${quotedTable}
175
+ FOR EACH ROW EXECUTE FUNCTION public._zero_track_change();
176
+ `)
177
+ count++
178
+ }
179
+ }
180
+
181
+ if (count > 0) {
182
+ log.debug.pglite(`installed change tracking on ${count} shard tables`)
183
+ }
184
+ }
185
+
140
186
  export async function getChangesSince(
141
187
  db: PGlite,
142
188
  watermark: number,
@@ -7,6 +7,7 @@ import {
7
7
  handleStartReplication,
8
8
  type ReplicationWriter,
9
9
  } from './handler'
10
+ import { Mutex } from '../mutex'
10
11
 
11
12
  // parse wire protocol RowDescription+DataRow response into columns/values
12
13
  function parseResponse(buf: Uint8Array): { columns: string[]; values: string[] } | null {
@@ -116,6 +117,7 @@ describe('handleReplicationQuery', () => {
116
117
  describe('handleStartReplication', () => {
117
118
  let db: PGlite
118
119
  let replicationPromise: Promise<void>
120
+ const testMutex = new Mutex()
119
121
 
120
122
  beforeEach(async () => {
121
123
  db = new PGlite()
@@ -160,7 +162,8 @@ describe('handleStartReplication', () => {
160
162
  replicationPromise = handleStartReplication(
161
163
  'START_REPLICATION SLOT "s" LOGICAL 0/0',
162
164
  writer,
163
- db
165
+ db,
166
+ testMutex
164
167
  )
165
168
 
166
169
  await new Promise((r) => setTimeout(r, 200))
@@ -175,7 +178,8 @@ describe('handleStartReplication', () => {
175
178
  replicationPromise = handleStartReplication(
176
179
  'START_REPLICATION SLOT "s" LOGICAL 0/0',
177
180
  writer,
178
- db
181
+ db,
182
+ testMutex
179
183
  )
180
184
 
181
185
  await new Promise((r) => setTimeout(r, 700))
@@ -190,7 +194,8 @@ describe('handleStartReplication', () => {
190
194
  replicationPromise = handleStartReplication(
191
195
  'START_REPLICATION SLOT "s" LOGICAL 0/0',
192
196
  writer,
193
- db
197
+ db,
198
+ testMutex
194
199
  )
195
200
 
196
201
  await new Promise((r) => setTimeout(r, 100))
@@ -220,7 +225,8 @@ describe('handleStartReplication', () => {
220
225
  replicationPromise = handleStartReplication(
221
226
  'START_REPLICATION SLOT "s" LOGICAL 0/0',
222
227
  writer,
223
- db
228
+ db,
229
+ testMutex
224
230
  )
225
231
 
226
232
  await new Promise((r) => setTimeout(r, 100))
@@ -246,7 +252,8 @@ describe('handleStartReplication', () => {
246
252
  replicationPromise = handleStartReplication(
247
253
  'START_REPLICATION SLOT "s" LOGICAL 0/0',
248
254
  writer,
249
- db
255
+ db,
256
+ testMutex
250
257
  )
251
258
 
252
259
  await new Promise((r) => setTimeout(r, 100))
@@ -271,7 +278,8 @@ describe('handleStartReplication', () => {
271
278
  replicationPromise = handleStartReplication(
272
279
  'START_REPLICATION SLOT "s" LOGICAL 0/0',
273
280
  writer,
274
- db
281
+ db,
282
+ testMutex
275
283
  )
276
284
 
277
285
  await new Promise((r) => setTimeout(r, 100))
@@ -291,7 +299,8 @@ describe('handleStartReplication', () => {
291
299
  replicationPromise = handleStartReplication(
292
300
  'START_REPLICATION SLOT "s" LOGICAL 0/0',
293
301
  writer,
294
- db
302
+ db,
303
+ testMutex
295
304
  )
296
305
 
297
306
  await new Promise((r) => setTimeout(r, 100))
@@ -313,7 +322,8 @@ describe('handleStartReplication', () => {
313
322
  replicationPromise = handleStartReplication(
314
323
  'START_REPLICATION SLOT "s" LOGICAL 0/0',
315
324
  writer,
316
- db
325
+ db,
326
+ testMutex
317
327
  )
318
328
 
319
329
  await new Promise((r) => setTimeout(r, 100))
@@ -7,9 +7,12 @@
7
7
  */
8
8
 
9
9
  import { log } from '../log.js'
10
+
11
+ import type { Mutex } from '../mutex.js'
10
12
  import {
11
13
  getChangesSince,
12
14
  getCurrentWatermark,
15
+ installTriggersOnShardTables,
13
16
  type ChangeRecord,
14
17
  } from './change-tracker.js'
15
18
  import {
@@ -198,7 +201,9 @@ export async function handleReplicationQuery(
198
201
  }
199
202
 
200
203
  if (upper.startsWith('CREATE_REPLICATION_SLOT')) {
201
- const match = trimmed.match(/CREATE_REPLICATION_SLOT\s+(?:"([^"]+)"|'([^']+)'|(\S+))/i)
204
+ const match = trimmed.match(
205
+ /CREATE_REPLICATION_SLOT\s+(?:"([^"]+)"|'([^']+)'|(\S+))/i
206
+ )
202
207
  const slotName = match?.[1] || match?.[2] || match?.[3] || 'zero_slot'
203
208
  const lsn = lsnToString(nextLsn())
204
209
  const snapshotName = `00000003-00000001-1`
@@ -238,6 +243,12 @@ export async function handleReplicationQuery(
238
243
  return buildCommandComplete('ALTER ROLE')
239
244
  }
240
245
 
246
+ // SET TRANSACTION - pglite rejects this if any query ran first (e.g. SET search_path).
247
+ // return synthetic response since pglite is single-connection and doesn't need isolation levels.
248
+ if (upper.startsWith('SET TRANSACTION') || upper.startsWith('SET SESSION')) {
249
+ return buildCommandComplete('SET')
250
+ }
251
+
241
252
  return null
242
253
  }
243
254
 
@@ -248,7 +259,8 @@ export async function handleReplicationQuery(
248
259
  export async function handleStartReplication(
249
260
  query: string,
250
261
  writer: ReplicationWriter,
251
- db: PGlite
262
+ db: PGlite,
263
+ mutex: Mutex
252
264
  ): Promise<void> {
253
265
  log.debug.proxy('replication: entering streaming mode')
254
266
 
@@ -262,8 +274,23 @@ export async function handleStartReplication(
262
274
 
263
275
  let lastWatermark = 0
264
276
 
265
- // set up LISTEN for real-time change notifications
266
- await db.exec(`
277
+ // declared outside mutex block so they're accessible in the poll loop
278
+ const tableKeyColumns = new Map<string, Set<string>>()
279
+ const excludedColumns = new Map<string, Set<string>>()
280
+ const booleanColumns = new Map<string, Set<string>>()
281
+
282
+ // acquire mutex for all setup queries to avoid conflicting with proxy connections.
283
+ // the change-streamer's initial copy also queries PGlite via the proxy, and
284
+ // direct db.query()/db.exec() calls here bypass the proxy's mutex, causing
285
+ // "already in transaction" errors when they interleave.
286
+ await mutex.acquire()
287
+ try {
288
+ // install change tracking triggers on shard schema tables (e.g. chat_0.clients)
289
+ // these track zero-cache's lastMutationID for .server promise resolution
290
+ await installTriggersOnShardTables(db)
291
+
292
+ // set up LISTEN for real-time change notifications
293
+ await db.exec(`
267
294
  CREATE OR REPLACE FUNCTION public._zero_notify_change() RETURNS TRIGGER AS $$
268
295
  BEGIN
269
296
  PERFORM pg_notify('_zero_changes', TG_TABLE_NAME);
@@ -272,77 +299,135 @@ export async function handleStartReplication(
272
299
  $$ LANGUAGE plpgsql;
273
300
  `)
274
301
 
275
- // install notify trigger on tracked tables (use configured publication if available)
276
- const pubName = process.env.ZERO_APP_PUBLICATIONS
277
- let tables: { tablename: string }[]
278
- if (pubName) {
279
- const result = await db.query<{ tablename: string }>(
280
- `SELECT tablename FROM pg_publication_tables
302
+ // install notify trigger on tracked tables (use configured publication if available)
303
+ const pubName = process.env.ZERO_APP_PUBLICATIONS
304
+ let tables: { tablename: string }[]
305
+ if (pubName) {
306
+ const result = await db.query<{ tablename: string }>(
307
+ `SELECT tablename FROM pg_publication_tables
281
308
  WHERE pubname = $1 AND schemaname = 'public' AND tablename NOT LIKE '_zero_%'`,
282
- [pubName]
283
- )
284
- tables = result.rows
285
- } else {
286
- const result = await db.query<{ tablename: string }>(
287
- `SELECT tablename FROM pg_tables
309
+ [pubName]
310
+ )
311
+ tables = result.rows
312
+ } else {
313
+ const result = await db.query<{ tablename: string }>(
314
+ `SELECT tablename FROM pg_tables
288
315
  WHERE schemaname = 'public'
289
316
  AND tablename NOT IN ('migrations', '_zero_changes')
290
317
  AND tablename NOT LIKE '_zero_%'`
291
- )
292
- tables = result.rows
293
- }
318
+ )
319
+ tables = result.rows
320
+ }
294
321
 
295
- for (const { tablename } of tables) {
296
- const quoted = '"' + tablename.replace(/"/g, '""') + '"'
297
- await db.exec(`
322
+ for (const { tablename } of tables) {
323
+ const quoted = '"' + tablename.replace(/"/g, '""') + '"'
324
+ await db.exec(`
298
325
  DROP TRIGGER IF EXISTS _zero_notify_trigger ON public.${quoted};
299
326
  CREATE TRIGGER _zero_notify_trigger
300
327
  AFTER INSERT OR UPDATE OR DELETE ON public.${quoted}
301
328
  FOR EACH STATEMENT EXECUTE FUNCTION public._zero_notify_change();
302
329
  `)
303
- }
330
+ }
304
331
 
305
- // build primary key lookup for all public tables
306
- const tableKeyColumns = new Map<string, Set<string>>()
307
- const pkResult = await db.query<{ table_name: string; column_name: string }>(
308
- `SELECT tc.table_name, kcu.column_name
309
- FROM information_schema.table_constraints tc
310
- JOIN information_schema.key_column_usage kcu
311
- ON tc.constraint_name = kcu.constraint_name
312
- AND tc.table_schema = kcu.table_schema
313
- WHERE tc.constraint_type = 'PRIMARY KEY'
314
- AND tc.table_schema = 'public'`
315
- )
316
- for (const { table_name, column_name } of pkResult.rows) {
317
- let keys = tableKeyColumns.get(table_name)
318
- if (!keys) {
319
- keys = new Set()
320
- tableKeyColumns.set(table_name, keys)
332
+ // discover shard schemas (e.g. chat_0) and install NOTIFY triggers
333
+ const shardSchemas = await db.query<{ nspname: string }>(
334
+ `SELECT nspname FROM pg_namespace
335
+ WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'public')
336
+ AND nspname NOT LIKE 'pg_%'
337
+ AND nspname NOT LIKE 'zero_%'
338
+ AND nspname NOT LIKE '_zero_%'
339
+ AND nspname NOT LIKE '%/%'`
340
+ )
341
+ const relevantSchemas = ['public', ...shardSchemas.rows.map((r) => r.nspname)]
342
+
343
+ for (const schema of relevantSchemas) {
344
+ if (schema === 'public') continue
345
+ const shardTables = await db.query<{ tablename: string }>(
346
+ `SELECT tablename FROM pg_tables WHERE schemaname = $1`,
347
+ [schema]
348
+ )
349
+ for (const { tablename } of shardTables.rows) {
350
+ const quotedSchema = '"' + schema.replace(/"/g, '""') + '"'
351
+ const quotedTable = '"' + tablename.replace(/"/g, '""') + '"'
352
+ await db.exec(`
353
+ DROP TRIGGER IF EXISTS _zero_notify_trigger ON ${quotedSchema}.${quotedTable};
354
+ CREATE TRIGGER _zero_notify_trigger
355
+ AFTER INSERT OR UPDATE OR DELETE ON ${quotedSchema}.${quotedTable}
356
+ FOR EACH STATEMENT EXECUTE FUNCTION public._zero_notify_change();
357
+ `)
358
+ }
359
+ if (shardTables.rows.length > 0) {
360
+ log.debug.proxy(
361
+ `installed notify triggers on ${shardTables.rows.length} tables in schema "${schema}"`
362
+ )
363
+ }
321
364
  }
322
- keys.add(column_name)
323
- }
324
- log.debug.proxy(`loaded primary keys for ${tableKeyColumns.size} tables`)
325
365
 
326
- // build excluded columns lookup (types zero-cache can't handle)
327
- const excludedColumns = new Map<string, Set<string>>()
328
- const UNSUPPORTED_TYPES = new Set(['tsvector', 'tsquery', 'USER-DEFINED'])
329
- const colResult = await db.query<{ table_name: string; column_name: string; data_type: string }>(
330
- `SELECT table_name, column_name, data_type
331
- FROM information_schema.columns
332
- WHERE table_schema = 'public'`
333
- )
334
- for (const { table_name, column_name, data_type } of colResult.rows) {
335
- if (UNSUPPORTED_TYPES.has(data_type)) {
336
- let cols = excludedColumns.get(table_name)
337
- if (!cols) {
338
- cols = new Set()
339
- excludedColumns.set(table_name, cols)
366
+ // build primary key lookup for all relevant schemas
367
+ for (const schema of relevantSchemas) {
368
+ const pkResult = await db.query<{ table_name: string; column_name: string }>(
369
+ `SELECT tc.table_name, kcu.column_name
370
+ FROM information_schema.table_constraints tc
371
+ JOIN information_schema.key_column_usage kcu
372
+ ON tc.constraint_name = kcu.constraint_name
373
+ AND tc.table_schema = kcu.table_schema
374
+ WHERE tc.constraint_type = 'PRIMARY KEY'
375
+ AND tc.table_schema = $1`,
376
+ [schema]
377
+ )
378
+ for (const { table_name, column_name } of pkResult.rows) {
379
+ const key = `${schema}.${table_name}`
380
+ let keys = tableKeyColumns.get(key)
381
+ if (!keys) {
382
+ keys = new Set()
383
+ tableKeyColumns.set(key, keys)
384
+ }
385
+ keys.add(column_name)
340
386
  }
341
- cols.add(column_name)
342
387
  }
343
- }
344
- if (excludedColumns.size > 0) {
345
- log.debug.proxy(`excluding unsupported columns: ${[...excludedColumns.entries()].map(([t, c]) => `${t}(${[...c].join(',')})`).join(', ')}`)
388
+ log.debug.proxy(`loaded primary keys for ${tableKeyColumns.size} tables`)
389
+
390
+ // build excluded columns lookup (types zero-cache can't handle)
391
+ // also track boolean columns so we can send correct typeOid (16) in RELATION messages.
392
+ const UNSUPPORTED_TYPES = new Set(['tsvector', 'tsquery', 'USER-DEFINED'])
393
+ for (const schema of relevantSchemas) {
394
+ const colResult = await db.query<{
395
+ table_name: string
396
+ column_name: string
397
+ data_type: string
398
+ }>(
399
+ `SELECT table_name, column_name, data_type
400
+ FROM information_schema.columns
401
+ WHERE table_schema = $1`,
402
+ [schema]
403
+ )
404
+ for (const { table_name, column_name, data_type } of colResult.rows) {
405
+ const key = `${schema}.${table_name}`
406
+ if (UNSUPPORTED_TYPES.has(data_type)) {
407
+ let cols = excludedColumns.get(key)
408
+ if (!cols) {
409
+ cols = new Set()
410
+ excludedColumns.set(key, cols)
411
+ }
412
+ cols.add(column_name)
413
+ }
414
+ if (data_type === 'boolean') {
415
+ let cols = booleanColumns.get(key)
416
+ if (!cols) {
417
+ cols = new Set()
418
+ booleanColumns.set(key, cols)
419
+ }
420
+ cols.add(column_name)
421
+ }
422
+ }
423
+ }
424
+ if (excludedColumns.size > 0) {
425
+ log.debug.proxy(
426
+ `excluding unsupported columns: ${[...excludedColumns.entries()].map(([t, c]) => `${t}(${[...c].join(',')})`).join(', ')}`
427
+ )
428
+ }
429
+ } finally {
430
+ mutex.release()
346
431
  }
347
432
 
348
433
  // track which tables we've sent RELATION messages for
@@ -356,10 +441,25 @@ export async function handleStartReplication(
356
441
  const poll = async () => {
357
442
  while (running) {
358
443
  try {
359
- const changes = await getChangesSince(db, lastWatermark, 100)
444
+ // acquire mutex to avoid conflicting with proxy connections
445
+ await mutex.acquire()
446
+ let changes: Awaited<ReturnType<typeof getChangesSince>>
447
+ try {
448
+ changes = await getChangesSince(db, lastWatermark, 100)
449
+ } finally {
450
+ mutex.release()
451
+ }
360
452
 
361
453
  if (changes.length > 0) {
362
- await streamChanges(changes, writer, sentRelations, txCounter++, tableKeyColumns, excludedColumns)
454
+ await streamChanges(
455
+ changes,
456
+ writer,
457
+ sentRelations,
458
+ txCounter++,
459
+ tableKeyColumns,
460
+ excludedColumns,
461
+ booleanColumns
462
+ )
363
463
  lastWatermark = changes[changes.length - 1].watermark
364
464
  }
365
465
 
@@ -391,7 +491,8 @@ async function streamChanges(
391
491
  sentRelations: Set<string>,
392
492
  txId: number,
393
493
  tableKeyColumns: Map<string, Set<string>>,
394
- excludedColumns: Map<string, Set<string>>
494
+ excludedColumns: Map<string, Set<string>>,
495
+ booleanColumns: Map<string, Set<string>>
395
496
  ): Promise<void> {
396
497
  const ts = nowMicros()
397
498
  const lsn = nextLsn()
@@ -401,8 +502,15 @@ async function streamChanges(
401
502
  writer.write(wrapCopyData(beginMsg))
402
503
 
403
504
  for (const change of changes) {
404
- const tableOid = getTableOid(change.table_name)
405
- const excluded = excludedColumns.get(change.table_name)
505
+ // parse schema-qualified name (schema.table or bare table)
506
+ const dot = change.table_name.indexOf('.')
507
+ const schema = dot !== -1 ? change.table_name.substring(0, dot) : 'public'
508
+ const tableName =
509
+ dot !== -1 ? change.table_name.substring(dot + 1) : change.table_name
510
+ const qualifiedKey = `${schema}.${tableName}`
511
+
512
+ const tableOid = getTableOid(qualifiedKey)
513
+ const excluded = excludedColumns.get(qualifiedKey)
406
514
 
407
515
  // filter out unsupported columns from row data
408
516
  let rowData = change.row_data
@@ -423,17 +531,19 @@ async function streamChanges(
423
531
  const row = rowData || oldData
424
532
  if (!row) continue
425
533
 
426
- const keySet = tableKeyColumns.get(change.table_name)
534
+ const keySet = tableKeyColumns.get(qualifiedKey)
535
+ const boolSet = booleanColumns.get(qualifiedKey)
427
536
  const columns = inferColumns(row).map((col) => ({
428
537
  ...col,
538
+ typeOid: boolSet?.has(col.name) ? 16 : col.typeOid,
429
539
  isKey: keySet?.has(col.name) ?? false,
430
540
  }))
431
541
 
432
542
  // send RELATION if not yet sent
433
- if (!sentRelations.has(change.table_name)) {
434
- const relMsg = encodeRelation(tableOid, 'public', change.table_name, 0x64, columns)
543
+ if (!sentRelations.has(qualifiedKey)) {
544
+ const relMsg = encodeRelation(tableOid, schema, tableName, 0x64, columns)
435
545
  writer.write(wrapCopyData(wrapXLogData(lsn, lsn, ts, relMsg)))
436
- sentRelations.add(change.table_name)
546
+ sentRelations.add(qualifiedKey)
437
547
  }
438
548
 
439
549
  // send the change
@@ -147,7 +147,15 @@ function encodeTupleData(
147
147
  values.push(null)
148
148
  totalSize += 1 // 'n' byte
149
149
  } else {
150
- const strVal = typeof val === 'object' ? JSON.stringify(val) : String(val)
150
+ // convert to postgresql text format
151
+ let strVal: string
152
+ if (typeof val === 'boolean') {
153
+ strVal = val ? 't' : 'f'
154
+ } else if (typeof val === 'object') {
155
+ strVal = JSON.stringify(val)
156
+ } else {
157
+ strVal = String(val)
158
+ }
151
159
  const bytes = encodeString(strVal)
152
160
  values.push(bytes)
153
161
  totalSize += 1 + 4 + bytes.length // 't' + len + data