orez 0.1.36 → 0.1.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-entry.js +0 -0
- package/dist/cli.js +7 -1
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -11
- package/dist/index.js.map +1 -1
- package/dist/pg-proxy.d.ts.map +1 -1
- package/dist/pg-proxy.js +8 -4
- package/dist/pg-proxy.js.map +1 -1
- package/dist/pglite-manager.d.ts +12 -0
- package/dist/pglite-manager.d.ts.map +1 -1
- package/dist/pglite-manager.js +81 -0
- package/dist/pglite-manager.js.map +1 -1
- package/dist/recovery.js +2 -2
- package/dist/recovery.js.map +1 -1
- package/dist/replication/change-tracker.js +9 -9
- package/dist/replication/change-tracker.js.map +1 -1
- package/dist/replication/handler.d.ts +12 -0
- package/dist/replication/handler.d.ts.map +1 -1
- package/dist/replication/handler.js +34 -6
- package/dist/replication/handler.js.map +1 -1
- package/dist/worker/browser-build-config.d.ts +59 -0
- package/dist/worker/browser-build-config.d.ts.map +1 -0
- package/dist/worker/browser-build-config.js +101 -0
- package/dist/worker/browser-build-config.js.map +1 -0
- package/dist/worker/browser-embed.d.ts +58 -0
- package/dist/worker/browser-embed.d.ts.map +1 -0
- package/dist/worker/browser-embed.js +195 -0
- package/dist/worker/browser-embed.js.map +1 -0
- package/dist/worker/cf-patches.d.ts +20 -0
- package/dist/worker/cf-patches.d.ts.map +1 -0
- package/dist/worker/cf-patches.js +94 -0
- package/dist/worker/cf-patches.js.map +1 -0
- package/dist/worker/index.d.ts +12 -0
- package/dist/worker/index.d.ts.map +1 -0
- package/dist/worker/index.js +105 -0
- package/dist/worker/index.js.map +1 -0
- package/dist/worker/shims/fastify.d.ts +80 -0
- package/dist/worker/shims/fastify.d.ts.map +1 -0
- package/dist/worker/shims/fastify.js +223 -0
- package/dist/worker/shims/fastify.js.map +1 -0
- package/dist/worker/shims/http-service.d.ts +104 -0
- package/dist/worker/shims/http-service.d.ts.map +1 -0
- package/dist/worker/shims/http-service.js +198 -0
- package/dist/worker/shims/http-service.js.map +1 -0
- package/dist/worker/shims/node-stub.d.ts +147 -0
- package/dist/worker/shims/node-stub.d.ts.map +1 -0
- package/dist/worker/shims/node-stub.js +204 -0
- package/dist/worker/shims/node-stub.js.map +1 -0
- package/dist/worker/shims/postgres.d.ts +115 -0
- package/dist/worker/shims/postgres.d.ts.map +1 -0
- package/dist/worker/shims/postgres.js +1181 -0
- package/dist/worker/shims/postgres.js.map +1 -0
- package/dist/worker/shims/sqlite-browser.d.ts +54 -0
- package/dist/worker/shims/sqlite-browser.d.ts.map +1 -0
- package/dist/worker/shims/sqlite-browser.js +144 -0
- package/dist/worker/shims/sqlite-browser.js.map +1 -0
- package/dist/worker/shims/sqlite.d.ts +126 -0
- package/dist/worker/shims/sqlite.d.ts.map +1 -0
- package/dist/worker/shims/sqlite.js +599 -0
- package/dist/worker/shims/sqlite.js.map +1 -0
- package/dist/worker/shims/stream-browser.d.ts +9 -0
- package/dist/worker/shims/stream-browser.d.ts.map +1 -0
- package/dist/worker/shims/stream-browser.js +13 -0
- package/dist/worker/shims/stream-browser.js.map +1 -0
- package/dist/worker/shims/ws-browser.d.ts +50 -0
- package/dist/worker/shims/ws-browser.d.ts.map +1 -0
- package/dist/worker/shims/ws-browser.js +105 -0
- package/dist/worker/shims/ws-browser.js.map +1 -0
- package/dist/worker/shims/ws.d.ts +62 -0
- package/dist/worker/shims/ws.d.ts.map +1 -0
- package/dist/worker/shims/ws.js +310 -0
- package/dist/worker/shims/ws.js.map +1 -0
- package/dist/worker/types.d.ts +57 -0
- package/dist/worker/types.d.ts.map +1 -0
- package/dist/worker/types.js +9 -0
- package/dist/worker/types.js.map +1 -0
- package/dist/worker/zero-cache-embed-cf.d.ts +63 -0
- package/dist/worker/zero-cache-embed-cf.d.ts.map +1 -0
- package/dist/worker/zero-cache-embed-cf.js +268 -0
- package/dist/worker/zero-cache-embed-cf.js.map +1 -0
- package/dist/worker/zero-cache-embed.d.ts +66 -0
- package/dist/worker/zero-cache-embed.d.ts.map +1 -0
- package/dist/worker/zero-cache-embed.js +200 -0
- package/dist/worker/zero-cache-embed.js.map +1 -0
- package/package.json +62 -3
- package/src/cli-entry.ts +0 -0
- package/src/cli.ts +8 -1
- package/src/config.ts +2 -0
- package/src/index.ts +15 -10
- package/src/integration/integration.test.ts +1 -1
- package/src/integration/restore-live-stress.test.ts +2 -2
- package/src/pg-proxy.ts +9 -4
- package/src/pglite-manager.ts +111 -0
- package/src/recovery.ts +2 -2
- package/src/replication/change-tracker.test.ts +1 -1
- package/src/replication/change-tracker.ts +9 -9
- package/src/replication/handler.test.ts +37 -0
- package/src/replication/handler.ts +46 -6
- package/src/wasm-sqlite.test.ts +2 -1
- package/src/worker/browser-build-config.test.ts +59 -0
- package/src/worker/browser-build-config.ts +105 -0
- package/src/worker/browser-embed.ts +306 -0
- package/src/worker/cf-patches.ts +114 -0
- package/src/worker/embed-integration.test.ts +321 -0
- package/src/worker/index.ts +138 -0
- package/src/worker/shims/fastify.test.ts +255 -0
- package/src/worker/shims/fastify.ts +292 -0
- package/src/worker/shims/http-service.test.ts +355 -0
- package/src/worker/shims/http-service.ts +293 -0
- package/src/worker/shims/node-stub.ts +223 -0
- package/src/worker/shims/postgres.test.ts +364 -0
- package/src/worker/shims/postgres.ts +1434 -0
- package/src/worker/shims/sqlite-browser.test.ts +233 -0
- package/src/worker/shims/sqlite-browser.ts +178 -0
- package/src/worker/shims/sqlite.test.ts +641 -0
- package/src/worker/shims/sqlite.ts +731 -0
- package/src/worker/shims/ws-browser.test.ts +184 -0
- package/src/worker/shims/ws-browser.ts +125 -0
- package/src/worker/shims/ws.test.ts +288 -0
- package/src/worker/shims/ws.ts +367 -0
- package/src/worker/types.ts +75 -0
- package/src/worker/worker-integration.test.ts +223 -0
- package/src/worker/worker.test.ts +136 -0
- package/src/worker/zero-cache-embed-cf.ts +367 -0
- package/src/worker/zero-cache-embed.ts +277 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "orez",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.38",
|
|
4
4
|
"description": "PGlite-powered zero-sync development backend. No Docker required.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -18,6 +18,65 @@
|
|
|
18
18
|
"./vite": {
|
|
19
19
|
"types": "./dist/vite-plugin.d.ts",
|
|
20
20
|
"import": "./dist/vite-plugin.js"
|
|
21
|
+
},
|
|
22
|
+
"./worker": {
|
|
23
|
+
"types": "./dist/worker/index.d.ts",
|
|
24
|
+
"import": "./dist/worker/index.js"
|
|
25
|
+
},
|
|
26
|
+
"./worker/cf-embed": {
|
|
27
|
+
"types": "./dist/worker/zero-cache-embed-cf.d.ts",
|
|
28
|
+
"import": "./dist/worker/zero-cache-embed-cf.js",
|
|
29
|
+
"default": "./dist/worker/zero-cache-embed-cf.js"
|
|
30
|
+
},
|
|
31
|
+
"./worker/cf-patches": {
|
|
32
|
+
"types": "./dist/worker/cf-patches.d.ts",
|
|
33
|
+
"import": "./dist/worker/cf-patches.js",
|
|
34
|
+
"default": "./dist/worker/cf-patches.js"
|
|
35
|
+
},
|
|
36
|
+
"./worker/shims/postgres": {
|
|
37
|
+
"types": "./dist/worker/shims/postgres.d.ts",
|
|
38
|
+
"import": "./dist/worker/shims/postgres.js",
|
|
39
|
+
"default": "./dist/worker/shims/postgres.js"
|
|
40
|
+
},
|
|
41
|
+
"./worker/shims/sqlite": {
|
|
42
|
+
"types": "./dist/worker/shims/sqlite.d.ts",
|
|
43
|
+
"import": "./dist/worker/shims/sqlite.js",
|
|
44
|
+
"default": "./dist/worker/shims/sqlite.js"
|
|
45
|
+
},
|
|
46
|
+
"./worker/shims/fastify": {
|
|
47
|
+
"types": "./dist/worker/shims/fastify.d.ts",
|
|
48
|
+
"import": "./dist/worker/shims/fastify.js",
|
|
49
|
+
"default": "./dist/worker/shims/fastify.js"
|
|
50
|
+
},
|
|
51
|
+
"./worker/shims/ws": {
|
|
52
|
+
"types": "./dist/worker/shims/ws.d.ts",
|
|
53
|
+
"import": "./dist/worker/shims/ws.js",
|
|
54
|
+
"default": "./dist/worker/shims/ws.js"
|
|
55
|
+
},
|
|
56
|
+
"./worker/shims/sqlite-browser": {
|
|
57
|
+
"types": "./dist/worker/shims/sqlite-browser.d.ts",
|
|
58
|
+
"import": "./dist/worker/shims/sqlite-browser.js",
|
|
59
|
+
"default": "./dist/worker/shims/sqlite-browser.js"
|
|
60
|
+
},
|
|
61
|
+
"./worker/shims/ws-browser": {
|
|
62
|
+
"types": "./dist/worker/shims/ws-browser.d.ts",
|
|
63
|
+
"import": "./dist/worker/shims/ws-browser.js",
|
|
64
|
+
"default": "./dist/worker/shims/ws-browser.js"
|
|
65
|
+
},
|
|
66
|
+
"./worker/shims/node-stub": {
|
|
67
|
+
"types": "./dist/worker/shims/node-stub.d.ts",
|
|
68
|
+
"import": "./dist/worker/shims/node-stub.js",
|
|
69
|
+
"default": "./dist/worker/shims/node-stub.js"
|
|
70
|
+
},
|
|
71
|
+
"./worker/browser-embed": {
|
|
72
|
+
"types": "./dist/worker/browser-embed.d.ts",
|
|
73
|
+
"import": "./dist/worker/browser-embed.js",
|
|
74
|
+
"default": "./dist/worker/browser-embed.js"
|
|
75
|
+
},
|
|
76
|
+
"./worker/browser-build-config": {
|
|
77
|
+
"types": "./dist/worker/browser-build-config.d.ts",
|
|
78
|
+
"import": "./dist/worker/browser-build-config.js",
|
|
79
|
+
"default": "./dist/worker/browser-build-config.js"
|
|
21
80
|
}
|
|
22
81
|
},
|
|
23
82
|
"bin": {
|
|
@@ -28,7 +87,7 @@
|
|
|
28
87
|
"src"
|
|
29
88
|
],
|
|
30
89
|
"scripts": {
|
|
31
|
-
"build": "tsc",
|
|
90
|
+
"build": "tsc && chmod +x dist/cli-entry.js",
|
|
32
91
|
"dev": "tsc --watch",
|
|
33
92
|
"test": "vitest run --exclude 'src/integration/' --exclude 'src/wasm-sqlite.test.ts'",
|
|
34
93
|
"test:integration": "vitest run src/integration/",
|
|
@@ -52,7 +111,7 @@
|
|
|
52
111
|
"dependencies": {
|
|
53
112
|
"@electric-sql/pglite": "0.4.1",
|
|
54
113
|
"@electric-sql/pglite-tools": "^0.3.1",
|
|
55
|
-
"bedrock-sqlite": "0.1.
|
|
114
|
+
"bedrock-sqlite": "0.1.38",
|
|
56
115
|
"citty": "^0.2.0",
|
|
57
116
|
"pg-gateway": "0.3.0-beta.4",
|
|
58
117
|
"pgsql-parser": "^17.9.11",
|
package/src/cli-entry.ts
CHANGED
|
File without changes
|
package/src/cli.ts
CHANGED
|
@@ -621,7 +621,7 @@ async function tryWireRestore(opts: {
|
|
|
621
621
|
)
|
|
622
622
|
|
|
623
623
|
// clear zero replication state (in _orez schema)
|
|
624
|
-
await sql.unsafe('TRUNCATE _orez.
|
|
624
|
+
await sql.unsafe('TRUNCATE _orez.changes').catch(() => {})
|
|
625
625
|
await sql.unsafe('TRUNCATE _orez._zero_replication_slots').catch(() => {})
|
|
626
626
|
log.orez('cleared zero replication state')
|
|
627
627
|
|
|
@@ -949,6 +949,12 @@ const main = defineCommand({
|
|
|
949
949
|
description: 'run pglite in-process instead of worker threads',
|
|
950
950
|
default: false,
|
|
951
951
|
},
|
|
952
|
+
'single-db': {
|
|
953
|
+
type: 'boolean',
|
|
954
|
+
description:
|
|
955
|
+
'use a single pglite instance for all databases (lighter for constrained environments)',
|
|
956
|
+
default: false,
|
|
957
|
+
},
|
|
952
958
|
'on-db-ready': {
|
|
953
959
|
type: 'string',
|
|
954
960
|
description: 'command to run after db+proxy are ready, before zero-cache starts',
|
|
@@ -1000,6 +1006,7 @@ const main = defineCommand({
|
|
|
1000
1006
|
disableWasmSqlite: args['disable-wasm-sqlite'],
|
|
1001
1007
|
forceWasmSqlite: args['force-wasm-sqlite'],
|
|
1002
1008
|
useWorkerThreads: !args['no-worker-threads'],
|
|
1009
|
+
singleDb: args['single-db'],
|
|
1003
1010
|
logLevel: (args['log-level'] as 'error' | 'warn' | 'info' | 'debug') || undefined,
|
|
1004
1011
|
onDbReady: args['on-db-ready'] || undefined,
|
|
1005
1012
|
onHealthy: args['on-healthy'] || undefined,
|
package/src/config.ts
CHANGED
|
@@ -18,6 +18,7 @@ export interface ZeroLiteConfig {
|
|
|
18
18
|
disableWasmSqlite: boolean
|
|
19
19
|
forceWasmSqlite: boolean
|
|
20
20
|
useWorkerThreads: boolean
|
|
21
|
+
singleDb: boolean
|
|
21
22
|
logLevel: LogLevel
|
|
22
23
|
pgliteOptions: Partial<PGliteOptions>
|
|
23
24
|
// lifecycle hooks
|
|
@@ -39,6 +40,7 @@ export function getConfig(overrides: Partial<ZeroLiteConfig> = {}): ZeroLiteConf
|
|
|
39
40
|
disableWasmSqlite: overrides.disableWasmSqlite ?? false,
|
|
40
41
|
forceWasmSqlite: overrides.forceWasmSqlite ?? false,
|
|
41
42
|
useWorkerThreads: overrides.useWorkerThreads ?? true,
|
|
43
|
+
singleDb: overrides.singleDb ?? false,
|
|
42
44
|
logLevel: overrides.logLevel || (process.env.OREZ_LOG_LEVEL as LogLevel) || 'warn',
|
|
43
45
|
pgliteOptions: overrides.pgliteOptions || {},
|
|
44
46
|
onDbReady: overrides.onDbReady,
|
package/src/index.ts
CHANGED
|
@@ -22,6 +22,8 @@ import { startPgProxy } from './pg-proxy.js'
|
|
|
22
22
|
import {
|
|
23
23
|
createPGliteInstances,
|
|
24
24
|
createPGliteWorkerInstances,
|
|
25
|
+
createSinglePGliteInstance,
|
|
26
|
+
createSinglePGliteWorkerInstance,
|
|
25
27
|
createPGliteWorker,
|
|
26
28
|
runMigrations,
|
|
27
29
|
} from './pglite-manager.js'
|
|
@@ -243,12 +245,17 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
243
245
|
writeFileSync(adminFile, String(adminPort))
|
|
244
246
|
}
|
|
245
247
|
|
|
246
|
-
// start pglite (
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
248
|
+
// start pglite instance(s).
|
|
249
|
+
// single-db mode uses one instance for all databases (lighter for constrained envs).
|
|
250
|
+
// otherwise, separate instances for postgres, zero_cvr, zero_cdb with optional
|
|
251
|
+
// worker threads for non-blocking WASM execution.
|
|
252
|
+
const instances = config.singleDb
|
|
253
|
+
? config.useWorkerThreads
|
|
254
|
+
? await createSinglePGliteWorkerInstance(config)
|
|
255
|
+
: await createSinglePGliteInstance(config)
|
|
256
|
+
: config.useWorkerThreads
|
|
257
|
+
? await createPGliteWorkerInstances(config)
|
|
258
|
+
: await createPGliteInstances(config)
|
|
252
259
|
const db = instances.postgres
|
|
253
260
|
const managedPub = getManagedPublicationConfig()
|
|
254
261
|
if (managedPub.managedByOrez) {
|
|
@@ -533,11 +540,9 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
533
540
|
|
|
534
541
|
// clear upstream replication tracking so zero-cache starts from a
|
|
535
542
|
// clean change stream baseline after full reset.
|
|
536
|
-
await db.exec(`TRUNCATE _orez.
|
|
543
|
+
await db.exec(`TRUNCATE _orez.changes`).catch(() => {})
|
|
537
544
|
await db.exec(`TRUNCATE _orez._zero_replication_slots`).catch(() => {})
|
|
538
|
-
await db
|
|
539
|
-
.exec(`ALTER SEQUENCE _orez._zero_watermark RESTART WITH 1`)
|
|
540
|
-
.catch(() => {})
|
|
545
|
+
await db.exec(`ALTER SEQUENCE _orez.watermark RESTART WITH 1`).catch(() => {})
|
|
541
546
|
log.orez('cleared upstream replication tracking state')
|
|
542
547
|
}
|
|
543
548
|
|
|
@@ -198,7 +198,7 @@ describe('orez integration', { timeout: 120000 }, () => {
|
|
|
198
198
|
const deadline = Date.now() + timeoutMs
|
|
199
199
|
while (Date.now() < deadline) {
|
|
200
200
|
const result = await pglite.query<{ count: string }>(
|
|
201
|
-
`SELECT count(*)::text as count FROM _orez.
|
|
201
|
+
`SELECT count(*)::text as count FROM _orez.changes`
|
|
202
202
|
)
|
|
203
203
|
if (Number(result.rows[0]?.count) === 0) return
|
|
204
204
|
await new Promise((r) => setTimeout(r, 100))
|
|
@@ -419,11 +419,11 @@ describe('live restore stress with connected frontend', { timeout: 360_000 }, ()
|
|
|
419
419
|
])
|
|
420
420
|
const tracked = await db.query<{ count: string }>(
|
|
421
421
|
`SELECT count(*)::text as count
|
|
422
|
-
FROM _orez.
|
|
422
|
+
FROM _orez.changes
|
|
423
423
|
WHERE table_name = 'public.restore_live_probe'`
|
|
424
424
|
)
|
|
425
425
|
if (Number(tracked.rows[0]?.count || '0') === 0) {
|
|
426
|
-
throw new Error('post-reset write was not captured in _orez.
|
|
426
|
+
throw new Error('post-reset write was not captured in _orez.changes')
|
|
427
427
|
}
|
|
428
428
|
|
|
429
429
|
await waitForPokeWithValue(downstreamAfterReset, marker, 30_000)
|
package/src/pg-proxy.ts
CHANGED
|
@@ -515,11 +515,16 @@ export async function startPgProxy(
|
|
|
515
515
|
? (dbInput as PGliteInstances)
|
|
516
516
|
: { postgres: dbInput as PGlite, cvr: dbInput as PGlite, cdb: dbInput as PGlite }
|
|
517
517
|
|
|
518
|
-
// per-instance mutexes for serializing pglite access
|
|
518
|
+
// per-instance mutexes for serializing pglite access.
|
|
519
|
+
// when all instances are the same object (single-db mode), share one mutex
|
|
520
|
+
// to prevent concurrent protocol messages on the same pglite instance.
|
|
521
|
+
const sharedInstance =
|
|
522
|
+
instances.postgres === instances.cvr && instances.postgres === instances.cdb
|
|
523
|
+
const pgMutex = new Mutex()
|
|
519
524
|
const mutexes = {
|
|
520
|
-
postgres:
|
|
521
|
-
cvr: new Mutex(),
|
|
522
|
-
cdb: new Mutex(),
|
|
525
|
+
postgres: pgMutex,
|
|
526
|
+
cvr: sharedInstance ? pgMutex : new Mutex(),
|
|
527
|
+
cdb: sharedInstance ? pgMutex : new Mutex(),
|
|
523
528
|
}
|
|
524
529
|
|
|
525
530
|
// per-instance transaction state: tracks which socket owns the current transaction
|
package/src/pglite-manager.ts
CHANGED
|
@@ -292,6 +292,117 @@ export async function createPGliteWorkerInstances(
|
|
|
292
292
|
}
|
|
293
293
|
}
|
|
294
294
|
|
|
295
|
+
/**
|
|
296
|
+
* create a single pglite instance shared across all databases.
|
|
297
|
+
*
|
|
298
|
+
* uses one instance for postgres, cvr, and cdb — much lighter than three
|
|
299
|
+
* separate instances. intended for constrained environments like cloudflare
|
|
300
|
+
* workers where running 3 pglite instances is too expensive.
|
|
301
|
+
*/
|
|
302
|
+
export async function createSinglePGliteInstance(
|
|
303
|
+
config: ZeroLiteConfig
|
|
304
|
+
): Promise<PGliteInstances> {
|
|
305
|
+
// migrate from old single-instance layout (pgdata → pgdata-postgres)
|
|
306
|
+
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
307
|
+
if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
|
|
308
|
+
const oldDataPath = resolve(config.dataDir, 'pgdata')
|
|
309
|
+
const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
|
|
310
|
+
if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
|
|
311
|
+
renameSync(oldDataPath, newDataPath)
|
|
312
|
+
log.debug.pglite('migrated pgdata → pgdata-postgres')
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
log.pglite('starting single shared pglite instance')
|
|
317
|
+
|
|
318
|
+
const db = await createInstance(config, 'postgres', true)
|
|
319
|
+
|
|
320
|
+
// postgres-specific setup
|
|
321
|
+
await db.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
322
|
+
|
|
323
|
+
// create publication only when explicitly configured
|
|
324
|
+
const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
|
|
325
|
+
if (pubName) {
|
|
326
|
+
const pubs = await db.query<{ count: string }>(
|
|
327
|
+
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
328
|
+
[pubName]
|
|
329
|
+
)
|
|
330
|
+
if (Number(pubs.rows[0].count) === 0) {
|
|
331
|
+
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
332
|
+
await db.exec(`CREATE PUBLICATION ${quoted}`)
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// same instance for all three — pg-proxy detects this and shares a mutex
|
|
337
|
+
return { postgres: db, cvr: db, cdb: db }
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* create a single worker-backed pglite instance shared across all databases.
|
|
342
|
+
*/
|
|
343
|
+
export async function createSinglePGliteWorkerInstance(
|
|
344
|
+
config: ZeroLiteConfig
|
|
345
|
+
): Promise<PGliteInstances> {
|
|
346
|
+
// migrate from old single-instance layout (pgdata → pgdata-postgres)
|
|
347
|
+
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
348
|
+
if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
|
|
349
|
+
const oldDataPath = resolve(config.dataDir, 'pgdata')
|
|
350
|
+
const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
|
|
351
|
+
if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
|
|
352
|
+
renameSync(oldDataPath, newDataPath)
|
|
353
|
+
log.debug.pglite('migrated pgdata → pgdata-postgres')
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const useMemory =
|
|
358
|
+
typeof pgliteDataDir === 'string' && pgliteDataDir.startsWith('memory://')
|
|
359
|
+
const {
|
|
360
|
+
dataDir: _ud,
|
|
361
|
+
debug: _dbg,
|
|
362
|
+
...userOpts
|
|
363
|
+
} = config.pgliteOptions as Record<string, any>
|
|
364
|
+
|
|
365
|
+
const dataPath = useMemory ? 'memory://' : resolve(config.dataDir, 'pgdata-postgres')
|
|
366
|
+
if (!useMemory) {
|
|
367
|
+
mkdirSync(dataPath, { recursive: true })
|
|
368
|
+
if (cleanStaleLocks(dataPath)) {
|
|
369
|
+
log.debug.pglite('cleaned stale locks in postgres')
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
log.pglite('starting single shared pglite worker thread')
|
|
374
|
+
|
|
375
|
+
const proxy = new PGliteWorkerProxy({
|
|
376
|
+
dataDir: dataPath,
|
|
377
|
+
name: 'postgres',
|
|
378
|
+
withExtensions: true,
|
|
379
|
+
debug: config.logLevel === 'debug' ? 1 : 0,
|
|
380
|
+
pgliteOptions: userOpts,
|
|
381
|
+
})
|
|
382
|
+
|
|
383
|
+
await proxy.waitReady
|
|
384
|
+
log.pglite('single worker thread ready')
|
|
385
|
+
|
|
386
|
+
// postgres-specific setup
|
|
387
|
+
await proxy.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
388
|
+
|
|
389
|
+
// create publication only when explicitly configured
|
|
390
|
+
const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
|
|
391
|
+
if (pubName) {
|
|
392
|
+
const pubs = await proxy.query<{ count: string }>(
|
|
393
|
+
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
394
|
+
[pubName]
|
|
395
|
+
)
|
|
396
|
+
if (Number(pubs.rows[0].count) === 0) {
|
|
397
|
+
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
398
|
+
await proxy.exec(`CREATE PUBLICATION ${quoted}`)
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const db = proxy as unknown as PGlite
|
|
403
|
+
return { postgres: db, cvr: db, cdb: db }
|
|
404
|
+
}
|
|
405
|
+
|
|
295
406
|
/** create a single worker-backed PGlite instance (for CVR/CDB recreation during reset) */
|
|
296
407
|
export function createPGliteWorker(dataDir: string, name: string): PGliteWorkerProxy {
|
|
297
408
|
return new PGliteWorkerProxy({
|
package/src/recovery.ts
CHANGED
|
@@ -100,9 +100,9 @@ export async function recoverFromCdcCorruption(ctx: RecoveryContext): Promise<vo
|
|
|
100
100
|
|
|
101
101
|
// clear upstream replication tracking
|
|
102
102
|
const db = instances.postgres
|
|
103
|
-
await db.exec(`TRUNCATE _orez.
|
|
103
|
+
await db.exec(`TRUNCATE _orez.changes`).catch(() => {})
|
|
104
104
|
await db.exec(`TRUNCATE _orez._zero_replication_slots`).catch(() => {})
|
|
105
|
-
await db.exec(`ALTER SEQUENCE _orez.
|
|
105
|
+
await db.exec(`ALTER SEQUENCE _orez.watermark RESTART WITH 1`).catch(() => {})
|
|
106
106
|
|
|
107
107
|
// drop stale shard schemas
|
|
108
108
|
const shardSchemas = await db.query<{ schemaname: string }>(
|
|
@@ -151,7 +151,7 @@ describe('change-tracker', () => {
|
|
|
151
151
|
try {
|
|
152
152
|
await db.exec(`CREATE PUBLICATION "zero_scope"`)
|
|
153
153
|
await installChangeTracking(db) // reinstall picks up publication scope
|
|
154
|
-
await db.exec(`TRUNCATE _orez.
|
|
154
|
+
await db.exec(`TRUNCATE _orez.changes`)
|
|
155
155
|
|
|
156
156
|
await db.exec(`INSERT INTO public.items (name, value) VALUES ('x', 1)`)
|
|
157
157
|
const changes = await getChangesSince(db, 0)
|
|
@@ -17,10 +17,10 @@ export async function installChangeTracking(db: PGlite): Promise<void> {
|
|
|
17
17
|
// create changes table and watermark sequence
|
|
18
18
|
// watermark is the primary key - monotonically increasing, no separate id needed
|
|
19
19
|
await db.exec(`
|
|
20
|
-
CREATE SEQUENCE IF NOT EXISTS _orez.
|
|
20
|
+
CREATE SEQUENCE IF NOT EXISTS _orez.watermark;
|
|
21
21
|
|
|
22
|
-
CREATE TABLE IF NOT EXISTS _orez.
|
|
23
|
-
watermark BIGINT NOT NULL DEFAULT nextval('_orez.
|
|
22
|
+
CREATE TABLE IF NOT EXISTS _orez.changes (
|
|
23
|
+
watermark BIGINT NOT NULL DEFAULT nextval('_orez.watermark') PRIMARY KEY,
|
|
24
24
|
table_name TEXT NOT NULL,
|
|
25
25
|
op TEXT NOT NULL CHECK (op IN ('INSERT', 'UPDATE', 'DELETE')),
|
|
26
26
|
row_data JSONB,
|
|
@@ -47,7 +47,7 @@ export async function installChangeTracking(db: PGlite): Promise<void> {
|
|
|
47
47
|
CREATE OR REPLACE FUNCTION public._zero_track_change() RETURNS TRIGGER AS $$
|
|
48
48
|
BEGIN
|
|
49
49
|
IF TG_OP = 'DELETE' THEN
|
|
50
|
-
INSERT INTO _orez.
|
|
50
|
+
INSERT INTO _orez.changes (table_name, op, old_data)
|
|
51
51
|
VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'DELETE', to_jsonb(OLD));
|
|
52
52
|
RETURN OLD;
|
|
53
53
|
ELSIF TG_OP = 'UPDATE' THEN
|
|
@@ -55,11 +55,11 @@ export async function installChangeTracking(db: PGlite): Promise<void> {
|
|
|
55
55
|
IF to_jsonb(NEW) = to_jsonb(OLD) THEN
|
|
56
56
|
RETURN NEW;
|
|
57
57
|
END IF;
|
|
58
|
-
INSERT INTO _orez.
|
|
58
|
+
INSERT INTO _orez.changes (table_name, op, row_data, old_data)
|
|
59
59
|
VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'UPDATE', to_jsonb(NEW), to_jsonb(OLD));
|
|
60
60
|
RETURN NEW;
|
|
61
61
|
ELSE
|
|
62
|
-
INSERT INTO _orez.
|
|
62
|
+
INSERT INTO _orez.changes (table_name, op, row_data)
|
|
63
63
|
VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'INSERT', to_jsonb(NEW));
|
|
64
64
|
RETURN NEW;
|
|
65
65
|
END IF;
|
|
@@ -228,7 +228,7 @@ export async function getChangesSince(
|
|
|
228
228
|
limit = 50000
|
|
229
229
|
): Promise<ChangeRecord[]> {
|
|
230
230
|
const result = await db.query<ChangeRecord>(
|
|
231
|
-
'SELECT watermark, table_name, op, row_data, old_data FROM _orez.
|
|
231
|
+
'SELECT watermark, table_name, op, row_data, old_data FROM _orez.changes WHERE watermark > $1 ORDER BY watermark LIMIT $2',
|
|
232
232
|
[watermark, limit]
|
|
233
233
|
)
|
|
234
234
|
return result.rows
|
|
@@ -239,14 +239,14 @@ export async function purgeConsumedChanges(
|
|
|
239
239
|
watermark: number
|
|
240
240
|
): Promise<number> {
|
|
241
241
|
const result = await db.exec(
|
|
242
|
-
`DELETE FROM _orez.
|
|
242
|
+
`DELETE FROM _orez.changes WHERE watermark <= ${Number(watermark)}`
|
|
243
243
|
)
|
|
244
244
|
return result[0]?.affectedRows ?? 0
|
|
245
245
|
}
|
|
246
246
|
|
|
247
247
|
export async function getCurrentWatermark(db: PGlite): Promise<number> {
|
|
248
248
|
const result = await db.query<{ last_value: string; is_called: boolean }>(
|
|
249
|
-
'SELECT last_value, is_called FROM _orez.
|
|
249
|
+
'SELECT last_value, is_called FROM _orez.watermark'
|
|
250
250
|
)
|
|
251
251
|
const { last_value, is_called } = result.rows[0]
|
|
252
252
|
if (!is_called) return 0
|
|
@@ -367,3 +367,40 @@ describe('handleStartReplication', () => {
|
|
|
367
367
|
expect(begins).toBeGreaterThanOrEqual(1)
|
|
368
368
|
}, 10_000)
|
|
369
369
|
})
|
|
370
|
+
|
|
371
|
+
describe('InProcessWriter', () => {
|
|
372
|
+
it('routes data to callback', async () => {
|
|
373
|
+
const { InProcessWriter } = await import('./handler')
|
|
374
|
+
const received: Uint8Array[] = []
|
|
375
|
+
const writer = new InProcessWriter((data) => received.push(data))
|
|
376
|
+
|
|
377
|
+
const msg = new Uint8Array([1, 2, 3])
|
|
378
|
+
writer.write(msg)
|
|
379
|
+
expect(received).toHaveLength(1)
|
|
380
|
+
expect(received[0]).toEqual(msg)
|
|
381
|
+
expect(writer.closed).toBe(false)
|
|
382
|
+
})
|
|
383
|
+
|
|
384
|
+
it('stops delivering after close', async () => {
|
|
385
|
+
const { InProcessWriter } = await import('./handler')
|
|
386
|
+
const received: Uint8Array[] = []
|
|
387
|
+
const writer = new InProcessWriter((data) => received.push(data))
|
|
388
|
+
|
|
389
|
+
writer.write(new Uint8Array([1]))
|
|
390
|
+
writer.close()
|
|
391
|
+
writer.write(new Uint8Array([2]))
|
|
392
|
+
|
|
393
|
+
expect(received).toHaveLength(1)
|
|
394
|
+
expect(writer.closed).toBe(true)
|
|
395
|
+
})
|
|
396
|
+
|
|
397
|
+
it('implements ReplicationWriter interface', async () => {
|
|
398
|
+
const { InProcessWriter } = await import('./handler')
|
|
399
|
+
const writer = new InProcessWriter(() => {})
|
|
400
|
+
|
|
401
|
+
// type check: can assign to ReplicationWriter
|
|
402
|
+
const rw: ReplicationWriter = writer
|
|
403
|
+
expect(rw.write).toBeDefined()
|
|
404
|
+
expect(rw.closed).toBe(false)
|
|
405
|
+
})
|
|
406
|
+
})
|
|
@@ -37,6 +37,34 @@ export interface ReplicationWriter {
|
|
|
37
37
|
readonly closed?: boolean
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* in-process replication writer. routes pgoutput data via callback
|
|
42
|
+
* instead of a TCP socket. used in CF Workers / embedded mode where
|
|
43
|
+
* there's no network between orez and zero-cache.
|
|
44
|
+
*/
|
|
45
|
+
export class InProcessWriter implements ReplicationWriter {
|
|
46
|
+
#onData: (data: Uint8Array) => void
|
|
47
|
+
#closed = false
|
|
48
|
+
|
|
49
|
+
constructor(onData: (data: Uint8Array) => void) {
|
|
50
|
+
this.#onData = onData
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
write(data: Uint8Array): void {
|
|
54
|
+
if (!this.#closed) {
|
|
55
|
+
this.#onData(data)
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
get closed(): boolean {
|
|
60
|
+
return this.#closed
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
close(): void {
|
|
64
|
+
this.#closed = true
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
40
68
|
// current lsn counter
|
|
41
69
|
let currentLsn = 0x1000000n
|
|
42
70
|
// persistent watermark across handler restarts so new handlers
|
|
@@ -303,7 +331,7 @@ export async function handleStartReplication(
|
|
|
303
331
|
db: PGlite,
|
|
304
332
|
mutex: Mutex
|
|
305
333
|
): Promise<void> {
|
|
306
|
-
log.repl('entering streaming mode')
|
|
334
|
+
log.debug.repl('entering streaming mode')
|
|
307
335
|
|
|
308
336
|
// send CopyBothResponse to enter streaming mode
|
|
309
337
|
const copyBoth = new Uint8Array(1 + 4 + 1 + 2)
|
|
@@ -365,7 +393,7 @@ export async function handleStartReplication(
|
|
|
365
393
|
const all = await db.query<{ tablename: string }>(
|
|
366
394
|
`SELECT tablename FROM pg_tables
|
|
367
395
|
WHERE schemaname = 'public'
|
|
368
|
-
AND tablename NOT IN ('migrations', '
|
|
396
|
+
AND tablename NOT IN ('migrations', 'changes')
|
|
369
397
|
AND tablename NOT LIKE '_zero_%'`
|
|
370
398
|
)
|
|
371
399
|
tables = all.rows
|
|
@@ -375,7 +403,7 @@ export async function handleStartReplication(
|
|
|
375
403
|
const ddlParts: string[] = [
|
|
376
404
|
`CREATE OR REPLACE FUNCTION public._zero_notify_change() RETURNS TRIGGER AS $$
|
|
377
405
|
BEGIN
|
|
378
|
-
PERFORM pg_notify('
|
|
406
|
+
PERFORM pg_notify('changes', TG_TABLE_NAME);
|
|
379
407
|
RETURN NULL;
|
|
380
408
|
END;
|
|
381
409
|
$$ LANGUAGE plpgsql;`,
|
|
@@ -577,11 +605,15 @@ export async function handleStartReplication(
|
|
|
577
605
|
// register direct wakeup so the proxy can signal us immediately
|
|
578
606
|
_replicationWakeup = wakeup
|
|
579
607
|
|
|
608
|
+
// expose on globalThis so external code (e.g. pglite-pool) can signal
|
|
609
|
+
// without importing from this module (works across separate bundles)
|
|
610
|
+
;(globalThis as any).__orez_signal_replication = wakeup
|
|
611
|
+
|
|
580
612
|
// also set up LISTEN as secondary signal
|
|
581
613
|
let unsubscribe: (() => Promise<void>) | null = null
|
|
582
614
|
try {
|
|
583
|
-
unsubscribe = await db.listen('
|
|
584
|
-
log.debug.proxy('replication: listening for
|
|
615
|
+
unsubscribe = await db.listen('changes', wakeup)
|
|
616
|
+
log.debug.proxy('replication: listening for changes notifications')
|
|
585
617
|
} catch {
|
|
586
618
|
log.debug.proxy('replication: LISTEN not available')
|
|
587
619
|
}
|
|
@@ -605,6 +637,9 @@ export async function handleStartReplication(
|
|
|
605
637
|
if (!queryPending) {
|
|
606
638
|
// check if a signal arrived while we were processing
|
|
607
639
|
if (!signalPending) {
|
|
640
|
+
log.debug.repl(
|
|
641
|
+
`waiting for signal (lastWm=${lastWatermark}, streamed=${hasStreamedOnce})`
|
|
642
|
+
)
|
|
608
643
|
const wasSignaled = await waitForWakeup(pollIntervalIdle)
|
|
609
644
|
if (writer.closed || db.closed) {
|
|
610
645
|
running = false
|
|
@@ -658,6 +693,9 @@ export async function handleStartReplication(
|
|
|
658
693
|
// try to acquire mutex without blocking proxy connections.
|
|
659
694
|
// post-sync: short backoff since writes signal us directly.
|
|
660
695
|
// pre-sync: yield more generously so zero-cache initial copy can finish.
|
|
696
|
+
log.debug.repl(
|
|
697
|
+
`pre-query: tryAcquire mutex (streamed=${hasStreamedOnce}, fails=${tryAcquireFailures})`
|
|
698
|
+
)
|
|
661
699
|
if (!mutex.tryAcquire()) {
|
|
662
700
|
if (hasStreamedOnce) {
|
|
663
701
|
// post-sync: block immediately. change query is fast (~0.5ms),
|
|
@@ -739,6 +777,7 @@ export async function handleStartReplication(
|
|
|
739
777
|
continue
|
|
740
778
|
}
|
|
741
779
|
|
|
780
|
+
log.debug.repl(`streaming ${changes.length} changes to writer`)
|
|
742
781
|
await streamChanges(
|
|
743
782
|
changes,
|
|
744
783
|
writer,
|
|
@@ -750,6 +789,7 @@ export async function handleStartReplication(
|
|
|
750
789
|
)
|
|
751
790
|
lastWatermark = batchEnd
|
|
752
791
|
lastStreamedWatermark = batchEnd
|
|
792
|
+
log.debug.repl(`streamed ok, watermark=${batchEnd}`)
|
|
753
793
|
hasStreamedOnce = true
|
|
754
794
|
|
|
755
795
|
// purge consumed changes periodically to free wasm memory
|
|
@@ -793,7 +833,7 @@ export async function handleStartReplication(
|
|
|
793
833
|
}
|
|
794
834
|
}
|
|
795
835
|
|
|
796
|
-
log.repl(`starting poll (lastWatermark=${lastWatermark})`)
|
|
836
|
+
log.debug.repl(`starting poll (lastWatermark=${lastWatermark})`)
|
|
797
837
|
try {
|
|
798
838
|
await poll()
|
|
799
839
|
} finally {
|
package/src/wasm-sqlite.test.ts
CHANGED
|
@@ -19,7 +19,8 @@ import { resolve } from 'node:path'
|
|
|
19
19
|
|
|
20
20
|
// import bedrock-sqlite directly (our wasm build)
|
|
21
21
|
// @ts-expect-error - CJS module
|
|
22
|
-
import
|
|
22
|
+
import bedrockSqlite from 'bedrock-sqlite'
|
|
23
|
+
const { Database } = bedrockSqlite
|
|
23
24
|
import { describe, test, expect, beforeEach, afterEach } from 'vitest'
|
|
24
25
|
|
|
25
26
|
// helper: temp db file
|