orez 0.1.43 → 0.1.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin/http-proxy.d.ts.map +1 -1
- package/dist/admin/http-proxy.js +3 -1
- package/dist/admin/http-proxy.js.map +1 -1
- package/dist/admin/log-store.d.ts.map +1 -1
- package/dist/admin/log-store.js +5 -1
- package/dist/admin/log-store.js.map +1 -1
- package/dist/admin/server.d.ts.map +1 -1
- package/dist/admin/server.js +25 -25
- package/dist/admin/server.js.map +1 -1
- package/dist/browser.d.ts +54 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +110 -0
- package/dist/browser.js.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -2
- package/dist/index.js.map +1 -1
- package/dist/pg-proxy-browser.d.ts +26 -0
- package/dist/pg-proxy-browser.d.ts.map +1 -0
- package/dist/pg-proxy-browser.js +1460 -0
- package/dist/pg-proxy-browser.js.map +1 -0
- package/dist/pg-proxy.d.ts.map +1 -1
- package/dist/pg-proxy.js +48 -34
- package/dist/pg-proxy.js.map +1 -1
- package/dist/pglite-ipc.d.ts.map +1 -1
- package/dist/pglite-ipc.js +3 -2
- package/dist/pglite-ipc.js.map +1 -1
- package/dist/pglite-manager.d.ts.map +1 -1
- package/dist/pglite-manager.js +33 -85
- package/dist/pglite-manager.js.map +1 -1
- package/dist/pglite-web-proxy.d.ts +38 -0
- package/dist/pglite-web-proxy.d.ts.map +1 -0
- package/dist/pglite-web-proxy.js +155 -0
- package/dist/pglite-web-proxy.js.map +1 -0
- package/dist/pglite-web-worker.d.ts +24 -0
- package/dist/pglite-web-worker.d.ts.map +1 -0
- package/dist/pglite-web-worker.js +119 -0
- package/dist/pglite-web-worker.js.map +1 -0
- package/dist/recovery.js +2 -2
- package/dist/recovery.js.map +1 -1
- package/dist/replication/change-tracker.js +9 -9
- package/dist/replication/change-tracker.js.map +1 -1
- package/dist/replication/handler.d.ts.map +1 -1
- package/dist/replication/handler.js +34 -26
- package/dist/replication/handler.js.map +1 -1
- package/dist/worker/browser-build-config.d.ts.map +1 -1
- package/dist/worker/browser-build-config.js +5 -2
- package/dist/worker/browser-build-config.js.map +1 -1
- package/dist/worker/browser-embed.d.ts.map +1 -1
- package/dist/worker/browser-embed.js +31 -26
- package/dist/worker/browser-embed.js.map +1 -1
- package/dist/worker/shims/fastify.d.ts +1 -0
- package/dist/worker/shims/fastify.d.ts.map +1 -1
- package/dist/worker/shims/fastify.js +31 -20
- package/dist/worker/shims/fastify.js.map +1 -1
- package/dist/worker/shims/postgres-browser.d.ts +12 -0
- package/dist/worker/shims/postgres-browser.d.ts.map +1 -0
- package/dist/worker/shims/postgres-browser.js +52 -0
- package/dist/worker/shims/postgres-browser.js.map +1 -0
- package/dist/worker/shims/postgres-socket.d.ts +83 -0
- package/dist/worker/shims/postgres-socket.d.ts.map +1 -0
- package/dist/worker/shims/postgres-socket.js +278 -0
- package/dist/worker/shims/postgres-socket.js.map +1 -0
- package/dist/worker/shims/postgres.d.ts.map +1 -1
- package/dist/worker/shims/postgres.js +18 -9
- package/dist/worker/shims/postgres.js.map +1 -1
- package/dist/worker/shims/stream-browser.d.ts +5 -4
- package/dist/worker/shims/stream-browser.d.ts.map +1 -1
- package/dist/worker/shims/stream-browser.js +7 -6
- package/dist/worker/shims/stream-browser.js.map +1 -1
- package/dist/worker/shims/ws-browser.d.ts.map +1 -1
- package/dist/worker/shims/ws-browser.js +43 -21
- package/dist/worker/shims/ws-browser.js.map +1 -1
- package/dist/worker/shims/ws.d.ts.map +1 -1
- package/dist/worker/shims/ws.js +81 -17
- package/dist/worker/shims/ws.js.map +1 -1
- package/package.json +11 -58
- package/src/admin/http-proxy.ts +4 -1
- package/src/admin/log-store.ts +5 -1
- package/src/admin/server.ts +26 -25
- package/src/browser.ts +195 -0
- package/src/cli.ts +1 -1
- package/src/index.ts +5 -2
- package/src/integration/integration.test.ts +1 -1
- package/src/integration/restore-live-stress.test.ts +2 -2
- package/src/pg-proxy-browser.ts +1673 -0
- package/src/pg-proxy.ts +48 -40
- package/src/pglite-ipc.ts +3 -2
- package/src/pglite-manager.ts +45 -107
- package/src/pglite-web-proxy.ts +180 -0
- package/src/pglite-web-worker.ts +132 -0
- package/src/recovery.ts +2 -2
- package/src/replication/change-tracker.test.ts +1 -1
- package/src/replication/change-tracker.ts +9 -9
- package/src/replication/handler.ts +37 -26
- package/src/worker/browser-build-config.test.ts +1 -1
- package/src/worker/browser-build-config.ts +5 -2
- package/src/worker/browser-embed.ts +33 -30
- package/src/worker/shims/fastify.ts +37 -24
- package/src/worker/shims/postgres-browser.ts +59 -0
- package/src/worker/shims/postgres-socket.test.ts +576 -0
- package/src/worker/shims/postgres-socket.ts +310 -0
- package/src/worker/shims/postgres.ts +30 -15
- package/src/worker/shims/stream-browser.ts +15 -0
- package/src/worker/shims/ws-browser.ts +38 -20
- package/src/worker/shims/ws.ts +76 -21
package/src/pg-proxy.ts
CHANGED
|
@@ -45,24 +45,45 @@ const SCHEMA_CACHE_TTL_MS = 30_000
|
|
|
45
45
|
// performance tracking
|
|
46
46
|
const proxyStats = { totalWaitMs: 0, totalExecMs: 0, count: 0, batches: 0 }
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
)
|
|
48
|
+
// query classification helpers — operate on pre-normalized (trimmed+lowercased) query strings
|
|
49
|
+
const SCHEMA_QUERY_MARKERS = [
|
|
50
|
+
'information_schema.',
|
|
51
|
+
'pg_catalog.',
|
|
52
|
+
'pg_tables',
|
|
53
|
+
'pg_namespace',
|
|
54
|
+
'pg_class',
|
|
55
|
+
'pg_attribute',
|
|
56
|
+
'pg_type',
|
|
57
|
+
'pg_publication',
|
|
58
|
+
]
|
|
59
|
+
const WRITE_PREFIXES = ['insert', 'update', 'delete', 'copy', 'truncate']
|
|
60
|
+
const DDL_PREFIXES = ['create', 'alter', 'drop']
|
|
61
|
+
const MUTATING_PREFIXES = [...WRITE_PREFIXES, ...DDL_PREFIXES]
|
|
62
|
+
|
|
63
|
+
function isCacheableNormalized(q: string): boolean {
|
|
64
|
+
// fast-fail: mutating queries are never cacheable
|
|
65
|
+
for (const p of MUTATING_PREFIXES) {
|
|
66
|
+
if (q.startsWith(p)) return false
|
|
67
|
+
}
|
|
68
|
+
// check if it touches schema/catalog tables
|
|
69
|
+
for (const marker of SCHEMA_QUERY_MARKERS) {
|
|
70
|
+
if (q.includes(marker)) return true
|
|
71
|
+
}
|
|
72
|
+
return false
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function isWriteNormalized(q: string): boolean {
|
|
76
|
+
for (const p of WRITE_PREFIXES) {
|
|
77
|
+
if (q.startsWith(p)) return true
|
|
78
|
+
}
|
|
79
|
+
return false
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function isDDLNormalized(q: string): boolean {
|
|
83
|
+
for (const p of DDL_PREFIXES) {
|
|
84
|
+
if (q.startsWith(p)) return true
|
|
85
|
+
}
|
|
86
|
+
return false
|
|
66
87
|
}
|
|
67
88
|
|
|
68
89
|
function extractQueryText(data: Uint8Array): string | null {
|
|
@@ -795,10 +816,13 @@ export async function startPgProxy(
|
|
|
795
816
|
// intercept and rewrite queries
|
|
796
817
|
data = interceptQuery(data)
|
|
797
818
|
|
|
798
|
-
//
|
|
819
|
+
// normalize query once for all classification checks
|
|
799
820
|
const isSimpleQuery = msgType === 0x51
|
|
800
821
|
const queryText = isSimpleQuery ? extractQueryText(data) : null
|
|
801
|
-
const
|
|
822
|
+
const queryNorm = queryText ? queryText.trimStart().toLowerCase() : null
|
|
823
|
+
const cacheable = queryNorm && isCacheableNormalized(queryNorm)
|
|
824
|
+
|
|
825
|
+
// cache Simple Query schema queries
|
|
802
826
|
if (cacheable) {
|
|
803
827
|
const cached = schemaQueryCache.get(queryText!)
|
|
804
828
|
if (cached && Date.now() < cached.expiresAt) {
|
|
@@ -861,15 +885,8 @@ export async function startPgProxy(
|
|
|
861
885
|
}
|
|
862
886
|
} else {
|
|
863
887
|
result = await execute()
|
|
864
|
-
if (
|
|
865
|
-
|
|
866
|
-
if (
|
|
867
|
-
q.startsWith('create') ||
|
|
868
|
-
q.startsWith('alter') ||
|
|
869
|
-
q.startsWith('drop')
|
|
870
|
-
) {
|
|
871
|
-
invalidateSchemaCache()
|
|
872
|
-
}
|
|
888
|
+
if (queryNorm && isDDLNormalized(queryNorm)) {
|
|
889
|
+
invalidateSchemaCache()
|
|
873
890
|
}
|
|
874
891
|
}
|
|
875
892
|
|
|
@@ -877,17 +894,8 @@ export async function startPgProxy(
|
|
|
877
894
|
result = stripResponseMessages(result, stripRfq)
|
|
878
895
|
|
|
879
896
|
// signal replication handler on postgres writes for instant sync
|
|
880
|
-
if (dbName === 'postgres' &&
|
|
881
|
-
|
|
882
|
-
if (
|
|
883
|
-
q.startsWith('insert') ||
|
|
884
|
-
q.startsWith('update') ||
|
|
885
|
-
q.startsWith('delete') ||
|
|
886
|
-
q.startsWith('copy') ||
|
|
887
|
-
q.startsWith('truncate')
|
|
888
|
-
) {
|
|
889
|
-
signalReplicationChange()
|
|
890
|
-
}
|
|
897
|
+
if (dbName === 'postgres' && queryNorm && isWriteNormalized(queryNorm)) {
|
|
898
|
+
signalReplicationChange()
|
|
891
899
|
}
|
|
892
900
|
|
|
893
901
|
return result
|
package/src/pglite-ipc.ts
CHANGED
|
@@ -26,13 +26,14 @@ interface PendingRequest {
|
|
|
26
26
|
const WRITE_PREFIXES = ['insert', 'update', 'delete', 'copy', 'truncate']
|
|
27
27
|
// shard-internal tables that the replication handler filters out.
|
|
28
28
|
// signaling for these just causes spurious wakeups + mutex contention.
|
|
29
|
-
|
|
29
|
+
// pre-lowercased so we don't call toLowerCase() per iteration
|
|
30
|
+
const SHARD_INTERNAL_TABLES = ['"replicas"', '"mutations"', '"replicationstate"']
|
|
30
31
|
function isReplicatedWrite(sql: string): boolean {
|
|
31
32
|
const q = sql.trimStart().toLowerCase()
|
|
32
33
|
if (!WRITE_PREFIXES.some((p) => q.startsWith(p))) return false
|
|
33
34
|
// skip shard-internal writes (zero-cache manages these, not replicated)
|
|
34
35
|
for (const t of SHARD_INTERNAL_TABLES) {
|
|
35
|
-
if (q.includes(t
|
|
36
|
+
if (q.includes(t)) return false
|
|
36
37
|
}
|
|
37
38
|
return true
|
|
38
39
|
}
|
package/src/pglite-manager.ts
CHANGED
|
@@ -78,6 +78,41 @@ export interface PGliteInstances {
|
|
|
78
78
|
cdb: PGlite
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
// shared setup extracted from the 4 factory functions below
|
|
82
|
+
|
|
83
|
+
/** migrate old single-instance pgdata dir to the new pgdata-postgres layout */
|
|
84
|
+
function migrateDataDir(config: ZeroLiteConfig): void {
|
|
85
|
+
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
86
|
+
if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
|
|
87
|
+
const oldDataPath = resolve(config.dataDir, 'pgdata')
|
|
88
|
+
const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
|
|
89
|
+
if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
|
|
90
|
+
renameSync(oldDataPath, newDataPath)
|
|
91
|
+
log.debug.pglite('migrated pgdata → pgdata-postgres')
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** create publication if ZERO_APP_PUBLICATIONS is set and publication doesn't exist */
|
|
97
|
+
async function ensurePublication(db: {
|
|
98
|
+
exec(sql: string): Promise<any>
|
|
99
|
+
query<T>(sql: string, params?: any[]): Promise<{ rows: T[] }>
|
|
100
|
+
}): Promise<void> {
|
|
101
|
+
await db.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
102
|
+
|
|
103
|
+
const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
|
|
104
|
+
if (pubName) {
|
|
105
|
+
const pubs = await db.query<{ count: string }>(
|
|
106
|
+
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
107
|
+
[pubName]
|
|
108
|
+
)
|
|
109
|
+
if (Number(pubs.rows[0].count) === 0) {
|
|
110
|
+
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
111
|
+
await db.exec(`CREATE PUBLICATION ${quoted}`)
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
81
116
|
// create a single pglite instance with given dataDir suffix
|
|
82
117
|
async function createInstance(
|
|
83
118
|
config: ZeroLiteConfig,
|
|
@@ -174,40 +209,15 @@ async function createInstance(
|
|
|
174
209
|
export async function createPGliteInstances(
|
|
175
210
|
config: ZeroLiteConfig
|
|
176
211
|
): Promise<PGliteInstances> {
|
|
177
|
-
|
|
178
|
-
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
179
|
-
if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
|
|
180
|
-
const oldDataPath = resolve(config.dataDir, 'pgdata')
|
|
181
|
-
const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
|
|
182
|
-
if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
|
|
183
|
-
renameSync(oldDataPath, newDataPath)
|
|
184
|
-
log.debug.pglite('migrated pgdata → pgdata-postgres')
|
|
185
|
-
}
|
|
186
|
-
}
|
|
212
|
+
migrateDataDir(config)
|
|
187
213
|
|
|
188
|
-
// create all 3 instances in parallel (only postgres needs app extensions)
|
|
189
214
|
const [postgres, cvr, cdb] = await Promise.all([
|
|
190
215
|
createInstance(config, 'postgres', true),
|
|
191
216
|
createInstance(config, 'cvr', false),
|
|
192
217
|
createInstance(config, 'cdb', false),
|
|
193
218
|
])
|
|
194
219
|
|
|
195
|
-
|
|
196
|
-
await postgres.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
197
|
-
|
|
198
|
-
// create publication only when explicitly configured
|
|
199
|
-
const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
|
|
200
|
-
if (pubName) {
|
|
201
|
-
const pubs = await postgres.query<{ count: string }>(
|
|
202
|
-
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
203
|
-
[pubName]
|
|
204
|
-
)
|
|
205
|
-
if (Number(pubs.rows[0].count) === 0) {
|
|
206
|
-
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
207
|
-
await postgres.exec(`CREATE PUBLICATION ${quoted}`)
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
|
|
220
|
+
await ensurePublication(postgres)
|
|
211
221
|
return { postgres, cvr, cdb }
|
|
212
222
|
}
|
|
213
223
|
|
|
@@ -221,17 +231,9 @@ export async function createPGliteInstances(
|
|
|
221
231
|
export async function createPGliteWorkerInstances(
|
|
222
232
|
config: ZeroLiteConfig
|
|
223
233
|
): Promise<PGliteInstances> {
|
|
224
|
-
|
|
225
|
-
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
226
|
-
if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
|
|
227
|
-
const oldDataPath = resolve(config.dataDir, 'pgdata')
|
|
228
|
-
const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
|
|
229
|
-
if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
|
|
230
|
-
renameSync(oldDataPath, newDataPath)
|
|
231
|
-
log.debug.pglite('migrated pgdata → pgdata-postgres')
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
+
migrateDataDir(config)
|
|
234
235
|
|
|
236
|
+
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
235
237
|
const useMemory =
|
|
236
238
|
typeof pgliteDataDir === 'string' && pgliteDataDir.startsWith('memory://')
|
|
237
239
|
const {
|
|
@@ -259,32 +261,15 @@ export async function createPGliteWorkerInstances(
|
|
|
259
261
|
|
|
260
262
|
log.pglite('starting worker threads for postgres, cvr, cdb')
|
|
261
263
|
|
|
262
|
-
// create all 3 worker proxies in parallel
|
|
263
264
|
const pgProxy = new PGliteWorkerProxy(makeWorkerConfig('postgres', true))
|
|
264
265
|
const cvrProxy = new PGliteWorkerProxy(makeWorkerConfig('cvr', false))
|
|
265
266
|
const cdbProxy = new PGliteWorkerProxy(makeWorkerConfig('cdb', false))
|
|
266
267
|
|
|
267
268
|
await Promise.all([pgProxy.waitReady, cvrProxy.waitReady, cdbProxy.waitReady])
|
|
268
|
-
|
|
269
269
|
log.pglite('all worker threads ready')
|
|
270
270
|
|
|
271
|
-
|
|
272
|
-
await pgProxy.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
273
|
-
|
|
274
|
-
// create publication only when explicitly configured
|
|
275
|
-
const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
|
|
276
|
-
if (pubName) {
|
|
277
|
-
const pubs = await pgProxy.query<{ count: string }>(
|
|
278
|
-
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
279
|
-
[pubName]
|
|
280
|
-
)
|
|
281
|
-
if (Number(pubs.rows[0].count) === 0) {
|
|
282
|
-
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
283
|
-
await pgProxy.exec(`CREATE PUBLICATION ${quoted}`)
|
|
284
|
-
}
|
|
285
|
-
}
|
|
271
|
+
await ensurePublication(pgProxy)
|
|
286
272
|
|
|
287
|
-
// cast to PGlite — our proxy implements the same interface surface
|
|
288
273
|
return {
|
|
289
274
|
postgres: pgProxy as unknown as PGlite,
|
|
290
275
|
cvr: cvrProxy as unknown as PGlite,
|
|
@@ -302,36 +287,11 @@ export async function createPGliteWorkerInstances(
|
|
|
302
287
|
export async function createSinglePGliteInstance(
|
|
303
288
|
config: ZeroLiteConfig
|
|
304
289
|
): Promise<PGliteInstances> {
|
|
305
|
-
|
|
306
|
-
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
307
|
-
if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
|
|
308
|
-
const oldDataPath = resolve(config.dataDir, 'pgdata')
|
|
309
|
-
const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
|
|
310
|
-
if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
|
|
311
|
-
renameSync(oldDataPath, newDataPath)
|
|
312
|
-
log.debug.pglite('migrated pgdata → pgdata-postgres')
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
|
|
290
|
+
migrateDataDir(config)
|
|
316
291
|
log.pglite('starting single shared pglite instance')
|
|
317
292
|
|
|
318
293
|
const db = await createInstance(config, 'postgres', true)
|
|
319
|
-
|
|
320
|
-
// postgres-specific setup
|
|
321
|
-
await db.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
322
|
-
|
|
323
|
-
// create publication only when explicitly configured
|
|
324
|
-
const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
|
|
325
|
-
if (pubName) {
|
|
326
|
-
const pubs = await db.query<{ count: string }>(
|
|
327
|
-
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
328
|
-
[pubName]
|
|
329
|
-
)
|
|
330
|
-
if (Number(pubs.rows[0].count) === 0) {
|
|
331
|
-
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
332
|
-
await db.exec(`CREATE PUBLICATION ${quoted}`)
|
|
333
|
-
}
|
|
334
|
-
}
|
|
294
|
+
await ensurePublication(db)
|
|
335
295
|
|
|
336
296
|
// same instance for all three — pg-proxy detects this and shares a mutex
|
|
337
297
|
return { postgres: db, cvr: db, cdb: db }
|
|
@@ -343,17 +303,9 @@ export async function createSinglePGliteInstance(
|
|
|
343
303
|
export async function createSinglePGliteWorkerInstance(
|
|
344
304
|
config: ZeroLiteConfig
|
|
345
305
|
): Promise<PGliteInstances> {
|
|
346
|
-
|
|
347
|
-
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
348
|
-
if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
|
|
349
|
-
const oldDataPath = resolve(config.dataDir, 'pgdata')
|
|
350
|
-
const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
|
|
351
|
-
if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
|
|
352
|
-
renameSync(oldDataPath, newDataPath)
|
|
353
|
-
log.debug.pglite('migrated pgdata → pgdata-postgres')
|
|
354
|
-
}
|
|
355
|
-
}
|
|
306
|
+
migrateDataDir(config)
|
|
356
307
|
|
|
308
|
+
const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
|
|
357
309
|
const useMemory =
|
|
358
310
|
typeof pgliteDataDir === 'string' && pgliteDataDir.startsWith('memory://')
|
|
359
311
|
const {
|
|
@@ -383,21 +335,7 @@ export async function createSinglePGliteWorkerInstance(
|
|
|
383
335
|
await proxy.waitReady
|
|
384
336
|
log.pglite('single worker thread ready')
|
|
385
337
|
|
|
386
|
-
|
|
387
|
-
await proxy.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
388
|
-
|
|
389
|
-
// create publication only when explicitly configured
|
|
390
|
-
const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
|
|
391
|
-
if (pubName) {
|
|
392
|
-
const pubs = await proxy.query<{ count: string }>(
|
|
393
|
-
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
394
|
-
[pubName]
|
|
395
|
-
)
|
|
396
|
-
if (Number(pubs.rows[0].count) === 0) {
|
|
397
|
-
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
398
|
-
await proxy.exec(`CREATE PUBLICATION ${quoted}`)
|
|
399
|
-
}
|
|
400
|
-
}
|
|
338
|
+
await ensurePublication(proxy)
|
|
401
339
|
|
|
402
340
|
const db = proxy as unknown as PGlite
|
|
403
341
|
return { postgres: db, cvr: db, cdb: db }
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PGlite Web Worker proxy — browser equivalent of pglite-ipc.ts.
|
|
3
|
+
*
|
|
4
|
+
* runs in the zero-cache worker, proxies calls to a Web Worker
|
|
5
|
+
* running the actual PGlite instance. mirrors PGliteWorkerProxy
|
|
6
|
+
* from pglite-ipc.ts but uses Web Worker postMessage instead of
|
|
7
|
+
* node worker_threads.
|
|
8
|
+
*
|
|
9
|
+
* ArrayBuffers are transferred (not copied) for execProtocolRaw
|
|
10
|
+
* to keep IPC overhead near-zero for wire protocol data.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { signalReplicationChange } from './replication/handler.js'
|
|
14
|
+
|
|
15
|
+
interface PendingRequest {
|
|
16
|
+
resolve: (value: any) => void
|
|
17
|
+
reject: (error: Error) => void
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const WRITE_PREFIXES = ['insert', 'update', 'delete', 'copy', 'truncate']
|
|
21
|
+
const SHARD_INTERNAL_TABLES = ['"replicas"', '"mutations"', '"replicationState"']
|
|
22
|
+
|
|
23
|
+
function isReplicatedWrite(sql: string): boolean {
|
|
24
|
+
const q = sql.trimStart().toLowerCase()
|
|
25
|
+
if (!WRITE_PREFIXES.some((p) => q.startsWith(p))) return false
|
|
26
|
+
for (const t of SHARD_INTERNAL_TABLES) {
|
|
27
|
+
if (q.includes(t.toLowerCase())) return false
|
|
28
|
+
}
|
|
29
|
+
return true
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export class PGliteWebProxy {
|
|
33
|
+
private worker: Worker
|
|
34
|
+
private pending = new Map<number, PendingRequest>()
|
|
35
|
+
private nextId = 1
|
|
36
|
+
private notificationCallbacks = new Map<string, Set<(payload: string) => void>>()
|
|
37
|
+
readonly name: string
|
|
38
|
+
|
|
39
|
+
readonly waitReady: Promise<void>
|
|
40
|
+
|
|
41
|
+
// PGlite compat flags
|
|
42
|
+
closed = false
|
|
43
|
+
ready = false
|
|
44
|
+
|
|
45
|
+
constructor(worker: Worker, name: string) {
|
|
46
|
+
this.name = name
|
|
47
|
+
this.worker = worker
|
|
48
|
+
|
|
49
|
+
let onReady: () => void
|
|
50
|
+
this.waitReady = new Promise<void>((resolveReady, rejectReady) => {
|
|
51
|
+
onReady = () => {
|
|
52
|
+
this.ready = true
|
|
53
|
+
resolveReady()
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const onMessage = (ev: MessageEvent) => {
|
|
57
|
+
const msg = ev.data
|
|
58
|
+
if (msg?.type === 'ready') {
|
|
59
|
+
this.worker.removeEventListener('message', onMessage)
|
|
60
|
+
this.installMessageHandler()
|
|
61
|
+
onReady()
|
|
62
|
+
} else if (msg?.type === 'error' && msg.id === 0) {
|
|
63
|
+
rejectReady(new Error(msg.message))
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
this.worker.addEventListener('message', onMessage)
|
|
68
|
+
this.worker.addEventListener('error', (ev) => {
|
|
69
|
+
rejectReady(new Error(String(ev)))
|
|
70
|
+
})
|
|
71
|
+
})
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
private installMessageHandler() {
|
|
75
|
+
this.worker.addEventListener('message', (ev: MessageEvent) => {
|
|
76
|
+
const msg = ev.data
|
|
77
|
+
if (!msg || typeof msg !== 'object') return
|
|
78
|
+
|
|
79
|
+
if (msg.type === 'notification') {
|
|
80
|
+
const callbacks = this.notificationCallbacks.get(msg.channel)
|
|
81
|
+
if (callbacks) {
|
|
82
|
+
for (const cb of callbacks) {
|
|
83
|
+
try {
|
|
84
|
+
cb(msg.payload)
|
|
85
|
+
} catch {}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const req = this.pending.get(msg.id)
|
|
92
|
+
if (!req) return
|
|
93
|
+
this.pending.delete(msg.id)
|
|
94
|
+
|
|
95
|
+
if (msg.type === 'error') {
|
|
96
|
+
const err = new Error(msg.message) as Error & { code?: string }
|
|
97
|
+
if (msg.code) err.code = msg.code
|
|
98
|
+
req.reject(err)
|
|
99
|
+
} else {
|
|
100
|
+
req.resolve(msg)
|
|
101
|
+
}
|
|
102
|
+
})
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
private send(msg: Record<string, unknown>, transfer?: Transferable[]): Promise<any> {
|
|
106
|
+
const id = this.nextId++
|
|
107
|
+
msg.id = id
|
|
108
|
+
return new Promise((resolve, reject) => {
|
|
109
|
+
this.pending.set(id, { resolve, reject })
|
|
110
|
+
if (transfer?.length) {
|
|
111
|
+
this.worker.postMessage(msg, transfer)
|
|
112
|
+
} else {
|
|
113
|
+
this.worker.postMessage(msg)
|
|
114
|
+
}
|
|
115
|
+
})
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async execProtocolRaw(
|
|
119
|
+
data: Uint8Array,
|
|
120
|
+
options?: { syncToFs?: boolean; throwOnError?: boolean }
|
|
121
|
+
): Promise<Uint8Array> {
|
|
122
|
+
// copy to a transferable buffer then transfer
|
|
123
|
+
const buf = new ArrayBuffer(data.byteLength)
|
|
124
|
+
new Uint8Array(buf).set(data)
|
|
125
|
+
const result = await this.send({ type: 'execProtocolRaw', data: buf, options }, [buf])
|
|
126
|
+
return new Uint8Array(result.data)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async query<T = any>(
|
|
130
|
+
sql: string,
|
|
131
|
+
params?: any[]
|
|
132
|
+
): Promise<{ rows: T[]; affectedRows?: number }> {
|
|
133
|
+
const result = await this.send({ type: 'query', sql, params })
|
|
134
|
+
// signal replication after writes on postgres instance (like orez-node's PGliteWorkerProxy)
|
|
135
|
+
if (this.name === 'postgres' && isReplicatedWrite(sql)) {
|
|
136
|
+
signalReplicationChange()
|
|
137
|
+
}
|
|
138
|
+
return { rows: result.rows ?? [], affectedRows: result.affectedRows }
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async exec(sql: string): Promise<{ affectedRows?: number }[]> {
|
|
142
|
+
const result = await this.send({ type: 'exec', sql })
|
|
143
|
+
if (this.name === 'postgres' && isReplicatedWrite(sql)) {
|
|
144
|
+
signalReplicationChange()
|
|
145
|
+
}
|
|
146
|
+
return result.results ?? []
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async listen(
|
|
150
|
+
channel: string,
|
|
151
|
+
callback: (payload: string) => void
|
|
152
|
+
): Promise<() => Promise<void>> {
|
|
153
|
+
let callbacks = this.notificationCallbacks.get(channel)
|
|
154
|
+
if (!callbacks) {
|
|
155
|
+
callbacks = new Set()
|
|
156
|
+
this.notificationCallbacks.set(channel, callbacks)
|
|
157
|
+
}
|
|
158
|
+
callbacks.add(callback)
|
|
159
|
+
|
|
160
|
+
const result = await this.send({ type: 'listen', channel })
|
|
161
|
+
const listenId = result.id
|
|
162
|
+
|
|
163
|
+
return async () => {
|
|
164
|
+
callbacks!.delete(callback)
|
|
165
|
+
if (callbacks!.size === 0) {
|
|
166
|
+
this.notificationCallbacks.delete(channel)
|
|
167
|
+
}
|
|
168
|
+
await this.send({ type: 'unlisten', listenId }).catch(() => {})
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
async close(): Promise<void> {
|
|
173
|
+
this.closed = true
|
|
174
|
+
this.ready = false
|
|
175
|
+
try {
|
|
176
|
+
await this.send({ type: 'close' })
|
|
177
|
+
} catch {}
|
|
178
|
+
this.worker.terminate()
|
|
179
|
+
}
|
|
180
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PGlite Web Worker — browser equivalent of pglite-worker-thread.ts.
|
|
3
|
+
*
|
|
4
|
+
* runs a single PGlite instance in a Web Worker. receives commands via
|
|
5
|
+
* postMessage, executes on PGlite, sends results back. ArrayBuffers
|
|
6
|
+
* are transferred (not copied) for execProtocolRaw.
|
|
7
|
+
*
|
|
8
|
+
* message protocol (same as pglite-worker-thread.ts):
|
|
9
|
+
* init: { type: 'init', dataDir, name, withExtensions, pgliteOptions }
|
|
10
|
+
* → { type: 'ready' }
|
|
11
|
+
*
|
|
12
|
+
* execProtocolRaw: { type: 'execProtocolRaw', id, data: ArrayBuffer, options }
|
|
13
|
+
* → { type: 'result', id, data: ArrayBuffer }
|
|
14
|
+
*
|
|
15
|
+
* query: { type: 'query', id, sql, params }
|
|
16
|
+
* → { type: 'result', id, rows, affectedRows }
|
|
17
|
+
*
|
|
18
|
+
* exec: { type: 'exec', id, sql }
|
|
19
|
+
* → { type: 'result', id, results: [{ affectedRows }] }
|
|
20
|
+
*
|
|
21
|
+
* listen/unlisten/close: same as pglite-worker-thread.ts
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
// NOTE: this file is meant to be bundled with PGlite as external
|
|
25
|
+
// the consumer provides PGlite via importScripts or ESM import
|
|
26
|
+
|
|
27
|
+
declare const self: any
|
|
28
|
+
|
|
29
|
+
const listeners = new Map<number, () => Promise<void>>()
|
|
30
|
+
let db: any // PGlite instance — type depends on how it's loaded
|
|
31
|
+
|
|
32
|
+
self.onmessage = async (ev: MessageEvent) => {
|
|
33
|
+
const msg = ev.data
|
|
34
|
+
if (!msg || typeof msg !== 'object') return
|
|
35
|
+
const { type, id } = msg
|
|
36
|
+
|
|
37
|
+
try {
|
|
38
|
+
switch (type) {
|
|
39
|
+
case 'init': {
|
|
40
|
+
// dynamically import PGlite (external, provided by consumer's bundler)
|
|
41
|
+
const { PGlite } = await import('@electric-sql/pglite')
|
|
42
|
+
db = new PGlite({
|
|
43
|
+
dataDir: msg.dataDir || 'idb://orez-pglite',
|
|
44
|
+
relaxedDurability: true,
|
|
45
|
+
...(msg.pgliteOptions || {}),
|
|
46
|
+
// extensions loaded by consumer if needed
|
|
47
|
+
})
|
|
48
|
+
await db.waitReady
|
|
49
|
+
|
|
50
|
+
// tune for throughput
|
|
51
|
+
await db.exec(`
|
|
52
|
+
SET work_mem = '16MB';
|
|
53
|
+
SET jit = off;
|
|
54
|
+
`)
|
|
55
|
+
|
|
56
|
+
self.postMessage({ type: 'ready' })
|
|
57
|
+
break
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
case 'execProtocolRaw': {
|
|
61
|
+
const input = new Uint8Array(msg.data as ArrayBuffer)
|
|
62
|
+
const result = await db.execProtocolRaw(input, msg.options)
|
|
63
|
+
const buf = new ArrayBuffer(result.byteLength)
|
|
64
|
+
new Uint8Array(buf).set(result)
|
|
65
|
+
self.postMessage({ type: 'result', id, data: buf }, [buf])
|
|
66
|
+
break
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
case 'query': {
|
|
70
|
+
const result = await db.query(msg.sql, msg.params)
|
|
71
|
+
self.postMessage({
|
|
72
|
+
type: 'result',
|
|
73
|
+
id,
|
|
74
|
+
rows: result.rows,
|
|
75
|
+
affectedRows: result.affectedRows,
|
|
76
|
+
})
|
|
77
|
+
break
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
case 'exec': {
|
|
81
|
+
const result = await db.exec(msg.sql)
|
|
82
|
+
const results = result.map((r: any) => ({ affectedRows: r.affectedRows ?? 0 }))
|
|
83
|
+
self.postMessage({ type: 'result', id, results })
|
|
84
|
+
break
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
case 'listen': {
|
|
88
|
+
const unsub = await db.listen(msg.channel, (payload: string) => {
|
|
89
|
+
self.postMessage({ type: 'notification', channel: msg.channel, payload })
|
|
90
|
+
})
|
|
91
|
+
listeners.set(id, unsub)
|
|
92
|
+
self.postMessage({ type: 'result', id })
|
|
93
|
+
break
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
case 'unlisten': {
|
|
97
|
+
const unsub = listeners.get(msg.listenId)
|
|
98
|
+
if (unsub) {
|
|
99
|
+
await unsub()
|
|
100
|
+
listeners.delete(msg.listenId)
|
|
101
|
+
}
|
|
102
|
+
self.postMessage({ type: 'result', id })
|
|
103
|
+
break
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
case 'close': {
|
|
107
|
+
for (const unsub of listeners.values()) {
|
|
108
|
+
await unsub().catch(() => {})
|
|
109
|
+
}
|
|
110
|
+
listeners.clear()
|
|
111
|
+
await db.close()
|
|
112
|
+
self.postMessage({ type: 'result', id })
|
|
113
|
+
break
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
default:
|
|
117
|
+
self.postMessage({
|
|
118
|
+
type: 'error',
|
|
119
|
+
id,
|
|
120
|
+
message: `unknown message type: ${type}`,
|
|
121
|
+
})
|
|
122
|
+
}
|
|
123
|
+
} catch (err: unknown) {
|
|
124
|
+
const error = err as { message?: string; code?: string }
|
|
125
|
+
self.postMessage({
|
|
126
|
+
type: 'error',
|
|
127
|
+
id,
|
|
128
|
+
message: error?.message || String(err),
|
|
129
|
+
code: error?.code,
|
|
130
|
+
})
|
|
131
|
+
}
|
|
132
|
+
}
|
package/src/recovery.ts
CHANGED
|
@@ -100,9 +100,9 @@ export async function recoverFromCdcCorruption(ctx: RecoveryContext): Promise<vo
|
|
|
100
100
|
|
|
101
101
|
// clear upstream replication tracking
|
|
102
102
|
const db = instances.postgres
|
|
103
|
-
await db.exec(`TRUNCATE _orez.
|
|
103
|
+
await db.exec(`TRUNCATE _orez._zero_changes`).catch(() => {})
|
|
104
104
|
await db.exec(`TRUNCATE _orez._zero_replication_slots`).catch(() => {})
|
|
105
|
-
await db.exec(`ALTER SEQUENCE _orez.
|
|
105
|
+
await db.exec(`ALTER SEQUENCE _orez._zero_watermark RESTART WITH 1`).catch(() => {})
|
|
106
106
|
|
|
107
107
|
// drop stale shard schemas
|
|
108
108
|
const shardSchemas = await db.query<{ schemaname: string }>(
|
|
@@ -151,7 +151,7 @@ describe('change-tracker', () => {
|
|
|
151
151
|
try {
|
|
152
152
|
await db.exec(`CREATE PUBLICATION "zero_scope"`)
|
|
153
153
|
await installChangeTracking(db) // reinstall picks up publication scope
|
|
154
|
-
await db.exec(`TRUNCATE _orez.
|
|
154
|
+
await db.exec(`TRUNCATE _orez._zero_changes`)
|
|
155
155
|
|
|
156
156
|
await db.exec(`INSERT INTO public.items (name, value) VALUES ('x', 1)`)
|
|
157
157
|
const changes = await getChangesSince(db, 0)
|