orez 0.0.63 → 0.0.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -31
- package/dist/admin/server.d.ts +1 -0
- package/dist/admin/server.d.ts.map +1 -1
- package/dist/admin/server.js +12 -2
- package/dist/admin/server.js.map +1 -1
- package/dist/admin/ui.d.ts.map +1 -1
- package/dist/admin/ui.js +4 -0
- package/dist/admin/ui.js.map +1 -1
- package/dist/cli.js +92 -39
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +138 -92
- package/dist/index.js.map +1 -1
- package/dist/log.d.ts +3 -0
- package/dist/log.d.ts.map +1 -1
- package/dist/log.js +22 -0
- package/dist/log.js.map +1 -1
- package/dist/pglite-manager.d.ts.map +1 -1
- package/dist/pglite-manager.js +8 -6
- package/dist/pglite-manager.js.map +1 -1
- package/dist/replication/change-tracker.d.ts.map +1 -1
- package/dist/replication/change-tracker.js +23 -21
- package/dist/replication/change-tracker.js.map +1 -1
- package/dist/replication/handler.d.ts.map +1 -1
- package/dist/replication/handler.js +12 -8
- package/dist/replication/handler.js.map +1 -1
- package/package.json +2 -2
- package/src/admin/server.ts +14 -2
- package/src/admin/ui.ts +4 -0
- package/src/cli.ts +120 -58
- package/src/index.ts +149 -91
- package/src/integration/restore-reset.test.ts +284 -0
- package/src/log.ts +25 -0
- package/src/pglite-manager.ts +11 -9
- package/src/replication/change-tracker.test.ts +17 -0
- package/src/replication/change-tracker.ts +28 -30
- package/src/replication/handler.ts +14 -8
package/src/index.ts
CHANGED
|
@@ -11,9 +11,14 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from '
|
|
|
11
11
|
import { createRequire } from 'node:module'
|
|
12
12
|
import { resolve } from 'node:path'
|
|
13
13
|
|
|
14
|
+
import {
|
|
15
|
+
createHttpLogStore,
|
|
16
|
+
startHttpProxy,
|
|
17
|
+
type HttpLogStore,
|
|
18
|
+
} from './admin/http-proxy.js'
|
|
14
19
|
import { createLogStore, type LogStore } from './admin/log-store.js'
|
|
15
20
|
import { getConfig, getConnectionString } from './config.js'
|
|
16
|
-
import { log, port, setLogLevel } from './log.js'
|
|
21
|
+
import { log, port, setLogLevel, setLogStore } from './log.js'
|
|
17
22
|
import { startPgProxy } from './pg-proxy.js'
|
|
18
23
|
import { createPGliteInstances, runMigrations } from './pglite-manager.js'
|
|
19
24
|
import { findPort } from './port.js'
|
|
@@ -97,6 +102,13 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
97
102
|
const logStore: LogStore | undefined =
|
|
98
103
|
adminPort > 0 ? createLogStore(config.dataDir) : undefined
|
|
99
104
|
|
|
105
|
+
// wire up logStore so all log.* calls flow to admin dashboard
|
|
106
|
+
setLogStore(logStore)
|
|
107
|
+
|
|
108
|
+
// create http log store for HTTP tab
|
|
109
|
+
const httpLog: HttpLogStore | undefined =
|
|
110
|
+
adminPort > 0 ? createHttpLogStore() : undefined
|
|
111
|
+
|
|
100
112
|
log.debug.orez(`data dir: ${resolve(config.dataDir)}`)
|
|
101
113
|
|
|
102
114
|
mkdirSync(config.dataDir, { recursive: true })
|
|
@@ -148,14 +160,35 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
148
160
|
// clean up stale sqlite replica from previous runs
|
|
149
161
|
cleanupStaleReplica(config)
|
|
150
162
|
|
|
163
|
+
// when admin is enabled, zero-cache runs on internal port with http proxy in front
|
|
164
|
+
let zeroInternalPort = config.zeroPort
|
|
165
|
+
let httpProxyServer: import('node:http').Server | null = null
|
|
166
|
+
if (httpLog && !config.skipZeroCache) {
|
|
167
|
+
zeroInternalPort = await findPort(config.zeroPort + 1000)
|
|
168
|
+
log.debug.orez(`http proxy: public ${config.zeroPort} → internal ${zeroInternalPort}`)
|
|
169
|
+
}
|
|
170
|
+
|
|
151
171
|
// start zero-cache
|
|
152
172
|
let zeroCacheProcess: ChildProcess | null = null
|
|
153
173
|
let zeroEnv: Record<string, string> = {}
|
|
154
174
|
if (!config.skipZeroCache) {
|
|
155
|
-
|
|
175
|
+
// use internal port when http proxy is enabled
|
|
176
|
+
const zeroConfig = httpLog ? { ...config, zeroPort: zeroInternalPort } : config
|
|
177
|
+
const result = await startZeroCache(zeroConfig, logStore)
|
|
156
178
|
zeroCacheProcess = result.process
|
|
157
179
|
zeroEnv = result.env
|
|
158
|
-
await waitForZeroCache(
|
|
180
|
+
await waitForZeroCache(zeroConfig)
|
|
181
|
+
|
|
182
|
+
// start http proxy in front of zero-cache when admin is enabled
|
|
183
|
+
if (httpLog) {
|
|
184
|
+
httpProxyServer = await startHttpProxy({
|
|
185
|
+
listenPort: config.zeroPort,
|
|
186
|
+
targetPort: zeroInternalPort,
|
|
187
|
+
httpLog,
|
|
188
|
+
})
|
|
189
|
+
log.debug.orez(`http proxy listening on ${config.zeroPort}`)
|
|
190
|
+
}
|
|
191
|
+
|
|
159
192
|
log.zero(`ready ${port(config.zeroPort, 'magenta')}`)
|
|
160
193
|
} else {
|
|
161
194
|
log.orez('skip zero-cache')
|
|
@@ -169,50 +202,82 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
169
202
|
})
|
|
170
203
|
}
|
|
171
204
|
|
|
172
|
-
|
|
173
|
-
|
|
205
|
+
const killZeroCache = async () => {
|
|
206
|
+
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
207
|
+
zeroCacheProcess.kill('SIGTERM')
|
|
208
|
+
await new Promise<void>((r) => {
|
|
209
|
+
const timeout = setTimeout(() => {
|
|
210
|
+
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
211
|
+
zeroCacheProcess.kill('SIGKILL')
|
|
212
|
+
}
|
|
213
|
+
r()
|
|
214
|
+
}, 3000)
|
|
215
|
+
zeroCacheProcess!.on('exit', () => {
|
|
216
|
+
clearTimeout(timeout)
|
|
217
|
+
r()
|
|
218
|
+
})
|
|
219
|
+
})
|
|
220
|
+
}
|
|
221
|
+
}
|
|
174
222
|
|
|
175
|
-
//
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
223
|
+
// simple restart without any state cleanup
|
|
224
|
+
const restartZeroCache = async () => {
|
|
225
|
+
await killZeroCache()
|
|
226
|
+
// use internal port when http proxy is enabled
|
|
227
|
+
const zeroConfig = httpLog ? { ...config, zeroPort: zeroInternalPort } : config
|
|
228
|
+
const result = await startZeroCache(zeroConfig, logStore)
|
|
229
|
+
zeroCacheProcess = result.process
|
|
230
|
+
zeroEnv = result.env
|
|
231
|
+
await waitForZeroCache(zeroConfig)
|
|
232
|
+
}
|
|
181
233
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
234
|
+
// unified reset function for zero state
|
|
235
|
+
// modes:
|
|
236
|
+
// 'cache-only' - deletes replica file only (fast, for minor sync issues)
|
|
237
|
+
// 'full' - deletes CVR/CDB + replica and recreates instances (for schema changes)
|
|
238
|
+
let resetInProgress = false
|
|
239
|
+
const resetFile = resolve(config.dataDir, 'orez.resetting')
|
|
240
|
+
const resetZeroState = async (mode: 'cache-only' | 'full'): Promise<void> => {
|
|
241
|
+
if (resetInProgress) {
|
|
242
|
+
log.orez('reset already in progress, skipping')
|
|
243
|
+
return
|
|
244
|
+
}
|
|
245
|
+
resetInProgress = true
|
|
246
|
+
// write marker file so pg_restore can wait for reset to complete
|
|
247
|
+
writeFileSync(resetFile, String(Date.now()))
|
|
186
248
|
|
|
187
|
-
|
|
249
|
+
try {
|
|
250
|
+
log.orez(`resetting zero state (${mode})...`)
|
|
251
|
+
|
|
252
|
+
// stop zero-cache first
|
|
253
|
+
log.orez('stopping zero-cache...')
|
|
254
|
+
await killZeroCache()
|
|
255
|
+
log.orez('zero-cache stopped')
|
|
256
|
+
|
|
257
|
+
if (mode === 'full') {
|
|
258
|
+
// give connections time to drain before closing instances
|
|
188
259
|
await new Promise((r) => setTimeout(r, 500))
|
|
189
260
|
|
|
190
261
|
// close CVR/CDB instances
|
|
191
262
|
log.orez('closing CVR/CDB...')
|
|
192
|
-
|
|
193
|
-
await instances.cvr.close()
|
|
194
|
-
} catch (e: any) {
|
|
263
|
+
await instances.cvr.close().catch((e: any) => {
|
|
195
264
|
log.debug.orez(`cvr close error (expected): ${e?.message || e}`)
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
await instances.cdb.close()
|
|
199
|
-
} catch (e: any) {
|
|
265
|
+
})
|
|
266
|
+
await instances.cdb.close().catch((e: any) => {
|
|
200
267
|
log.debug.orez(`cdb close error (expected): ${e?.message || e}`)
|
|
201
|
-
}
|
|
268
|
+
})
|
|
202
269
|
log.orez('CVR/CDB closed')
|
|
203
270
|
|
|
204
|
-
// delete
|
|
205
|
-
log.orez('deleting
|
|
271
|
+
// delete CVR/CDB data directories
|
|
272
|
+
log.orez('deleting CVR/CDB data...')
|
|
206
273
|
const { rmSync } = await import('node:fs')
|
|
207
274
|
for (const dir of ['pgdata-cvr', 'pgdata-cdb']) {
|
|
208
275
|
try {
|
|
209
276
|
rmSync(resolve(config.dataDir, dir), { recursive: true, force: true })
|
|
210
277
|
} catch {}
|
|
211
278
|
}
|
|
212
|
-
cleanupStaleReplica(config)
|
|
213
|
-
log.orez('zero state files deleted')
|
|
214
279
|
|
|
215
|
-
// recreate CVR/CDB
|
|
280
|
+
// recreate CVR/CDB instances
|
|
216
281
|
log.orez('recreating CVR/CDB...')
|
|
217
282
|
const { PGlite } = await import('@electric-sql/pglite')
|
|
218
283
|
mkdirSync(resolve(config.dataDir, 'pgdata-cvr'), { recursive: true })
|
|
@@ -228,78 +293,67 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
228
293
|
await instances.cvr.waitReady
|
|
229
294
|
await instances.cdb.waitReady
|
|
230
295
|
log.orez('CVR/CDB recreated')
|
|
296
|
+
}
|
|
231
297
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
298
|
+
// always clean up replica file
|
|
299
|
+
cleanupStaleReplica(config)
|
|
300
|
+
log.orez('replica cleaned up')
|
|
301
|
+
|
|
302
|
+
// re-run on-db-ready hook after full reset (re-runs migrations, syncs publication)
|
|
303
|
+
if (mode === 'full' && config.onDbReady) {
|
|
304
|
+
log.orez('re-running on-db-ready...')
|
|
305
|
+
const upstreamUrl = getConnectionString(config, 'postgres')
|
|
306
|
+
const cvrUrl = getConnectionString(config, 'zero_cvr')
|
|
307
|
+
const cdbUrl = getConnectionString(config, 'zero_cdb')
|
|
308
|
+
await runHook(config.onDbReady, 'on-db-ready', {
|
|
309
|
+
ZERO_UPSTREAM_DB: upstreamUrl,
|
|
310
|
+
ZERO_CVR_DB: cvrUrl,
|
|
311
|
+
ZERO_CHANGE_DB: cdbUrl,
|
|
312
|
+
DATABASE_URL: upstreamUrl,
|
|
313
|
+
OREZ_PG_PORT: String(config.pgPort),
|
|
243
314
|
})
|
|
244
315
|
|
|
245
|
-
|
|
246
|
-
log.orez('
|
|
247
|
-
|
|
248
|
-
} catch (err: any) {
|
|
249
|
-
log.orez(`SIGUSR1 reset failed: ${err?.message || err}`)
|
|
250
|
-
console.error(err)
|
|
251
|
-
} finally {
|
|
252
|
-
resettingZeroState = false
|
|
316
|
+
// re-install change tracking on any tables created/modified by on-db-ready
|
|
317
|
+
log.debug.orez('re-installing change tracking after on-db-ready')
|
|
318
|
+
await installChangeTracking(db)
|
|
253
319
|
}
|
|
254
|
-
})
|
|
255
|
-
}
|
|
256
320
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
const
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
await new Promise<void>((r) => {
|
|
278
|
-
const timeout = setTimeout(() => {
|
|
279
|
-
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
280
|
-
zeroCacheProcess.kill('SIGKILL')
|
|
281
|
-
}
|
|
282
|
-
r()
|
|
283
|
-
}, 3000)
|
|
284
|
-
zeroCacheProcess!.on('exit', () => {
|
|
285
|
-
clearTimeout(timeout)
|
|
286
|
-
r()
|
|
287
|
-
})
|
|
288
|
-
})
|
|
321
|
+
// restart zero-cache
|
|
322
|
+
log.orez('starting zero-cache...')
|
|
323
|
+
// use internal port when http proxy is enabled
|
|
324
|
+
const zeroConfig = httpLog ? { ...config, zeroPort: zeroInternalPort } : config
|
|
325
|
+
const result = await startZeroCache(zeroConfig, logStore)
|
|
326
|
+
zeroCacheProcess = result.process
|
|
327
|
+
zeroEnv = result.env
|
|
328
|
+
|
|
329
|
+
await waitForZeroCache(zeroConfig)
|
|
330
|
+
log.orez(`zero state reset complete (${mode})`)
|
|
331
|
+
log.zero(`ready ${port(config.zeroPort, 'magenta')}`)
|
|
332
|
+
} catch (err: any) {
|
|
333
|
+
log.orez(`reset failed: ${err?.message || err}`)
|
|
334
|
+
throw err
|
|
335
|
+
} finally {
|
|
336
|
+
resetInProgress = false
|
|
337
|
+
// remove marker file so pg_restore knows we're done
|
|
338
|
+
try {
|
|
339
|
+
unlinkSync(resetFile)
|
|
340
|
+
} catch {}
|
|
289
341
|
}
|
|
290
342
|
}
|
|
291
343
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
344
|
+
// handle SIGUSR1 to reset zero state (sent by pg_restore)
|
|
345
|
+
if (!config.skipZeroCache) {
|
|
346
|
+
process.on('SIGUSR1', () => {
|
|
347
|
+
log.orez('received SIGUSR1')
|
|
348
|
+
resetZeroState('full').catch((err) => {
|
|
349
|
+
log.orez(`SIGUSR1 reset failed: ${err?.message || err}`)
|
|
350
|
+
})
|
|
351
|
+
})
|
|
299
352
|
}
|
|
300
353
|
|
|
301
354
|
const stop = async () => {
|
|
302
355
|
log.debug.orez('shutting down')
|
|
356
|
+
httpProxyServer?.close()
|
|
303
357
|
await killZeroCache()
|
|
304
358
|
pgServer.close()
|
|
305
359
|
await Promise.all([
|
|
@@ -321,9 +375,13 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
321
375
|
pgPort: config.pgPort,
|
|
322
376
|
zeroPort: config.zeroPort,
|
|
323
377
|
logStore,
|
|
378
|
+
httpLog,
|
|
324
379
|
zeroEnv,
|
|
325
|
-
restartZero: config.skipZeroCache ? undefined :
|
|
326
|
-
|
|
380
|
+
restartZero: config.skipZeroCache ? undefined : restartZeroCache,
|
|
381
|
+
// cache-only reset: just replica file (fast, for minor sync issues)
|
|
382
|
+
resetZero: config.skipZeroCache ? undefined : () => resetZeroState('cache-only'),
|
|
383
|
+
// full reset: CVR/CDB + replica (for schema changes, used by pg_restore via SIGUSR1)
|
|
384
|
+
resetZeroFull: config.skipZeroCache ? undefined : () => resetZeroState('full'),
|
|
327
385
|
}
|
|
328
386
|
}
|
|
329
387
|
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* regression test for restore/reset integration.
|
|
3
|
+
*
|
|
4
|
+
* covers the real integration boundary that previously regressed:
|
|
5
|
+
* - restore data through wire protocol
|
|
6
|
+
* - trigger full zero-state reset via pid-file + SIGUSR1 (same path as pg_restore)
|
|
7
|
+
* - verify zero-cache restarts and live replication still works
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { existsSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
11
|
+
import { homedir, tmpdir } from 'node:os'
|
|
12
|
+
import { join } from 'node:path'
|
|
13
|
+
|
|
14
|
+
import { loadModule } from 'pgsql-parser'
|
|
15
|
+
import postgres from 'postgres'
|
|
16
|
+
import { afterAll, beforeAll, describe, expect, test } from 'vitest'
|
|
17
|
+
import WebSocket from 'ws'
|
|
18
|
+
|
|
19
|
+
import { execDumpFile } from '../cli.js'
|
|
20
|
+
import { startZeroLite } from '../index.js'
|
|
21
|
+
|
|
22
|
+
import type { PGlite } from '@electric-sql/pglite'
|
|
23
|
+
|
|
24
|
+
class Queue<T> {
|
|
25
|
+
private items: T[] = []
|
|
26
|
+
private waiters: Array<{
|
|
27
|
+
resolve: (v: T) => void
|
|
28
|
+
timer?: ReturnType<typeof setTimeout>
|
|
29
|
+
}> = []
|
|
30
|
+
|
|
31
|
+
enqueue(item: T) {
|
|
32
|
+
const waiter = this.waiters.shift()
|
|
33
|
+
if (waiter) {
|
|
34
|
+
if (waiter.timer) clearTimeout(waiter.timer)
|
|
35
|
+
waiter.resolve(item)
|
|
36
|
+
} else {
|
|
37
|
+
this.items.push(item)
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
dequeue(fallback?: T, timeoutMs = 10000): Promise<T> {
|
|
42
|
+
if (this.items.length > 0) {
|
|
43
|
+
return Promise.resolve(this.items.shift()!)
|
|
44
|
+
}
|
|
45
|
+
return new Promise<T>((resolve) => {
|
|
46
|
+
const waiter: { resolve: (v: T) => void; timer?: ReturnType<typeof setTimeout> } = {
|
|
47
|
+
resolve,
|
|
48
|
+
}
|
|
49
|
+
if (fallback !== undefined) {
|
|
50
|
+
waiter.timer = setTimeout(() => {
|
|
51
|
+
const idx = this.waiters.indexOf(waiter)
|
|
52
|
+
if (idx >= 0) this.waiters.splice(idx, 1)
|
|
53
|
+
resolve(fallback)
|
|
54
|
+
}, timeoutMs)
|
|
55
|
+
}
|
|
56
|
+
this.waiters.push(waiter)
|
|
57
|
+
})
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function generateFallbackDump(): string {
|
|
62
|
+
return [
|
|
63
|
+
'SET statement_timeout = 0;',
|
|
64
|
+
"SET client_encoding = 'UTF8';",
|
|
65
|
+
'SET standard_conforming_strings = on;',
|
|
66
|
+
'',
|
|
67
|
+
'CREATE TABLE IF NOT EXISTS restore_seed (',
|
|
68
|
+
' id integer PRIMARY KEY,',
|
|
69
|
+
' note text NOT NULL',
|
|
70
|
+
');',
|
|
71
|
+
'',
|
|
72
|
+
"INSERT INTO restore_seed (id, note) VALUES (1, 'seeded by fallback dump');",
|
|
73
|
+
'',
|
|
74
|
+
].join('\n')
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function resolveDumpFile(): { path: string; cleanup: boolean } {
|
|
78
|
+
const envDump = process.env.OREZ_RESTORE_SQL_DUMP
|
|
79
|
+
if (envDump && existsSync(envDump)) {
|
|
80
|
+
return { path: envDump, cleanup: false }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const chatCandidates = [
|
|
84
|
+
join(homedir(), 'chat', 'tmp', 'restore.sql'),
|
|
85
|
+
join(homedir(), 'chat', 'tmp', 'backup.sql'),
|
|
86
|
+
join(homedir(), 'chat', 'restore.sql'),
|
|
87
|
+
join(homedir(), 'chat', 'backup.sql'),
|
|
88
|
+
]
|
|
89
|
+
for (const candidate of chatCandidates) {
|
|
90
|
+
if (existsSync(candidate)) {
|
|
91
|
+
return { path: candidate, cleanup: false }
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const tmpDump = join(tmpdir(), `orez-restore-reset-${Date.now()}.sql`)
|
|
96
|
+
writeFileSync(tmpDump, generateFallbackDump())
|
|
97
|
+
return { path: tmpDump, cleanup: true }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
describe('restore/reset integration regression', { timeout: 150_000 }, () => {
|
|
101
|
+
let db: PGlite
|
|
102
|
+
let pgPort: number
|
|
103
|
+
let zeroPort: number
|
|
104
|
+
let shutdown: () => Promise<void>
|
|
105
|
+
let dataDir: string
|
|
106
|
+
let dumpFile: string
|
|
107
|
+
let dumpFileIsTemp = false
|
|
108
|
+
|
|
109
|
+
beforeAll(async () => {
|
|
110
|
+
await loadModule()
|
|
111
|
+
|
|
112
|
+
const dump = resolveDumpFile()
|
|
113
|
+
dumpFile = dump.path
|
|
114
|
+
dumpFileIsTemp = dump.cleanup
|
|
115
|
+
|
|
116
|
+
dataDir = `.orez-restore-reset-test-${Date.now()}`
|
|
117
|
+
|
|
118
|
+
const started = await startZeroLite({
|
|
119
|
+
pgPort: 27000 + Math.floor(Math.random() * 1000),
|
|
120
|
+
zeroPort: 28000 + Math.floor(Math.random() * 1000),
|
|
121
|
+
dataDir,
|
|
122
|
+
logLevel: 'warn',
|
|
123
|
+
skipZeroCache: false,
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
db = started.db
|
|
127
|
+
pgPort = started.pgPort
|
|
128
|
+
zeroPort = started.zeroPort
|
|
129
|
+
shutdown = started.stop
|
|
130
|
+
|
|
131
|
+
await waitForZero(zeroPort, 90_000)
|
|
132
|
+
}, 120_000)
|
|
133
|
+
|
|
134
|
+
afterAll(async () => {
|
|
135
|
+
if (shutdown) await shutdown()
|
|
136
|
+
if (dataDir) {
|
|
137
|
+
try {
|
|
138
|
+
rmSync(dataDir, { recursive: true, force: true })
|
|
139
|
+
} catch {}
|
|
140
|
+
}
|
|
141
|
+
if (dumpFileIsTemp && dumpFile) {
|
|
142
|
+
try {
|
|
143
|
+
unlinkSync(dumpFile)
|
|
144
|
+
} catch {}
|
|
145
|
+
}
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
test('wire restore + pid signal full reset keeps zero-cache healthy', async () => {
|
|
149
|
+
const sql = postgres({
|
|
150
|
+
host: '127.0.0.1',
|
|
151
|
+
port: pgPort,
|
|
152
|
+
user: 'user',
|
|
153
|
+
password: 'password',
|
|
154
|
+
database: 'postgres',
|
|
155
|
+
max: 1,
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
try {
|
|
159
|
+
const wireDb = { exec: (query: string) => sql.unsafe(query) as Promise<unknown> }
|
|
160
|
+
await execDumpFile(wireDb, dumpFile)
|
|
161
|
+
} finally {
|
|
162
|
+
await sql.end({ timeout: 1 }).catch(() => {})
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// mirror pg_restore behavior: read pid file and signal SIGUSR1 for full reset
|
|
166
|
+
const pidFile = join(dataDir, 'orez.pid')
|
|
167
|
+
const pid = Number(readFileSync(pidFile, 'utf-8').trim())
|
|
168
|
+
expect(pid).toBeGreaterThan(0)
|
|
169
|
+
process.kill(pid, 'SIGUSR1')
|
|
170
|
+
|
|
171
|
+
await waitForZero(zeroPort, 90_000)
|
|
172
|
+
|
|
173
|
+
// prove zero-cache is alive after reset and still streams live writes
|
|
174
|
+
await db.exec(`
|
|
175
|
+
CREATE TABLE IF NOT EXISTS reset_probe (
|
|
176
|
+
id text PRIMARY KEY,
|
|
177
|
+
value text NOT NULL
|
|
178
|
+
)
|
|
179
|
+
`)
|
|
180
|
+
|
|
181
|
+
const downstream = new Queue<unknown>()
|
|
182
|
+
const ws = connectAndSubscribe(zeroPort, downstream, {
|
|
183
|
+
table: 'reset_probe',
|
|
184
|
+
orderBy: [['id', 'asc']],
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
try {
|
|
188
|
+
await drainInitialPokes(downstream)
|
|
189
|
+
|
|
190
|
+
await db.query(`INSERT INTO reset_probe (id, value) VALUES ($1, $2)`, [
|
|
191
|
+
`post-reset-${Date.now()}`,
|
|
192
|
+
'ok',
|
|
193
|
+
])
|
|
194
|
+
|
|
195
|
+
const poke = await waitForPokePart(downstream, 30_000)
|
|
196
|
+
expect(poke.rowsPatch).toEqual(
|
|
197
|
+
expect.arrayContaining([
|
|
198
|
+
expect.objectContaining({
|
|
199
|
+
op: 'put',
|
|
200
|
+
tableName: 'reset_probe',
|
|
201
|
+
value: expect.objectContaining({
|
|
202
|
+
value: 'ok',
|
|
203
|
+
}),
|
|
204
|
+
}),
|
|
205
|
+
])
|
|
206
|
+
)
|
|
207
|
+
} finally {
|
|
208
|
+
ws.close()
|
|
209
|
+
}
|
|
210
|
+
})
|
|
211
|
+
})
|
|
212
|
+
|
|
213
|
+
function connectAndSubscribe(
|
|
214
|
+
port: number,
|
|
215
|
+
downstream: Queue<unknown>,
|
|
216
|
+
query: Record<string, unknown>
|
|
217
|
+
): WebSocket {
|
|
218
|
+
const ws = new WebSocket(
|
|
219
|
+
`ws://localhost:${port}/sync/v4/connect` +
|
|
220
|
+
`?clientGroupID=restore-reset-cg-${Date.now()}&clientID=restore-reset-client&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
ws.on('message', (data) => {
|
|
224
|
+
downstream.enqueue(JSON.parse(data.toString()))
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
ws.on('open', () => {
|
|
228
|
+
ws.send(
|
|
229
|
+
JSON.stringify([
|
|
230
|
+
'initConnection',
|
|
231
|
+
{
|
|
232
|
+
desiredQueriesPatch: [{ op: 'put', hash: 'q1', ast: query }],
|
|
233
|
+
},
|
|
234
|
+
])
|
|
235
|
+
)
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
return ws
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async function drainInitialPokes(downstream: Queue<unknown>) {
|
|
242
|
+
let settled = false
|
|
243
|
+
const timeout = Date.now() + 30_000
|
|
244
|
+
|
|
245
|
+
while (!settled && Date.now() < timeout) {
|
|
246
|
+
const msg = (await downstream.dequeue('timeout' as any, 3000)) as any
|
|
247
|
+
if (msg === 'timeout') {
|
|
248
|
+
settled = true
|
|
249
|
+
} else if (Array.isArray(msg) && msg[0] === 'pokeEnd') {
|
|
250
|
+
const next = (await downstream.dequeue('timeout' as any, 2000)) as any
|
|
251
|
+
if (next === 'timeout') {
|
|
252
|
+
settled = true
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async function waitForPokePart(
|
|
259
|
+
downstream: Queue<unknown>,
|
|
260
|
+
timeoutMs = 10_000
|
|
261
|
+
): Promise<Record<string, any>> {
|
|
262
|
+
const deadline = Date.now() + timeoutMs
|
|
263
|
+
while (Date.now() < deadline) {
|
|
264
|
+
const remaining = Math.max(1000, deadline - Date.now())
|
|
265
|
+
const msg = (await downstream.dequeue('timeout' as any, remaining)) as any
|
|
266
|
+
if (msg === 'timeout') throw new Error('timed out waiting for pokePart')
|
|
267
|
+
if (Array.isArray(msg) && msg[0] === 'pokePart' && msg[1]?.rowsPatch) {
|
|
268
|
+
return msg[1]
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
throw new Error('timed out waiting for pokePart')
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
async function waitForZero(port: number, timeoutMs = 30_000) {
|
|
275
|
+
const deadline = Date.now() + timeoutMs
|
|
276
|
+
while (Date.now() < deadline) {
|
|
277
|
+
try {
|
|
278
|
+
const res = await fetch(`http://localhost:${port}/`)
|
|
279
|
+
if (res.ok || res.status === 404) return
|
|
280
|
+
} catch {}
|
|
281
|
+
await new Promise((r) => setTimeout(r, 500))
|
|
282
|
+
}
|
|
283
|
+
throw new Error(`zero-cache not ready on port ${port} after ${timeoutMs}ms`)
|
|
284
|
+
}
|
package/src/log.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { LogStore } from './admin/log-store.js'
|
|
1
2
|
import type { LogLevel } from './config.js'
|
|
2
3
|
|
|
3
4
|
const RESET = '\x1b[0m'
|
|
@@ -20,11 +21,17 @@ const LEVEL_PRIORITY: Record<LogLevel, number> = {
|
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
let currentLevel: LogLevel = 'warn'
|
|
24
|
+
let logStore: LogStore | undefined
|
|
23
25
|
|
|
24
26
|
export function setLogLevel(level: LogLevel) {
|
|
25
27
|
currentLevel = level
|
|
26
28
|
}
|
|
27
29
|
|
|
30
|
+
/** hook up logStore for admin dashboard observability */
|
|
31
|
+
export function setLogStore(store: LogStore | undefined) {
|
|
32
|
+
logStore = store
|
|
33
|
+
}
|
|
34
|
+
|
|
28
35
|
function prefix(label: string, color: string): string {
|
|
29
36
|
return `${BOLD}${color}[${label}]${RESET}`
|
|
30
37
|
}
|
|
@@ -39,12 +46,30 @@ export function url(u: string): string {
|
|
|
39
46
|
return `${COLORS.yellow}${u}${RESET}`
|
|
40
47
|
}
|
|
41
48
|
|
|
49
|
+
// map logger labels to logStore source names
|
|
50
|
+
const LABEL_TO_SOURCE: Record<string, string> = {
|
|
51
|
+
orez: 'orez',
|
|
52
|
+
'orez:pg': 'orez',
|
|
53
|
+
pglite: 'pglite',
|
|
54
|
+
'pg-proxy': 'proxy',
|
|
55
|
+
'orez:zero': 'zero',
|
|
56
|
+
'orez:s3': 's3',
|
|
57
|
+
}
|
|
58
|
+
|
|
42
59
|
function makeLogger(label: string, color: string, level: LogLevel = 'info') {
|
|
43
60
|
const p = prefix(label, color)
|
|
61
|
+
const source = LABEL_TO_SOURCE[label] || 'orez'
|
|
62
|
+
// zero logs are handled specially in startZeroCache with better level detection
|
|
63
|
+
const skipLogStore = source === 'zero'
|
|
44
64
|
return (...args: unknown[]) => {
|
|
45
65
|
if (LEVEL_PRIORITY[level] <= LEVEL_PRIORITY[currentLevel]) {
|
|
46
66
|
console.info(p, ...args)
|
|
47
67
|
}
|
|
68
|
+
// always push to logStore if available (admin captures all levels)
|
|
69
|
+
if (logStore && !skipLogStore) {
|
|
70
|
+
const msg = args.map((a) => (typeof a === 'string' ? a : String(a))).join(' ')
|
|
71
|
+
logStore.push(source, level, msg)
|
|
72
|
+
}
|
|
48
73
|
}
|
|
49
74
|
}
|
|
50
75
|
|
package/src/pglite-manager.ts
CHANGED
|
@@ -150,15 +150,17 @@ export async function createPGliteInstances(
|
|
|
150
150
|
// postgres-specific setup
|
|
151
151
|
await postgres.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
|
|
152
152
|
|
|
153
|
-
// create
|
|
154
|
-
const pubName = process.env.ZERO_APP_PUBLICATIONS
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
153
|
+
// create publication only when explicitly configured
|
|
154
|
+
const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
|
|
155
|
+
if (pubName) {
|
|
156
|
+
const pubs = await postgres.query<{ count: string }>(
|
|
157
|
+
`SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
|
|
158
|
+
[pubName]
|
|
159
|
+
)
|
|
160
|
+
if (Number(pubs.rows[0].count) === 0) {
|
|
161
|
+
const quoted = '"' + pubName.replace(/"/g, '""') + '"'
|
|
162
|
+
await postgres.exec(`CREATE PUBLICATION ${quoted}`)
|
|
163
|
+
}
|
|
162
164
|
}
|
|
163
165
|
|
|
164
166
|
return { postgres, cvr, cdb }
|