orez 0.0.46 → 0.0.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -8
- package/dist/admin/http-proxy.d.ts +31 -0
- package/dist/admin/http-proxy.d.ts.map +1 -0
- package/dist/admin/http-proxy.js +140 -0
- package/dist/admin/http-proxy.js.map +1 -0
- package/dist/admin/log-store.d.ts +22 -0
- package/dist/admin/log-store.d.ts.map +1 -0
- package/dist/admin/log-store.js +86 -0
- package/dist/admin/log-store.js.map +1 -0
- package/dist/admin/server.d.ts +19 -0
- package/dist/admin/server.d.ts.map +1 -0
- package/dist/admin/server.js +110 -0
- package/dist/admin/server.js.map +1 -0
- package/dist/admin/ui.d.ts +2 -0
- package/dist/admin/ui.d.ts.map +1 -0
- package/dist/admin/ui.js +683 -0
- package/dist/admin/ui.js.map +1 -0
- package/dist/cli.js +48 -1
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +4 -0
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +180 -20
- package/dist/index.js.map +1 -1
- package/dist/log.d.ts +9 -0
- package/dist/log.d.ts.map +1 -1
- package/dist/log.js +24 -1
- package/dist/log.js.map +1 -1
- package/dist/pg-proxy.d.ts.map +1 -1
- package/dist/pg-proxy.js +19 -4
- package/dist/pg-proxy.js.map +1 -1
- package/dist/pglite-manager.d.ts +1 -0
- package/dist/pglite-manager.d.ts.map +1 -1
- package/dist/pglite-manager.js +1 -1
- package/dist/pglite-manager.js.map +1 -1
- package/dist/replication/handler.d.ts.map +1 -1
- package/dist/replication/handler.js +20 -2
- package/dist/replication/handler.js.map +1 -1
- package/dist/vite-plugin.d.ts +3 -0
- package/dist/vite-plugin.d.ts.map +1 -1
- package/dist/vite-plugin.js +24 -0
- package/dist/vite-plugin.js.map +1 -1
- package/package.json +4 -2
- package/src/admin/http-proxy.ts +186 -0
- package/src/admin/log-store.ts +111 -0
- package/src/admin/server.ts +148 -0
- package/src/admin/ui.ts +682 -0
- package/src/cli.ts +49 -1
- package/src/config.ts +8 -0
- package/src/index.ts +192 -20
- package/src/log.ts +25 -1
- package/src/pg-proxy.ts +26 -6
- package/src/pglite-manager.ts +1 -1
- package/src/replication/handler.ts +21 -2
- package/src/vite-plugin.ts +28 -0
package/src/cli.ts
CHANGED
|
@@ -750,6 +750,11 @@ const main = defineCommand({
|
|
|
750
750
|
description: 'use native @rocicorp/zero-sqlite3 instead of wasm bedrock-sqlite',
|
|
751
751
|
default: false,
|
|
752
752
|
},
|
|
753
|
+
'log-env': {
|
|
754
|
+
type: 'boolean',
|
|
755
|
+
description: 'log ZERO_* and related environment variables on startup',
|
|
756
|
+
default: false,
|
|
757
|
+
},
|
|
753
758
|
'on-db-ready': {
|
|
754
759
|
type: 'string',
|
|
755
760
|
description: 'command to run after db+proxy are ready, before zero-cache starts',
|
|
@@ -760,6 +765,21 @@ const main = defineCommand({
|
|
|
760
765
|
description: 'command to run once all services are healthy',
|
|
761
766
|
default: '',
|
|
762
767
|
},
|
|
768
|
+
admin: {
|
|
769
|
+
type: 'boolean',
|
|
770
|
+
description: 'start admin web ui',
|
|
771
|
+
default: false,
|
|
772
|
+
},
|
|
773
|
+
'admin-port': {
|
|
774
|
+
type: 'string',
|
|
775
|
+
description: 'admin ui port (default: auto)',
|
|
776
|
+
default: '0',
|
|
777
|
+
},
|
|
778
|
+
'admin-logs': {
|
|
779
|
+
type: 'boolean',
|
|
780
|
+
description: 'write logs to .orez/logs/ (default: true when --admin)',
|
|
781
|
+
default: true,
|
|
782
|
+
},
|
|
763
783
|
},
|
|
764
784
|
subCommands: {
|
|
765
785
|
s3: s3Command,
|
|
@@ -767,7 +787,8 @@ const main = defineCommand({
|
|
|
767
787
|
pg_restore: pgRestoreCommand,
|
|
768
788
|
},
|
|
769
789
|
async run({ args }) {
|
|
770
|
-
const
|
|
790
|
+
const startTime = Date.now()
|
|
791
|
+
const { config, stop, logStore, zeroEnv, actions, httpLogStore } = await startZeroLite({
|
|
771
792
|
pgPort: Number(args['pg-port']),
|
|
772
793
|
zeroPort: Number(args['zero-port']),
|
|
773
794
|
dataDir: args['data-dir'],
|
|
@@ -778,7 +799,11 @@ const main = defineCommand({
|
|
|
778
799
|
skipZeroCache: args['skip-zero-cache'],
|
|
779
800
|
disableWasmSqlite: args['disable-wasm-sqlite'],
|
|
780
801
|
logLevel: (args['log-level'] as 'error' | 'warn' | 'info' | 'debug') || undefined,
|
|
802
|
+
logEnv: args['log-env'],
|
|
781
803
|
onDbReady: args['on-db-ready'],
|
|
804
|
+
admin: args.admin,
|
|
805
|
+
adminPort: Number(args['admin-port']),
|
|
806
|
+
adminLogs: args['admin-logs'],
|
|
782
807
|
})
|
|
783
808
|
|
|
784
809
|
let s3Server: import('node:http').Server | null = null
|
|
@@ -790,6 +815,27 @@ const main = defineCommand({
|
|
|
790
815
|
})
|
|
791
816
|
}
|
|
792
817
|
|
|
818
|
+
let adminServer: import('node:http').Server | null = null
|
|
819
|
+
if (args.admin && logStore) {
|
|
820
|
+
const { findPort } = await import('./port.js')
|
|
821
|
+
const adminPort = Number(args['admin-port']) || (config.zeroPort + 2)
|
|
822
|
+
const resolvedPort = await findPort(adminPort)
|
|
823
|
+
const { startAdminServer } = await import('./admin/server.js')
|
|
824
|
+
adminServer = await startAdminServer({
|
|
825
|
+
port: resolvedPort,
|
|
826
|
+
logStore,
|
|
827
|
+
config,
|
|
828
|
+
zeroEnv,
|
|
829
|
+
actions,
|
|
830
|
+
startTime,
|
|
831
|
+
httpLog: httpLogStore || undefined,
|
|
832
|
+
})
|
|
833
|
+
log.orez(`admin: http://127.0.0.1:${resolvedPort}`)
|
|
834
|
+
if (args['admin-logs']) {
|
|
835
|
+
log.orez(`logs: ${resolve(args['data-dir'], 'logs', 'orez.log')}`)
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
793
839
|
log.orez('ready')
|
|
794
840
|
log.orez(
|
|
795
841
|
`pg: postgresql://${config.pgUser}:${config.pgPassword}@127.0.0.1:${config.pgPort}/postgres`
|
|
@@ -817,11 +863,13 @@ const main = defineCommand({
|
|
|
817
863
|
}
|
|
818
864
|
|
|
819
865
|
process.on('SIGINT', async () => {
|
|
866
|
+
adminServer?.close()
|
|
820
867
|
s3Server?.close()
|
|
821
868
|
await stop()
|
|
822
869
|
process.exit(0)
|
|
823
870
|
})
|
|
824
871
|
process.on('SIGTERM', async () => {
|
|
872
|
+
adminServer?.close()
|
|
825
873
|
s3Server?.close()
|
|
826
874
|
await stop()
|
|
827
875
|
process.exit(0)
|
package/src/config.ts
CHANGED
|
@@ -14,8 +14,12 @@ export interface ZeroLiteConfig {
|
|
|
14
14
|
disableWasmSqlite: boolean
|
|
15
15
|
logLevel: LogLevel
|
|
16
16
|
pgliteOptions: Partial<PGliteOptions>
|
|
17
|
+
logEnv: boolean
|
|
17
18
|
onDbReady: string
|
|
18
19
|
beforeZero: ((db: import('@electric-sql/pglite').PGlite) => Promise<void>) | null
|
|
20
|
+
admin: boolean
|
|
21
|
+
adminPort: number
|
|
22
|
+
adminLogs: boolean
|
|
19
23
|
}
|
|
20
24
|
|
|
21
25
|
export function getConfig(overrides: Partial<ZeroLiteConfig> = {}): ZeroLiteConfig {
|
|
@@ -31,8 +35,12 @@ export function getConfig(overrides: Partial<ZeroLiteConfig> = {}): ZeroLiteConf
|
|
|
31
35
|
disableWasmSqlite: overrides.disableWasmSqlite ?? false,
|
|
32
36
|
logLevel: overrides.logLevel || 'warn',
|
|
33
37
|
pgliteOptions: overrides.pgliteOptions || {},
|
|
38
|
+
logEnv: overrides.logEnv ?? false,
|
|
34
39
|
onDbReady: overrides.onDbReady || '',
|
|
35
40
|
beforeZero: overrides.beforeZero || null,
|
|
41
|
+
admin: overrides.admin ?? false,
|
|
42
|
+
adminPort: overrides.adminPort || 0,
|
|
43
|
+
adminLogs: overrides.adminLogs ?? true,
|
|
36
44
|
}
|
|
37
45
|
}
|
|
38
46
|
|
package/src/index.ts
CHANGED
|
@@ -7,21 +7,23 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { spawn, type ChildProcess } from 'node:child_process'
|
|
10
|
-
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
10
|
+
import { existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
11
11
|
import { createRequire } from 'node:module'
|
|
12
12
|
import { totalmem } from 'node:os'
|
|
13
13
|
import { dirname, resolve } from 'node:path'
|
|
14
14
|
import { fileURLToPath } from 'node:url'
|
|
15
15
|
|
|
16
16
|
import { getConfig, getConnectionString } from './config.js'
|
|
17
|
-
import { log, port, setLogLevel } from './log.js'
|
|
17
|
+
import { log, port, setLogLevel, addLogListener } from './log.js'
|
|
18
18
|
import { startPgProxy } from './pg-proxy.js'
|
|
19
|
-
import { createPGliteInstances, runMigrations } from './pglite-manager.js'
|
|
19
|
+
import { createInstance, createPGliteInstances, runMigrations } from './pglite-manager.js'
|
|
20
20
|
import { findPort } from './port.js'
|
|
21
21
|
import { installChangeTracking } from './replication/change-tracker.js'
|
|
22
22
|
|
|
23
23
|
import type { ZeroLiteConfig } from './config.js'
|
|
24
24
|
import type { PGlite } from '@electric-sql/pglite'
|
|
25
|
+
import type { LogStore } from './admin/log-store.js'
|
|
26
|
+
import type { HttpLogStore } from './admin/http-proxy.js'
|
|
25
27
|
|
|
26
28
|
export { getConfig, getConnectionString } from './config.js'
|
|
27
29
|
export type { LogLevel, ZeroLiteConfig } from './config.js'
|
|
@@ -43,6 +45,21 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
43
45
|
const config = getConfig(overrides)
|
|
44
46
|
setLogLevel(config.logLevel)
|
|
45
47
|
|
|
48
|
+
// when admin ui enabled, create log store and capture all log output
|
|
49
|
+
const SOURCE_MAP: Record<string, string> = {
|
|
50
|
+
'orez': 'orez', 'pglite': 'pglite', 'pg-proxy': 'proxy',
|
|
51
|
+
'zero': 'zero', 'zero-cache': 'zero', 'orez/s3': 's3',
|
|
52
|
+
}
|
|
53
|
+
let logStore: LogStore | null = null
|
|
54
|
+
let removeLogListener: (() => void) | null = null
|
|
55
|
+
if (config.admin) {
|
|
56
|
+
const { createLogStore } = await import('./admin/log-store.js')
|
|
57
|
+
logStore = createLogStore(config.dataDir, config.adminLogs)
|
|
58
|
+
removeLogListener = addLogListener((source, level, msg) => {
|
|
59
|
+
logStore!.push(SOURCE_MAP[source] || source, level, msg)
|
|
60
|
+
})
|
|
61
|
+
}
|
|
62
|
+
|
|
46
63
|
// find available ports
|
|
47
64
|
const pgPort = await findPort(config.pgPort)
|
|
48
65
|
const zeroPort = config.skipZeroCache
|
|
@@ -73,7 +90,7 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
73
90
|
// start tcp proxy (routes connections to correct instance by database name)
|
|
74
91
|
const pgServer = await startPgProxy(instances, config)
|
|
75
92
|
|
|
76
|
-
log.
|
|
93
|
+
log.pglite(`postgres up ${port(pgPort, 'green')}`)
|
|
77
94
|
if (migrationsApplied > 0)
|
|
78
95
|
log.orez(
|
|
79
96
|
`${migrationsApplied} migration${migrationsApplied === 1 ? '' : 's'} applied`
|
|
@@ -128,18 +145,121 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
128
145
|
// clean up stale lock files from previous crash (keep replica for fast restart)
|
|
129
146
|
cleanupStaleLockFiles(config)
|
|
130
147
|
|
|
131
|
-
//
|
|
148
|
+
// http proxy for admin traffic logging
|
|
149
|
+
let httpLogStore: HttpLogStore | null = null
|
|
150
|
+
let httpProxyServer: import('node:http').Server | null = null
|
|
151
|
+
let zeroInternalPort = zeroPort
|
|
152
|
+
if (config.admin && !config.skipZeroCache) {
|
|
153
|
+
const { createHttpLogStore } = await import('./admin/http-proxy.js')
|
|
154
|
+
httpLogStore = createHttpLogStore()
|
|
155
|
+
zeroInternalPort = await findPort(zeroPort + 100)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// start zero-cache with auto-recovery for stale change db
|
|
132
159
|
let zeroCacheProcess: ChildProcess | null = null
|
|
160
|
+
let zeroEnv: Record<string, string> = {}
|
|
161
|
+
const cdbResets = { count: 0, lastReset: 0 }
|
|
162
|
+
const MAX_CDB_RESETS = 10
|
|
163
|
+
const MIN_RESET_INTERVAL_MS = 60_000
|
|
164
|
+
|
|
133
165
|
if (!config.skipZeroCache) {
|
|
134
|
-
|
|
135
|
-
|
|
166
|
+
let currentResult = await startZeroCache(config, zeroInternalPort)
|
|
167
|
+
zeroCacheProcess = currentResult.child
|
|
168
|
+
zeroEnv = currentResult.env
|
|
169
|
+
|
|
170
|
+
// watch for stale changeLog crashes and auto-recover
|
|
171
|
+
const attachCdbRecovery = (result: typeof currentResult) => {
|
|
172
|
+
result.child.on('exit', async (code) => {
|
|
173
|
+
if (code === 0 || code === null) return
|
|
174
|
+
if (!result.stderrBuf.includes('changeLog_pkey')) return
|
|
175
|
+
|
|
176
|
+
const now = Date.now()
|
|
177
|
+
if (cdbResets.count >= MAX_CDB_RESETS) {
|
|
178
|
+
log.zero('change db reset limit reached, not retrying')
|
|
179
|
+
return
|
|
180
|
+
}
|
|
181
|
+
const elapsed = now - cdbResets.lastReset
|
|
182
|
+
if (elapsed < MIN_RESET_INTERVAL_MS) {
|
|
183
|
+
log.zero(`change db reset too soon (${Math.round(elapsed / 1000)}s ago), not retrying`)
|
|
184
|
+
return
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
cdbResets.count++
|
|
188
|
+
cdbResets.lastReset = now
|
|
189
|
+
log.zero(`stale change db detected, resetting (${cdbResets.count}/${MAX_CDB_RESETS})`)
|
|
190
|
+
|
|
191
|
+
try {
|
|
192
|
+
await instances.cdb.close()
|
|
193
|
+
const cdbPath = resolve(config.dataDir, 'pgdata-cdb')
|
|
194
|
+
rmSync(cdbPath, { recursive: true, force: true })
|
|
195
|
+
instances.cdb = await createInstance(config, 'cdb', false)
|
|
196
|
+
|
|
197
|
+
currentResult = await startZeroCache(config, zeroInternalPort)
|
|
198
|
+
zeroCacheProcess = currentResult.child
|
|
199
|
+
attachCdbRecovery(currentResult)
|
|
200
|
+
await waitForZeroCache(config, undefined, zeroInternalPort)
|
|
201
|
+
log.zero(`recovered, ready ${port(config.zeroPort, 'magenta')}`)
|
|
202
|
+
} catch (err) {
|
|
203
|
+
log.zero(`recovery failed: ${err}`)
|
|
204
|
+
}
|
|
205
|
+
})
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
attachCdbRecovery(currentResult)
|
|
209
|
+
await waitForZeroCache(config, undefined, zeroInternalPort)
|
|
136
210
|
log.zero(`ready ${port(config.zeroPort, 'magenta')}`)
|
|
211
|
+
|
|
212
|
+
// start http proxy for admin traffic logging
|
|
213
|
+
if (httpLogStore) {
|
|
214
|
+
const { startHttpProxy } = await import('./admin/http-proxy.js')
|
|
215
|
+
httpProxyServer = await startHttpProxy({
|
|
216
|
+
listenPort: zeroPort,
|
|
217
|
+
targetPort: zeroInternalPort,
|
|
218
|
+
httpLog: httpLogStore,
|
|
219
|
+
})
|
|
220
|
+
}
|
|
137
221
|
} else {
|
|
138
222
|
log.orez('skip zero-cache')
|
|
139
223
|
}
|
|
140
224
|
|
|
225
|
+
// admin action handlers
|
|
226
|
+
const actions = {
|
|
227
|
+
restartZero: config.skipZeroCache ? undefined : async () => {
|
|
228
|
+
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
229
|
+
zeroCacheProcess.kill('SIGTERM')
|
|
230
|
+
await new Promise<void>((r) => {
|
|
231
|
+
const t = setTimeout(() => { zeroCacheProcess?.kill('SIGKILL'); r() }, 3000)
|
|
232
|
+
zeroCacheProcess!.on('exit', () => { clearTimeout(t); r() })
|
|
233
|
+
})
|
|
234
|
+
}
|
|
235
|
+
const zc = await startZeroCache(config, zeroInternalPort)
|
|
236
|
+
zeroCacheProcess = zc.child
|
|
237
|
+
await waitForZeroCache(config, undefined, zeroInternalPort)
|
|
238
|
+
log.zero(`restarted ${port(config.zeroPort, 'magenta')}`)
|
|
239
|
+
},
|
|
240
|
+
resetZero: config.skipZeroCache ? undefined : async () => {
|
|
241
|
+
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
242
|
+
zeroCacheProcess.kill('SIGTERM')
|
|
243
|
+
await new Promise<void>((r) => {
|
|
244
|
+
const t = setTimeout(() => { zeroCacheProcess?.kill('SIGKILL'); r() }, 3000)
|
|
245
|
+
zeroCacheProcess!.on('exit', () => { clearTimeout(t); r() })
|
|
246
|
+
})
|
|
247
|
+
}
|
|
248
|
+
const replicaPath = resolve(config.dataDir, 'zero-replica.db')
|
|
249
|
+
for (const suffix of ['', '-wal', '-shm', '-wal2']) {
|
|
250
|
+
try { if (existsSync(replicaPath + suffix)) unlinkSync(replicaPath + suffix) } catch {}
|
|
251
|
+
}
|
|
252
|
+
const zc = await startZeroCache(config, zeroInternalPort)
|
|
253
|
+
zeroCacheProcess = zc.child
|
|
254
|
+
await waitForZeroCache(config, undefined, zeroInternalPort)
|
|
255
|
+
log.zero(`reset and restarted ${port(config.zeroPort, 'magenta')}`)
|
|
256
|
+
},
|
|
257
|
+
}
|
|
258
|
+
|
|
141
259
|
const stop = async () => {
|
|
142
260
|
log.debug.orez('shutting down')
|
|
261
|
+
removeLogListener?.()
|
|
262
|
+
httpProxyServer?.close()
|
|
143
263
|
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
144
264
|
zeroCacheProcess.kill('SIGTERM')
|
|
145
265
|
// wait up to 3s for graceful exit, then force kill
|
|
@@ -165,7 +285,7 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
165
285
|
log.debug.orez('stopped')
|
|
166
286
|
}
|
|
167
287
|
|
|
168
|
-
return { config, stop, db, instances, pgPort: config.pgPort, zeroPort: config.zeroPort }
|
|
288
|
+
return { config, stop, db, instances, pgPort: config.pgPort, zeroPort: config.zeroPort, logStore, zeroEnv, actions, httpLogStore }
|
|
169
289
|
}
|
|
170
290
|
|
|
171
291
|
function cleanupStaleLockFiles(config: ZeroLiteConfig): void {
|
|
@@ -244,7 +364,7 @@ function writeSqliteShim(): string {
|
|
|
244
364
|
return registerPath
|
|
245
365
|
}
|
|
246
366
|
|
|
247
|
-
async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
367
|
+
async function startZeroCache(config: ZeroLiteConfig, portOverride?: number): Promise<{ child: ChildProcess; env: Record<string, string>; stderrBuf: string }> {
|
|
248
368
|
// resolve @rocicorp/zero entry for finding zero-cache modules
|
|
249
369
|
const zeroEntry = resolvePackage('@rocicorp/zero')
|
|
250
370
|
|
|
@@ -263,7 +383,7 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
263
383
|
// defaults that can be overridden by user env
|
|
264
384
|
const defaults: Record<string, string> = {
|
|
265
385
|
NODE_ENV: 'development',
|
|
266
|
-
ZERO_LOG_LEVEL:
|
|
386
|
+
ZERO_LOG_LEVEL: 'info',
|
|
267
387
|
ZERO_NUM_SYNC_WORKERS: '1',
|
|
268
388
|
// disable query planner — it relies on scanStatus which causes infinite
|
|
269
389
|
// loops with wasm sqlite and has caused freezes with native too.
|
|
@@ -289,7 +409,7 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
289
409
|
ZERO_CVR_DB: cvrUrl,
|
|
290
410
|
ZERO_CHANGE_DB: cdbUrl,
|
|
291
411
|
ZERO_REPLICA_FILE: resolve(config.dataDir, 'zero-replica.db'),
|
|
292
|
-
ZERO_PORT: String(config.zeroPort),
|
|
412
|
+
ZERO_PORT: String(portOverride || config.zeroPort),
|
|
293
413
|
}
|
|
294
414
|
|
|
295
415
|
const zeroCacheBin = resolve(zeroEntry, '..', 'cli.js')
|
|
@@ -313,31 +433,82 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
313
433
|
env.NODE_OPTIONS = `--max-old-space-size=${heapMB} ${existing}`.trim()
|
|
314
434
|
}
|
|
315
435
|
|
|
436
|
+
// log env vars if --log-env was passed
|
|
437
|
+
if (config.logEnv) {
|
|
438
|
+
const zeroVars = Object.entries(env)
|
|
439
|
+
.filter(([key]) => key.startsWith('ZERO_') || key === 'NODE_ENV')
|
|
440
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
441
|
+
log.orez('zero-cache env:')
|
|
442
|
+
for (const [key, value] of zeroVars) {
|
|
443
|
+
log.orez(` ${key}=${value}`)
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
|
|
316
447
|
const child = spawn(zeroCacheBin, [], {
|
|
317
448
|
env,
|
|
318
449
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
319
450
|
})
|
|
320
451
|
|
|
452
|
+
// zero-cache uses structured logging when piped (not a tty).
|
|
453
|
+
// multiline format: timestamp + "[" on one line, context lines, "] message" on another.
|
|
454
|
+
// single-line format: timestamp + [ context ] message, or timestamp + key=val,... message
|
|
455
|
+
// we buffer multiline blocks and extract just the message.
|
|
456
|
+
const timestampRe = /^\d{4}-\d{2}-\d{2}T[\d:.+\-Z]+\s*/
|
|
457
|
+
let inBlock = false
|
|
458
|
+
const zeroLog = (line: string) => {
|
|
459
|
+
let stripped = line.replace(timestampRe, '')
|
|
460
|
+
|
|
461
|
+
// start of multiline context block: line ends with "[" (possibly after timestamp)
|
|
462
|
+
if (!inBlock && /^\[?\s*$/.test(stripped)) {
|
|
463
|
+
inBlock = true
|
|
464
|
+
return
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
// inside multiline block: skip context lines, look for "] message"
|
|
468
|
+
if (inBlock) {
|
|
469
|
+
const closeMatch = stripped.match(/^\]\s*(.*)$/)
|
|
470
|
+
if (closeMatch) {
|
|
471
|
+
inBlock = false
|
|
472
|
+
const msg = closeMatch[1].trim()
|
|
473
|
+
if (msg) log.zero(msg)
|
|
474
|
+
}
|
|
475
|
+
// context continuation lines like "'pid=8278'," — skip
|
|
476
|
+
return
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// single-line: strip inline [ context ] and key=val prefixes
|
|
480
|
+
stripped = stripped.replace(/\[.*?\]\s*/g, '')
|
|
481
|
+
stripped = stripped.replace(/^(?:\w+=\S+,)*\w+=\S+\s+/, '')
|
|
482
|
+
stripped = stripped.trim()
|
|
483
|
+
|
|
484
|
+
if (!stripped || /^[\[\]',\s]*$/.test(stripped)) return
|
|
485
|
+
|
|
486
|
+
log.zero(stripped)
|
|
487
|
+
}
|
|
488
|
+
|
|
321
489
|
child.stdout?.on('data', (data: Buffer) => {
|
|
322
490
|
const lines = data.toString().trim().split('\n')
|
|
323
491
|
for (const line of lines) {
|
|
324
|
-
|
|
492
|
+
zeroLog(line)
|
|
325
493
|
}
|
|
326
494
|
})
|
|
327
495
|
|
|
328
|
-
|
|
496
|
+
const result = { child, env, stderrBuf: '' }
|
|
497
|
+
|
|
329
498
|
child.stderr?.on('data', (data: Buffer) => {
|
|
330
499
|
const chunk = data.toString()
|
|
331
|
-
stderrBuf += chunk
|
|
500
|
+
result.stderrBuf += chunk
|
|
332
501
|
const lines = chunk.trim().split('\n')
|
|
333
502
|
for (const line of lines) {
|
|
334
|
-
|
|
503
|
+
zeroLog(line)
|
|
335
504
|
}
|
|
336
505
|
})
|
|
337
506
|
|
|
338
507
|
child.on('exit', (code) => {
|
|
339
508
|
if (code !== 0 && code !== null) {
|
|
340
|
-
|
|
509
|
+
// changeLog_pkey errors are handled by the recovery logic in startZeroLite
|
|
510
|
+
if (result.stderrBuf.includes('changeLog_pkey')) return
|
|
511
|
+
if (result.stderrBuf.includes('Could not locate the bindings file')) {
|
|
341
512
|
log.zero(
|
|
342
513
|
'native @rocicorp/zero-sqlite3 not found — native deps were not compiled.\n' +
|
|
343
514
|
'either:\n' +
|
|
@@ -346,7 +517,7 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
346
517
|
' or add "trustedDependencies": ["@rocicorp/zero-sqlite3"] to package.json'
|
|
347
518
|
)
|
|
348
519
|
} else {
|
|
349
|
-
const lastLines = stderrBuf.trim().split('\n').slice(-5).join('\n')
|
|
520
|
+
const lastLines = result.stderrBuf.trim().split('\n').slice(-5).join('\n')
|
|
350
521
|
if (lastLines) {
|
|
351
522
|
log.zero(`exited with code ${code}:\n${lastLines}`)
|
|
352
523
|
} else {
|
|
@@ -356,15 +527,16 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
356
527
|
}
|
|
357
528
|
})
|
|
358
529
|
|
|
359
|
-
return
|
|
530
|
+
return result
|
|
360
531
|
}
|
|
361
532
|
|
|
362
533
|
async function waitForZeroCache(
|
|
363
534
|
config: ZeroLiteConfig,
|
|
364
|
-
timeoutMs = 120000
|
|
535
|
+
timeoutMs = 120000,
|
|
536
|
+
portOverride?: number,
|
|
365
537
|
): Promise<void> {
|
|
366
538
|
const start = Date.now()
|
|
367
|
-
const url = `http://127.0.0.1:${config.zeroPort}/`
|
|
539
|
+
const url = `http://127.0.0.1:${portOverride || config.zeroPort}/`
|
|
368
540
|
|
|
369
541
|
while (Date.now() - start < timeoutMs) {
|
|
370
542
|
try {
|
package/src/log.ts
CHANGED
|
@@ -25,6 +25,17 @@ export function setLogLevel(level: LogLevel) {
|
|
|
25
25
|
currentLevel = level
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
type LogListener = (source: string, level: LogLevel, msg: string) => void
|
|
29
|
+
const listeners: LogListener[] = []
|
|
30
|
+
|
|
31
|
+
export function addLogListener(fn: LogListener) {
|
|
32
|
+
listeners.push(fn)
|
|
33
|
+
return () => {
|
|
34
|
+
const idx = listeners.indexOf(fn)
|
|
35
|
+
if (idx !== -1) listeners.splice(idx, 1)
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
28
39
|
function prefix(label: string, color: string): string {
|
|
29
40
|
return `${BOLD}${color}[${label}]${RESET}`
|
|
30
41
|
}
|
|
@@ -37,6 +48,12 @@ export function port(n: number, color: keyof typeof COLORS): string {
|
|
|
37
48
|
function makeLogger(label: string, color: string, level: LogLevel = 'info') {
|
|
38
49
|
const p = prefix(label, color)
|
|
39
50
|
return (...args: unknown[]) => {
|
|
51
|
+
// always notify listeners (they capture everything for admin ui)
|
|
52
|
+
if (listeners.length > 0) {
|
|
53
|
+
const msg = args.map((a) => (typeof a === 'string' ? a : String(a))).join(' ')
|
|
54
|
+
for (const fn of listeners) fn(label, level, msg)
|
|
55
|
+
}
|
|
56
|
+
// only print to terminal if level passes filter
|
|
40
57
|
if (LEVEL_PRIORITY[level] <= LEVEL_PRIORITY[currentLevel]) {
|
|
41
58
|
console.info(p, ...args)
|
|
42
59
|
}
|
|
@@ -47,8 +64,15 @@ export const log = {
|
|
|
47
64
|
orez: makeLogger('orez', COLORS.cyan, 'warn'),
|
|
48
65
|
pglite: makeLogger('pglite', COLORS.green, 'warn'),
|
|
49
66
|
proxy: makeLogger('pg-proxy', COLORS.yellow, 'warn'),
|
|
50
|
-
zero: makeLogger('zero
|
|
67
|
+
zero: makeLogger('zero', COLORS.magenta, 'warn'),
|
|
51
68
|
s3: makeLogger('orez/s3', COLORS.blue, 'warn'),
|
|
69
|
+
info: {
|
|
70
|
+
orez: makeLogger('orez', COLORS.cyan, 'info'),
|
|
71
|
+
pglite: makeLogger('pglite', COLORS.green, 'info'),
|
|
72
|
+
proxy: makeLogger('pg-proxy', COLORS.yellow, 'info'),
|
|
73
|
+
zero: makeLogger('zero', COLORS.magenta, 'info'),
|
|
74
|
+
s3: makeLogger('orez/s3', COLORS.blue, 'info'),
|
|
75
|
+
},
|
|
52
76
|
debug: {
|
|
53
77
|
orez: makeLogger('orez', COLORS.cyan, 'debug'),
|
|
54
78
|
pglite: makeLogger('pglite', COLORS.green, 'debug'),
|
package/src/pg-proxy.ts
CHANGED
|
@@ -464,6 +464,14 @@ async function performHandshake(
|
|
|
464
464
|
return { params }
|
|
465
465
|
}
|
|
466
466
|
|
|
467
|
+
// ── connection tracking ──
|
|
468
|
+
|
|
469
|
+
// per-database active connection count. pglite is single-session so all
|
|
470
|
+
// connections share one transaction context. we skip ROLLBACK on close when
|
|
471
|
+
// other connections are still active to avoid killing their transactions.
|
|
472
|
+
const activeConns: Record<string, number> = {}
|
|
473
|
+
let connCounter = 0
|
|
474
|
+
|
|
467
475
|
// ── message loop ──
|
|
468
476
|
|
|
469
477
|
// process messages from a connected, authenticated client.
|
|
@@ -674,6 +682,7 @@ export async function startPgProxy(
|
|
|
674
682
|
|
|
675
683
|
let dbName = 'postgres'
|
|
676
684
|
let isReplicationConnection = false
|
|
685
|
+
const connId = ++connCounter
|
|
677
686
|
|
|
678
687
|
try {
|
|
679
688
|
// perform startup handshake
|
|
@@ -682,18 +691,29 @@ export async function startPgProxy(
|
|
|
682
691
|
dbName = params.database || 'postgres'
|
|
683
692
|
isReplicationConnection = params.replication === 'database'
|
|
684
693
|
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
694
|
+
// track active connections per database
|
|
695
|
+
activeConns[dbName] = (activeConns[dbName] || 0) + 1
|
|
696
|
+
|
|
697
|
+
console.info(`[orez-proxy#${connId}] connect db=${dbName} repl=${params.replication || 'none'}`)
|
|
688
698
|
|
|
689
699
|
const { db } = getDbContext(dbName)
|
|
690
700
|
await db.waitReady
|
|
691
701
|
|
|
692
702
|
// clean up pglite session state when client disconnects.
|
|
693
|
-
// pglite is single-session — all connections share one session
|
|
694
|
-
//
|
|
695
|
-
//
|
|
703
|
+
// pglite is single-session — all connections share one session.
|
|
704
|
+
// only ROLLBACK + reset when this is the LAST connection for this db,
|
|
705
|
+
// to avoid killing another connection's active transaction.
|
|
696
706
|
socket.on('close', async () => {
|
|
707
|
+
activeConns[dbName] = Math.max(0, (activeConns[dbName] || 1) - 1)
|
|
708
|
+
const remaining = activeConns[dbName]
|
|
709
|
+
const shouldRollback = remaining === 0
|
|
710
|
+
|
|
711
|
+
console.info(
|
|
712
|
+
`[orez-proxy#${connId}] close [${dbName}] (remaining=${remaining}, shouldRollback=${shouldRollback})`
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
if (!shouldRollback) return
|
|
716
|
+
|
|
697
717
|
const { db: closeDb, mutex: closeMutex } = getDbContext(dbName)
|
|
698
718
|
await closeMutex.acquire()
|
|
699
719
|
try {
|
package/src/pglite-manager.ts
CHANGED
|
@@ -33,6 +33,9 @@ import {
|
|
|
33
33
|
import type { Mutex } from '../mutex.js'
|
|
34
34
|
import type { PGlite } from '@electric-sql/pglite'
|
|
35
35
|
|
|
36
|
+
// track concurrent replication handlers to detect reconnect-purge race
|
|
37
|
+
let activeHandlerCount = 0
|
|
38
|
+
|
|
36
39
|
export interface ReplicationWriter {
|
|
37
40
|
write(data: Uint8Array): void
|
|
38
41
|
}
|
|
@@ -263,6 +266,9 @@ export async function handleStartReplication(
|
|
|
263
266
|
db: PGlite,
|
|
264
267
|
mutex: Mutex
|
|
265
268
|
): Promise<void> {
|
|
269
|
+
activeHandlerCount++
|
|
270
|
+
const handlerId = activeHandlerCount
|
|
271
|
+
console.info(`[orez-repl#${handlerId}] START_REPLICATION (active handlers: ${activeHandlerCount})`)
|
|
266
272
|
log.debug.proxy('replication: entering streaming mode')
|
|
267
273
|
|
|
268
274
|
// send CopyBothResponse to enter streaming mode
|
|
@@ -457,6 +463,8 @@ export async function handleStartReplication(
|
|
|
457
463
|
mutex.release()
|
|
458
464
|
}
|
|
459
465
|
|
|
466
|
+
console.info(`[orez-repl#${handlerId}] setup complete, starting poll (lastWatermark=${lastWatermark})`)
|
|
467
|
+
|
|
460
468
|
// track which tables we've sent RELATION messages for
|
|
461
469
|
const sentRelations = new Set<string>()
|
|
462
470
|
let txCounter = 1
|
|
@@ -469,6 +477,7 @@ export async function handleStartReplication(
|
|
|
469
477
|
const purgeEveryN = 10
|
|
470
478
|
let running = true
|
|
471
479
|
let pollsSincePurge = 0
|
|
480
|
+
let lastIdleLog = 0
|
|
472
481
|
|
|
473
482
|
const poll = async () => {
|
|
474
483
|
while (running) {
|
|
@@ -483,6 +492,8 @@ export async function handleStartReplication(
|
|
|
483
492
|
}
|
|
484
493
|
|
|
485
494
|
if (changes.length > 0) {
|
|
495
|
+
const tables = [...new Set(changes.map(c => c.table_name))].join(',')
|
|
496
|
+
console.info(`[orez-repl#${handlerId}] found ${changes.length} changes [${tables}] (wm ${lastWatermark}→${changes[changes.length - 1].watermark}, type=${typeof changes[0].watermark})`)
|
|
486
497
|
await streamChanges(
|
|
487
498
|
changes,
|
|
488
499
|
writer,
|
|
@@ -502,12 +513,19 @@ export async function handleStartReplication(
|
|
|
502
513
|
try {
|
|
503
514
|
const purged = await purgeConsumedChanges(db, lastWatermark)
|
|
504
515
|
if (purged > 0) {
|
|
505
|
-
|
|
516
|
+
console.info(`[orez-repl#${handlerId}] purged ${purged} changes (wm<=${lastWatermark})`)
|
|
506
517
|
}
|
|
507
518
|
} finally {
|
|
508
519
|
mutex.release()
|
|
509
520
|
}
|
|
510
521
|
}
|
|
522
|
+
} else {
|
|
523
|
+
// throttled idle logging (every 10s)
|
|
524
|
+
const now = Date.now()
|
|
525
|
+
if (now - lastIdleLog > 10000) {
|
|
526
|
+
lastIdleLog = now
|
|
527
|
+
console.info(`[orez-repl#${handlerId}] idle (lastWatermark=${lastWatermark}, type=${typeof lastWatermark})`)
|
|
528
|
+
}
|
|
511
529
|
}
|
|
512
530
|
|
|
513
531
|
// send keepalive
|
|
@@ -531,7 +549,8 @@ export async function handleStartReplication(
|
|
|
531
549
|
|
|
532
550
|
log.debug.proxy('replication: starting poll loop')
|
|
533
551
|
await poll()
|
|
534
|
-
|
|
552
|
+
activeHandlerCount--
|
|
553
|
+
console.info(`[orez-repl#${handlerId}] poll loop exited (remaining handlers: ${activeHandlerCount})`)
|
|
535
554
|
}
|
|
536
555
|
|
|
537
556
|
async function streamChanges(
|