orez 0.0.46 → 0.0.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -8
- package/dist/admin/http-proxy.d.ts +31 -0
- package/dist/admin/http-proxy.d.ts.map +1 -0
- package/dist/admin/http-proxy.js +140 -0
- package/dist/admin/http-proxy.js.map +1 -0
- package/dist/admin/log-store.d.ts +22 -0
- package/dist/admin/log-store.d.ts.map +1 -0
- package/dist/admin/log-store.js +86 -0
- package/dist/admin/log-store.js.map +1 -0
- package/dist/admin/server.d.ts +19 -0
- package/dist/admin/server.d.ts.map +1 -0
- package/dist/admin/server.js +110 -0
- package/dist/admin/server.js.map +1 -0
- package/dist/admin/ui.d.ts +2 -0
- package/dist/admin/ui.d.ts.map +1 -0
- package/dist/admin/ui.js +683 -0
- package/dist/admin/ui.js.map +1 -0
- package/dist/cli.js +48 -1
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +4 -0
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +211 -20
- package/dist/index.js.map +1 -1
- package/dist/log.d.ts +9 -0
- package/dist/log.d.ts.map +1 -1
- package/dist/log.js +24 -1
- package/dist/log.js.map +1 -1
- package/dist/pg-proxy.d.ts.map +1 -1
- package/dist/pg-proxy.js +19 -4
- package/dist/pg-proxy.js.map +1 -1
- package/dist/pglite-manager.d.ts +1 -0
- package/dist/pglite-manager.d.ts.map +1 -1
- package/dist/pglite-manager.js +8 -2
- package/dist/pglite-manager.js.map +1 -1
- package/dist/replication/change-tracker.d.ts.map +1 -1
- package/dist/replication/change-tracker.js +16 -29
- package/dist/replication/change-tracker.js.map +1 -1
- package/dist/replication/handler.d.ts.map +1 -1
- package/dist/replication/handler.js +42 -7
- package/dist/replication/handler.js.map +1 -1
- package/dist/vite-plugin.d.ts +3 -0
- package/dist/vite-plugin.d.ts.map +1 -1
- package/dist/vite-plugin.js +24 -0
- package/dist/vite-plugin.js.map +1 -1
- package/package.json +4 -2
- package/src/admin/http-proxy.ts +190 -0
- package/src/admin/log-store.ts +114 -0
- package/src/admin/server.ts +152 -0
- package/src/admin/ui.ts +684 -0
- package/src/cli.ts +62 -13
- package/src/config.ts +8 -0
- package/src/index.ts +239 -20
- package/src/log.ts +25 -1
- package/src/pg-proxy.ts +27 -5
- package/src/pglite-manager.ts +9 -2
- package/src/replication/change-tracker.ts +20 -30
- package/src/replication/handler.ts +54 -8
- package/src/replication/pgoutput-encoder.test.ts +217 -0
- package/src/replication/zero-compat.test.ts +232 -1
- package/src/shim/hooks.mjs +33 -0
- package/src/vite-plugin.ts +28 -0
package/src/cli.ts
CHANGED
|
@@ -750,6 +750,11 @@ const main = defineCommand({
|
|
|
750
750
|
description: 'use native @rocicorp/zero-sqlite3 instead of wasm bedrock-sqlite',
|
|
751
751
|
default: false,
|
|
752
752
|
},
|
|
753
|
+
'log-env': {
|
|
754
|
+
type: 'boolean',
|
|
755
|
+
description: 'log ZERO_* and related environment variables on startup',
|
|
756
|
+
default: false,
|
|
757
|
+
},
|
|
753
758
|
'on-db-ready': {
|
|
754
759
|
type: 'string',
|
|
755
760
|
description: 'command to run after db+proxy are ready, before zero-cache starts',
|
|
@@ -760,6 +765,21 @@ const main = defineCommand({
|
|
|
760
765
|
description: 'command to run once all services are healthy',
|
|
761
766
|
default: '',
|
|
762
767
|
},
|
|
768
|
+
admin: {
|
|
769
|
+
type: 'boolean',
|
|
770
|
+
description: 'start admin web ui',
|
|
771
|
+
default: false,
|
|
772
|
+
},
|
|
773
|
+
'admin-port': {
|
|
774
|
+
type: 'string',
|
|
775
|
+
description: 'admin ui port (default: auto)',
|
|
776
|
+
default: '0',
|
|
777
|
+
},
|
|
778
|
+
'admin-logs': {
|
|
779
|
+
type: 'boolean',
|
|
780
|
+
description: 'write logs to .orez/logs/ (default: true when --admin)',
|
|
781
|
+
default: true,
|
|
782
|
+
},
|
|
763
783
|
},
|
|
764
784
|
subCommands: {
|
|
765
785
|
s3: s3Command,
|
|
@@ -767,19 +787,25 @@ const main = defineCommand({
|
|
|
767
787
|
pg_restore: pgRestoreCommand,
|
|
768
788
|
},
|
|
769
789
|
async run({ args }) {
|
|
770
|
-
const
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
790
|
+
const startTime = Date.now()
|
|
791
|
+
const { config, stop, logStore, zeroEnv, actions, httpLogStore } =
|
|
792
|
+
await startZeroLite({
|
|
793
|
+
pgPort: Number(args['pg-port']),
|
|
794
|
+
zeroPort: Number(args['zero-port']),
|
|
795
|
+
dataDir: args['data-dir'],
|
|
796
|
+
migrationsDir: args.migrations,
|
|
797
|
+
seedFile: args.seed,
|
|
798
|
+
pgUser: args['pg-user'],
|
|
799
|
+
pgPassword: args['pg-password'],
|
|
800
|
+
skipZeroCache: args['skip-zero-cache'],
|
|
801
|
+
disableWasmSqlite: args['disable-wasm-sqlite'],
|
|
802
|
+
logLevel: (args['log-level'] as 'error' | 'warn' | 'info' | 'debug') || undefined,
|
|
803
|
+
logEnv: args['log-env'],
|
|
804
|
+
onDbReady: args['on-db-ready'],
|
|
805
|
+
admin: args.admin,
|
|
806
|
+
adminPort: Number(args['admin-port']),
|
|
807
|
+
adminLogs: args['admin-logs'],
|
|
808
|
+
})
|
|
783
809
|
|
|
784
810
|
let s3Server: import('node:http').Server | null = null
|
|
785
811
|
if (args.s3) {
|
|
@@ -790,6 +816,27 @@ const main = defineCommand({
|
|
|
790
816
|
})
|
|
791
817
|
}
|
|
792
818
|
|
|
819
|
+
let adminServer: import('node:http').Server | null = null
|
|
820
|
+
if (args.admin && logStore) {
|
|
821
|
+
const { findPort } = await import('./port.js')
|
|
822
|
+
const adminPort = Number(args['admin-port']) || config.zeroPort + 2
|
|
823
|
+
const resolvedPort = await findPort(adminPort)
|
|
824
|
+
const { startAdminServer } = await import('./admin/server.js')
|
|
825
|
+
adminServer = await startAdminServer({
|
|
826
|
+
port: resolvedPort,
|
|
827
|
+
logStore,
|
|
828
|
+
config,
|
|
829
|
+
zeroEnv,
|
|
830
|
+
actions,
|
|
831
|
+
startTime,
|
|
832
|
+
httpLog: httpLogStore || undefined,
|
|
833
|
+
})
|
|
834
|
+
log.orez(`admin: http://127.0.0.1:${resolvedPort}`)
|
|
835
|
+
if (args['admin-logs']) {
|
|
836
|
+
log.orez(`logs: ${resolve(args['data-dir'], 'logs', 'orez.log')}`)
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
793
840
|
log.orez('ready')
|
|
794
841
|
log.orez(
|
|
795
842
|
`pg: postgresql://${config.pgUser}:${config.pgPassword}@127.0.0.1:${config.pgPort}/postgres`
|
|
@@ -817,11 +864,13 @@ const main = defineCommand({
|
|
|
817
864
|
}
|
|
818
865
|
|
|
819
866
|
process.on('SIGINT', async () => {
|
|
867
|
+
adminServer?.close()
|
|
820
868
|
s3Server?.close()
|
|
821
869
|
await stop()
|
|
822
870
|
process.exit(0)
|
|
823
871
|
})
|
|
824
872
|
process.on('SIGTERM', async () => {
|
|
873
|
+
adminServer?.close()
|
|
825
874
|
s3Server?.close()
|
|
826
875
|
await stop()
|
|
827
876
|
process.exit(0)
|
package/src/config.ts
CHANGED
|
@@ -14,8 +14,12 @@ export interface ZeroLiteConfig {
|
|
|
14
14
|
disableWasmSqlite: boolean
|
|
15
15
|
logLevel: LogLevel
|
|
16
16
|
pgliteOptions: Partial<PGliteOptions>
|
|
17
|
+
logEnv: boolean
|
|
17
18
|
onDbReady: string
|
|
18
19
|
beforeZero: ((db: import('@electric-sql/pglite').PGlite) => Promise<void>) | null
|
|
20
|
+
admin: boolean
|
|
21
|
+
adminPort: number
|
|
22
|
+
adminLogs: boolean
|
|
19
23
|
}
|
|
20
24
|
|
|
21
25
|
export function getConfig(overrides: Partial<ZeroLiteConfig> = {}): ZeroLiteConfig {
|
|
@@ -31,8 +35,12 @@ export function getConfig(overrides: Partial<ZeroLiteConfig> = {}): ZeroLiteConf
|
|
|
31
35
|
disableWasmSqlite: overrides.disableWasmSqlite ?? false,
|
|
32
36
|
logLevel: overrides.logLevel || 'warn',
|
|
33
37
|
pgliteOptions: overrides.pgliteOptions || {},
|
|
38
|
+
logEnv: overrides.logEnv ?? false,
|
|
34
39
|
onDbReady: overrides.onDbReady || '',
|
|
35
40
|
beforeZero: overrides.beforeZero || null,
|
|
41
|
+
admin: overrides.admin ?? false,
|
|
42
|
+
adminPort: overrides.adminPort || 0,
|
|
43
|
+
adminLogs: overrides.adminLogs ?? true,
|
|
36
44
|
}
|
|
37
45
|
}
|
|
38
46
|
|
package/src/index.ts
CHANGED
|
@@ -7,19 +7,28 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { spawn, type ChildProcess } from 'node:child_process'
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
existsSync,
|
|
12
|
+
mkdirSync,
|
|
13
|
+
readFileSync,
|
|
14
|
+
rmSync,
|
|
15
|
+
unlinkSync,
|
|
16
|
+
writeFileSync,
|
|
17
|
+
} from 'node:fs'
|
|
11
18
|
import { createRequire } from 'node:module'
|
|
12
19
|
import { totalmem } from 'node:os'
|
|
13
20
|
import { dirname, resolve } from 'node:path'
|
|
14
21
|
import { fileURLToPath } from 'node:url'
|
|
15
22
|
|
|
16
23
|
import { getConfig, getConnectionString } from './config.js'
|
|
17
|
-
import { log, port, setLogLevel } from './log.js'
|
|
24
|
+
import { log, port, setLogLevel, addLogListener } from './log.js'
|
|
18
25
|
import { startPgProxy } from './pg-proxy.js'
|
|
19
|
-
import { createPGliteInstances, runMigrations } from './pglite-manager.js'
|
|
26
|
+
import { createInstance, createPGliteInstances, runMigrations } from './pglite-manager.js'
|
|
20
27
|
import { findPort } from './port.js'
|
|
21
28
|
import { installChangeTracking } from './replication/change-tracker.js'
|
|
22
29
|
|
|
30
|
+
import type { HttpLogStore } from './admin/http-proxy.js'
|
|
31
|
+
import type { LogStore } from './admin/log-store.js'
|
|
23
32
|
import type { ZeroLiteConfig } from './config.js'
|
|
24
33
|
import type { PGlite } from '@electric-sql/pglite'
|
|
25
34
|
|
|
@@ -43,6 +52,25 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
43
52
|
const config = getConfig(overrides)
|
|
44
53
|
setLogLevel(config.logLevel)
|
|
45
54
|
|
|
55
|
+
// when admin ui enabled, create log store and capture all log output
|
|
56
|
+
const SOURCE_MAP: Record<string, string> = {
|
|
57
|
+
orez: 'orez',
|
|
58
|
+
pglite: 'pglite',
|
|
59
|
+
'pg-proxy': 'proxy',
|
|
60
|
+
zero: 'zero',
|
|
61
|
+
'zero-cache': 'zero',
|
|
62
|
+
'orez/s3': 's3',
|
|
63
|
+
}
|
|
64
|
+
let logStore: LogStore | null = null
|
|
65
|
+
let removeLogListener: (() => void) | null = null
|
|
66
|
+
if (config.admin) {
|
|
67
|
+
const { createLogStore } = await import('./admin/log-store.js')
|
|
68
|
+
logStore = createLogStore(config.dataDir, config.adminLogs)
|
|
69
|
+
removeLogListener = addLogListener((source, level, msg) => {
|
|
70
|
+
logStore!.push(SOURCE_MAP[source] || source, level, msg)
|
|
71
|
+
})
|
|
72
|
+
}
|
|
73
|
+
|
|
46
74
|
// find available ports
|
|
47
75
|
const pgPort = await findPort(config.pgPort)
|
|
48
76
|
const zeroPort = config.skipZeroCache
|
|
@@ -73,7 +101,7 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
73
101
|
// start tcp proxy (routes connections to correct instance by database name)
|
|
74
102
|
const pgServer = await startPgProxy(instances, config)
|
|
75
103
|
|
|
76
|
-
log.
|
|
104
|
+
log.pglite(`postgres up ${port(pgPort, 'green')}`)
|
|
77
105
|
if (migrationsApplied > 0)
|
|
78
106
|
log.orez(
|
|
79
107
|
`${migrationsApplied} migration${migrationsApplied === 1 ? '' : 's'} applied`
|
|
@@ -128,18 +156,143 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
128
156
|
// clean up stale lock files from previous crash (keep replica for fast restart)
|
|
129
157
|
cleanupStaleLockFiles(config)
|
|
130
158
|
|
|
131
|
-
//
|
|
159
|
+
// http proxy for admin traffic logging
|
|
160
|
+
let httpLogStore: HttpLogStore | null = null
|
|
161
|
+
let httpProxyServer: import('node:http').Server | null = null
|
|
162
|
+
let zeroInternalPort = zeroPort
|
|
163
|
+
if (config.admin && !config.skipZeroCache) {
|
|
164
|
+
const { createHttpLogStore } = await import('./admin/http-proxy.js')
|
|
165
|
+
httpLogStore = createHttpLogStore()
|
|
166
|
+
zeroInternalPort = await findPort(zeroPort + 100)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// start zero-cache with auto-recovery for stale change db
|
|
132
170
|
let zeroCacheProcess: ChildProcess | null = null
|
|
171
|
+
let zeroEnv: Record<string, string> = {}
|
|
172
|
+
const cdbResets = { count: 0, lastReset: 0 }
|
|
173
|
+
const MAX_CDB_RESETS = 10
|
|
174
|
+
const MIN_RESET_INTERVAL_MS = 60_000
|
|
175
|
+
|
|
133
176
|
if (!config.skipZeroCache) {
|
|
134
|
-
|
|
135
|
-
|
|
177
|
+
let currentResult = await startZeroCache(config, zeroInternalPort)
|
|
178
|
+
zeroCacheProcess = currentResult.child
|
|
179
|
+
zeroEnv = currentResult.env
|
|
180
|
+
|
|
181
|
+
// watch for stale changeLog crashes and auto-recover
|
|
182
|
+
const attachCdbRecovery = (result: typeof currentResult) => {
|
|
183
|
+
result.child.on('exit', async (code) => {
|
|
184
|
+
if (code === 0 || code === null) return
|
|
185
|
+
if (!result.stderrBuf.includes('changeLog_pkey')) return
|
|
186
|
+
|
|
187
|
+
const now = Date.now()
|
|
188
|
+
if (cdbResets.count >= MAX_CDB_RESETS) {
|
|
189
|
+
log.zero('change db reset limit reached, not retrying')
|
|
190
|
+
return
|
|
191
|
+
}
|
|
192
|
+
const elapsed = now - cdbResets.lastReset
|
|
193
|
+
if (elapsed < MIN_RESET_INTERVAL_MS) {
|
|
194
|
+
log.zero(
|
|
195
|
+
`change db reset too soon (${Math.round(elapsed / 1000)}s ago), not retrying`
|
|
196
|
+
)
|
|
197
|
+
return
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
cdbResets.count++
|
|
201
|
+
cdbResets.lastReset = now
|
|
202
|
+
log.zero(
|
|
203
|
+
`stale change db detected, resetting (${cdbResets.count}/${MAX_CDB_RESETS})`
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
try {
|
|
207
|
+
await instances.cdb.close()
|
|
208
|
+
const cdbPath = resolve(config.dataDir, 'pgdata-cdb')
|
|
209
|
+
rmSync(cdbPath, { recursive: true, force: true })
|
|
210
|
+
instances.cdb = await createInstance(config, 'cdb', false)
|
|
211
|
+
|
|
212
|
+
currentResult = await startZeroCache(config, zeroInternalPort)
|
|
213
|
+
zeroCacheProcess = currentResult.child
|
|
214
|
+
attachCdbRecovery(currentResult)
|
|
215
|
+
await waitForZeroCache(config, undefined, zeroInternalPort)
|
|
216
|
+
log.zero(`recovered, ready ${port(config.zeroPort, 'magenta')}`)
|
|
217
|
+
} catch (err) {
|
|
218
|
+
log.zero(`recovery failed: ${err}`)
|
|
219
|
+
}
|
|
220
|
+
})
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
attachCdbRecovery(currentResult)
|
|
224
|
+
await waitForZeroCache(config, undefined, zeroInternalPort)
|
|
136
225
|
log.zero(`ready ${port(config.zeroPort, 'magenta')}`)
|
|
226
|
+
|
|
227
|
+
// start http proxy for admin traffic logging
|
|
228
|
+
if (httpLogStore) {
|
|
229
|
+
const { startHttpProxy } = await import('./admin/http-proxy.js')
|
|
230
|
+
httpProxyServer = await startHttpProxy({
|
|
231
|
+
listenPort: zeroPort,
|
|
232
|
+
targetPort: zeroInternalPort,
|
|
233
|
+
httpLog: httpLogStore,
|
|
234
|
+
})
|
|
235
|
+
}
|
|
137
236
|
} else {
|
|
138
237
|
log.orez('skip zero-cache')
|
|
139
238
|
}
|
|
140
239
|
|
|
240
|
+
// admin action handlers
|
|
241
|
+
const actions = {
|
|
242
|
+
restartZero: config.skipZeroCache
|
|
243
|
+
? undefined
|
|
244
|
+
: async () => {
|
|
245
|
+
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
246
|
+
zeroCacheProcess.kill('SIGTERM')
|
|
247
|
+
await new Promise<void>((r) => {
|
|
248
|
+
const t = setTimeout(() => {
|
|
249
|
+
zeroCacheProcess?.kill('SIGKILL')
|
|
250
|
+
r()
|
|
251
|
+
}, 3000)
|
|
252
|
+
zeroCacheProcess!.on('exit', () => {
|
|
253
|
+
clearTimeout(t)
|
|
254
|
+
r()
|
|
255
|
+
})
|
|
256
|
+
})
|
|
257
|
+
}
|
|
258
|
+
const zc = await startZeroCache(config, zeroInternalPort)
|
|
259
|
+
zeroCacheProcess = zc.child
|
|
260
|
+
await waitForZeroCache(config, undefined, zeroInternalPort)
|
|
261
|
+
log.zero(`restarted ${port(config.zeroPort, 'magenta')}`)
|
|
262
|
+
},
|
|
263
|
+
resetZero: config.skipZeroCache
|
|
264
|
+
? undefined
|
|
265
|
+
: async () => {
|
|
266
|
+
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
267
|
+
zeroCacheProcess.kill('SIGTERM')
|
|
268
|
+
await new Promise<void>((r) => {
|
|
269
|
+
const t = setTimeout(() => {
|
|
270
|
+
zeroCacheProcess?.kill('SIGKILL')
|
|
271
|
+
r()
|
|
272
|
+
}, 3000)
|
|
273
|
+
zeroCacheProcess!.on('exit', () => {
|
|
274
|
+
clearTimeout(t)
|
|
275
|
+
r()
|
|
276
|
+
})
|
|
277
|
+
})
|
|
278
|
+
}
|
|
279
|
+
const replicaPath = resolve(config.dataDir, 'zero-replica.db')
|
|
280
|
+
for (const suffix of ['', '-wal', '-shm', '-wal2']) {
|
|
281
|
+
try {
|
|
282
|
+
if (existsSync(replicaPath + suffix)) unlinkSync(replicaPath + suffix)
|
|
283
|
+
} catch {}
|
|
284
|
+
}
|
|
285
|
+
const zc = await startZeroCache(config, zeroInternalPort)
|
|
286
|
+
zeroCacheProcess = zc.child
|
|
287
|
+
await waitForZeroCache(config, undefined, zeroInternalPort)
|
|
288
|
+
log.zero(`reset and restarted ${port(config.zeroPort, 'magenta')}`)
|
|
289
|
+
},
|
|
290
|
+
}
|
|
291
|
+
|
|
141
292
|
const stop = async () => {
|
|
142
293
|
log.debug.orez('shutting down')
|
|
294
|
+
removeLogListener?.()
|
|
295
|
+
httpProxyServer?.close()
|
|
143
296
|
if (zeroCacheProcess && !zeroCacheProcess.killed) {
|
|
144
297
|
zeroCacheProcess.kill('SIGTERM')
|
|
145
298
|
// wait up to 3s for graceful exit, then force kill
|
|
@@ -165,7 +318,18 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
|
|
|
165
318
|
log.debug.orez('stopped')
|
|
166
319
|
}
|
|
167
320
|
|
|
168
|
-
return {
|
|
321
|
+
return {
|
|
322
|
+
config,
|
|
323
|
+
stop,
|
|
324
|
+
db,
|
|
325
|
+
instances,
|
|
326
|
+
pgPort: config.pgPort,
|
|
327
|
+
zeroPort: config.zeroPort,
|
|
328
|
+
logStore,
|
|
329
|
+
zeroEnv,
|
|
330
|
+
actions,
|
|
331
|
+
httpLogStore,
|
|
332
|
+
}
|
|
169
333
|
}
|
|
170
334
|
|
|
171
335
|
function cleanupStaleLockFiles(config: ZeroLiteConfig): void {
|
|
@@ -244,7 +408,10 @@ function writeSqliteShim(): string {
|
|
|
244
408
|
return registerPath
|
|
245
409
|
}
|
|
246
410
|
|
|
247
|
-
async function startZeroCache(
|
|
411
|
+
async function startZeroCache(
|
|
412
|
+
config: ZeroLiteConfig,
|
|
413
|
+
portOverride?: number
|
|
414
|
+
): Promise<{ child: ChildProcess; env: Record<string, string>; stderrBuf: string }> {
|
|
248
415
|
// resolve @rocicorp/zero entry for finding zero-cache modules
|
|
249
416
|
const zeroEntry = resolvePackage('@rocicorp/zero')
|
|
250
417
|
|
|
@@ -263,7 +430,7 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
263
430
|
// defaults that can be overridden by user env
|
|
264
431
|
const defaults: Record<string, string> = {
|
|
265
432
|
NODE_ENV: 'development',
|
|
266
|
-
ZERO_LOG_LEVEL:
|
|
433
|
+
ZERO_LOG_LEVEL: 'info',
|
|
267
434
|
ZERO_NUM_SYNC_WORKERS: '1',
|
|
268
435
|
// disable query planner — it relies on scanStatus which causes infinite
|
|
269
436
|
// loops with wasm sqlite and has caused freezes with native too.
|
|
@@ -289,7 +456,7 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
289
456
|
ZERO_CVR_DB: cvrUrl,
|
|
290
457
|
ZERO_CHANGE_DB: cdbUrl,
|
|
291
458
|
ZERO_REPLICA_FILE: resolve(config.dataDir, 'zero-replica.db'),
|
|
292
|
-
ZERO_PORT: String(config.zeroPort),
|
|
459
|
+
ZERO_PORT: String(portOverride || config.zeroPort),
|
|
293
460
|
}
|
|
294
461
|
|
|
295
462
|
const zeroCacheBin = resolve(zeroEntry, '..', 'cli.js')
|
|
@@ -313,31 +480,82 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
313
480
|
env.NODE_OPTIONS = `--max-old-space-size=${heapMB} ${existing}`.trim()
|
|
314
481
|
}
|
|
315
482
|
|
|
483
|
+
// log env vars if --log-env was passed
|
|
484
|
+
if (config.logEnv) {
|
|
485
|
+
const zeroVars = Object.entries(env)
|
|
486
|
+
.filter(([key]) => key.startsWith('ZERO_') || key === 'NODE_ENV')
|
|
487
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
488
|
+
log.orez('zero-cache env:')
|
|
489
|
+
for (const [key, value] of zeroVars) {
|
|
490
|
+
log.orez(` ${key}=${value}`)
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
316
494
|
const child = spawn(zeroCacheBin, [], {
|
|
317
495
|
env,
|
|
318
496
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
319
497
|
})
|
|
320
498
|
|
|
499
|
+
// zero-cache uses structured logging when piped (not a tty).
|
|
500
|
+
// multiline format: timestamp + "[" on one line, context lines, "] message" on another.
|
|
501
|
+
// single-line format: timestamp + [ context ] message, or timestamp + key=val,... message
|
|
502
|
+
// we buffer multiline blocks and extract just the message.
|
|
503
|
+
const timestampRe = /^\d{4}-\d{2}-\d{2}T[\d:.+\-Z]+\s*/
|
|
504
|
+
let inBlock = false
|
|
505
|
+
const zeroLog = (line: string) => {
|
|
506
|
+
let stripped = line.replace(timestampRe, '')
|
|
507
|
+
|
|
508
|
+
// start of multiline context block: line ends with "[" (possibly after timestamp)
|
|
509
|
+
if (!inBlock && /^\[?\s*$/.test(stripped)) {
|
|
510
|
+
inBlock = true
|
|
511
|
+
return
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// inside multiline block: skip context lines, look for "] message"
|
|
515
|
+
if (inBlock) {
|
|
516
|
+
const closeMatch = stripped.match(/^\]\s*(.*)$/)
|
|
517
|
+
if (closeMatch) {
|
|
518
|
+
inBlock = false
|
|
519
|
+
const msg = closeMatch[1].trim()
|
|
520
|
+
if (msg) log.zero(msg)
|
|
521
|
+
}
|
|
522
|
+
// context continuation lines like "'pid=8278'," — skip
|
|
523
|
+
return
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// single-line: strip inline [ context ] and key=val prefixes
|
|
527
|
+
stripped = stripped.replace(/\[.*?\]\s*/g, '')
|
|
528
|
+
stripped = stripped.replace(/^(?:\w+=\S+,)*\w+=\S+\s+/, '')
|
|
529
|
+
stripped = stripped.trim()
|
|
530
|
+
|
|
531
|
+
if (!stripped || /^[\[\]',\s]*$/.test(stripped)) return
|
|
532
|
+
|
|
533
|
+
log.zero(stripped)
|
|
534
|
+
}
|
|
535
|
+
|
|
321
536
|
child.stdout?.on('data', (data: Buffer) => {
|
|
322
537
|
const lines = data.toString().trim().split('\n')
|
|
323
538
|
for (const line of lines) {
|
|
324
|
-
|
|
539
|
+
zeroLog(line)
|
|
325
540
|
}
|
|
326
541
|
})
|
|
327
542
|
|
|
328
|
-
|
|
543
|
+
const result = { child, env, stderrBuf: '' }
|
|
544
|
+
|
|
329
545
|
child.stderr?.on('data', (data: Buffer) => {
|
|
330
546
|
const chunk = data.toString()
|
|
331
|
-
stderrBuf += chunk
|
|
547
|
+
result.stderrBuf += chunk
|
|
332
548
|
const lines = chunk.trim().split('\n')
|
|
333
549
|
for (const line of lines) {
|
|
334
|
-
|
|
550
|
+
zeroLog(line)
|
|
335
551
|
}
|
|
336
552
|
})
|
|
337
553
|
|
|
338
554
|
child.on('exit', (code) => {
|
|
339
555
|
if (code !== 0 && code !== null) {
|
|
340
|
-
|
|
556
|
+
// changeLog_pkey errors are handled by the recovery logic in startZeroLite
|
|
557
|
+
if (result.stderrBuf.includes('changeLog_pkey')) return
|
|
558
|
+
if (result.stderrBuf.includes('Could not locate the bindings file')) {
|
|
341
559
|
log.zero(
|
|
342
560
|
'native @rocicorp/zero-sqlite3 not found — native deps were not compiled.\n' +
|
|
343
561
|
'either:\n' +
|
|
@@ -346,7 +564,7 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
346
564
|
' or add "trustedDependencies": ["@rocicorp/zero-sqlite3"] to package.json'
|
|
347
565
|
)
|
|
348
566
|
} else {
|
|
349
|
-
const lastLines = stderrBuf.trim().split('\n').slice(-5).join('\n')
|
|
567
|
+
const lastLines = result.stderrBuf.trim().split('\n').slice(-5).join('\n')
|
|
350
568
|
if (lastLines) {
|
|
351
569
|
log.zero(`exited with code ${code}:\n${lastLines}`)
|
|
352
570
|
} else {
|
|
@@ -356,15 +574,16 @@ async function startZeroCache(config: ZeroLiteConfig): Promise<ChildProcess> {
|
|
|
356
574
|
}
|
|
357
575
|
})
|
|
358
576
|
|
|
359
|
-
return
|
|
577
|
+
return result
|
|
360
578
|
}
|
|
361
579
|
|
|
362
580
|
async function waitForZeroCache(
|
|
363
581
|
config: ZeroLiteConfig,
|
|
364
|
-
timeoutMs = 120000
|
|
582
|
+
timeoutMs = 120000,
|
|
583
|
+
portOverride?: number
|
|
365
584
|
): Promise<void> {
|
|
366
585
|
const start = Date.now()
|
|
367
|
-
const url = `http://127.0.0.1:${config.zeroPort}/`
|
|
586
|
+
const url = `http://127.0.0.1:${portOverride || config.zeroPort}/`
|
|
368
587
|
|
|
369
588
|
while (Date.now() - start < timeoutMs) {
|
|
370
589
|
try {
|
package/src/log.ts
CHANGED
|
@@ -25,6 +25,17 @@ export function setLogLevel(level: LogLevel) {
|
|
|
25
25
|
currentLevel = level
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
type LogListener = (source: string, level: LogLevel, msg: string) => void
|
|
29
|
+
const listeners: LogListener[] = []
|
|
30
|
+
|
|
31
|
+
export function addLogListener(fn: LogListener) {
|
|
32
|
+
listeners.push(fn)
|
|
33
|
+
return () => {
|
|
34
|
+
const idx = listeners.indexOf(fn)
|
|
35
|
+
if (idx !== -1) listeners.splice(idx, 1)
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
28
39
|
function prefix(label: string, color: string): string {
|
|
29
40
|
return `${BOLD}${color}[${label}]${RESET}`
|
|
30
41
|
}
|
|
@@ -37,6 +48,12 @@ export function port(n: number, color: keyof typeof COLORS): string {
|
|
|
37
48
|
function makeLogger(label: string, color: string, level: LogLevel = 'info') {
|
|
38
49
|
const p = prefix(label, color)
|
|
39
50
|
return (...args: unknown[]) => {
|
|
51
|
+
// always notify listeners (they capture everything for admin ui)
|
|
52
|
+
if (listeners.length > 0) {
|
|
53
|
+
const msg = args.map((a) => (typeof a === 'string' ? a : String(a))).join(' ')
|
|
54
|
+
for (const fn of listeners) fn(label, level, msg)
|
|
55
|
+
}
|
|
56
|
+
// only print to terminal if level passes filter
|
|
40
57
|
if (LEVEL_PRIORITY[level] <= LEVEL_PRIORITY[currentLevel]) {
|
|
41
58
|
console.info(p, ...args)
|
|
42
59
|
}
|
|
@@ -47,8 +64,15 @@ export const log = {
|
|
|
47
64
|
orez: makeLogger('orez', COLORS.cyan, 'warn'),
|
|
48
65
|
pglite: makeLogger('pglite', COLORS.green, 'warn'),
|
|
49
66
|
proxy: makeLogger('pg-proxy', COLORS.yellow, 'warn'),
|
|
50
|
-
zero: makeLogger('zero
|
|
67
|
+
zero: makeLogger('zero', COLORS.magenta, 'warn'),
|
|
51
68
|
s3: makeLogger('orez/s3', COLORS.blue, 'warn'),
|
|
69
|
+
info: {
|
|
70
|
+
orez: makeLogger('orez', COLORS.cyan, 'info'),
|
|
71
|
+
pglite: makeLogger('pglite', COLORS.green, 'info'),
|
|
72
|
+
proxy: makeLogger('pg-proxy', COLORS.yellow, 'info'),
|
|
73
|
+
zero: makeLogger('zero', COLORS.magenta, 'info'),
|
|
74
|
+
s3: makeLogger('orez/s3', COLORS.blue, 'info'),
|
|
75
|
+
},
|
|
52
76
|
debug: {
|
|
53
77
|
orez: makeLogger('orez', COLORS.cyan, 'debug'),
|
|
54
78
|
pglite: makeLogger('pglite', COLORS.green, 'debug'),
|
package/src/pg-proxy.ts
CHANGED
|
@@ -464,6 +464,14 @@ async function performHandshake(
|
|
|
464
464
|
return { params }
|
|
465
465
|
}
|
|
466
466
|
|
|
467
|
+
// ── connection tracking ──
|
|
468
|
+
|
|
469
|
+
// per-database active connection count. pglite is single-session so all
|
|
470
|
+
// connections share one transaction context. we skip ROLLBACK on close when
|
|
471
|
+
// other connections are still active to avoid killing their transactions.
|
|
472
|
+
const activeConns: Record<string, number> = {}
|
|
473
|
+
let connCounter = 0
|
|
474
|
+
|
|
467
475
|
// ── message loop ──
|
|
468
476
|
|
|
469
477
|
// process messages from a connected, authenticated client.
|
|
@@ -674,6 +682,7 @@ export async function startPgProxy(
|
|
|
674
682
|
|
|
675
683
|
let dbName = 'postgres'
|
|
676
684
|
let isReplicationConnection = false
|
|
685
|
+
const connId = ++connCounter
|
|
677
686
|
|
|
678
687
|
try {
|
|
679
688
|
// perform startup handshake
|
|
@@ -682,18 +691,31 @@ export async function startPgProxy(
|
|
|
682
691
|
dbName = params.database || 'postgres'
|
|
683
692
|
isReplicationConnection = params.replication === 'database'
|
|
684
693
|
|
|
685
|
-
|
|
686
|
-
|
|
694
|
+
// track active connections per database
|
|
695
|
+
activeConns[dbName] = (activeConns[dbName] || 0) + 1
|
|
696
|
+
|
|
697
|
+
console.info(
|
|
698
|
+
`[orez-proxy#${connId}] connect db=${dbName} repl=${params.replication || 'none'}`
|
|
687
699
|
)
|
|
688
700
|
|
|
689
701
|
const { db } = getDbContext(dbName)
|
|
690
702
|
await db.waitReady
|
|
691
703
|
|
|
692
704
|
// clean up pglite session state when client disconnects.
|
|
693
|
-
// pglite is single-session — all connections share one session
|
|
694
|
-
//
|
|
695
|
-
//
|
|
705
|
+
// pglite is single-session — all connections share one session.
|
|
706
|
+
// only ROLLBACK + reset when this is the LAST connection for this db,
|
|
707
|
+
// to avoid killing another connection's active transaction.
|
|
696
708
|
socket.on('close', async () => {
|
|
709
|
+
activeConns[dbName] = Math.max(0, (activeConns[dbName] || 1) - 1)
|
|
710
|
+
const remaining = activeConns[dbName]
|
|
711
|
+
const shouldRollback = remaining === 0
|
|
712
|
+
|
|
713
|
+
console.info(
|
|
714
|
+
`[orez-proxy#${connId}] close [${dbName}] (remaining=${remaining}, shouldRollback=${shouldRollback})`
|
|
715
|
+
)
|
|
716
|
+
|
|
717
|
+
if (!shouldRollback) return
|
|
718
|
+
|
|
697
719
|
const { db: closeDb, mutex: closeMutex } = getDbContext(dbName)
|
|
698
720
|
await closeMutex.acquire()
|
|
699
721
|
try {
|
package/src/pglite-manager.ts
CHANGED
|
@@ -16,7 +16,7 @@ export interface PGliteInstances {
|
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
// create a single pglite instance with given dataDir suffix
|
|
19
|
-
async function createInstance(
|
|
19
|
+
export async function createInstance(
|
|
20
20
|
config: ZeroLiteConfig,
|
|
21
21
|
name: string,
|
|
22
22
|
withExtensions: boolean
|
|
@@ -133,8 +133,15 @@ export async function runMigrations(db: PGlite, config: ZeroLiteConfig): Promise
|
|
|
133
133
|
continue
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
+
const filePath = join(migrationsDir, file)
|
|
137
|
+
if (!existsSync(filePath)) {
|
|
138
|
+
// .ts-only custom migrations are handled by the app's own migration runner
|
|
139
|
+
log.debug.orez(`skipping migration (no .sql file): ${name}`)
|
|
140
|
+
continue
|
|
141
|
+
}
|
|
142
|
+
|
|
136
143
|
log.debug.orez(`applying migration: ${name}`)
|
|
137
|
-
const sql = readFileSync(
|
|
144
|
+
const sql = readFileSync(filePath, 'utf-8')
|
|
138
145
|
|
|
139
146
|
// split by drizzle's statement-breakpoint marker
|
|
140
147
|
const statements = sql
|