orez 0.1.6 → 0.1.7

package/src/index.ts

@@ -23,6 +23,9 @@ import { createPGliteInstances, runMigrations } from './pglite-manager.js'
 import { findPort } from './port.js'
 import { installChangeTracking } from './replication/change-tracker.js'
 import {
+  formatNativeBootstrapInstructions,
+  hasMissingNativeBinarySignature,
+  inspectNativeSqliteBinary,
   resolveSqliteMode,
   resolveSqliteModeConfig,
   type SqliteMode,
@@ -32,6 +35,8 @@ import {
 import type { ZeroLiteConfig } from './config.js'
 import type { PGlite } from '@electric-sql/pglite'
 
+type ZeroChildProcess = ChildProcess & { __orezTail?: string[] }
+
 export { getConfig, getConnectionString } from './config.js'
 export type { Hook, LogLevel, ZeroLiteConfig } from './config.js'
 
@@ -126,6 +131,12 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
   const pidFile = resolve(config.dataDir, 'orez.pid')
   writeFileSync(pidFile, String(process.pid))
 
+  // write admin port file so pg_restore can find it
+  const adminFile = resolve(config.dataDir, 'orez.admin')
+  if (adminPort > 0) {
+    writeFileSync(adminFile, String(adminPort))
+  }
+
   // start pglite (separate instances for postgres, zero_cvr, zero_cdb)
   const instances = await createPGliteInstances(config)
   const db = instances.postgres
@@ -191,7 +202,7 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
     )
     zeroCacheProcess = result.process
     zeroEnv = result.env
-    await waitForZeroCache(zeroConfig)
+    await waitForZeroCache(zeroConfig, zeroCacheProcess, 60000, sqliteMode)
 
     // start http proxy in front of zero-cache when admin is enabled
     if (httpLog) {
@@ -247,7 +258,7 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
     )
     zeroCacheProcess = result.process
     zeroEnv = result.env
-    await waitForZeroCache(zeroConfig)
+    await waitForZeroCache(zeroConfig, zeroCacheProcess, 60000, sqliteMode)
   }
 
   // unified reset function for zero state
@@ -312,6 +323,38 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
         await instances.cvr.waitReady
         await instances.cdb.waitReady
         log.orez('CVR/CDB recreated')
+
+        // remove stale zero shard schemas from upstream; these can outlive CVR/CDB
+        // and cause dispatcher errors after full reset.
+        const shardSchemas = await db.query<{ schemaname: string }>(
+          `SELECT DISTINCT schemaname
+           FROM pg_tables
+           WHERE tablename IN ('clients', 'replicas', 'mutations')
+             AND schemaname NOT IN (
+               'pg_catalog',
+               'information_schema',
+               'pg_toast',
+               'public',
+               '_orez'
+             )
+             AND schemaname NOT LIKE 'pg_%'`
+        )
+        for (const { schemaname } of shardSchemas.rows) {
+          const quoted = '"' + schemaname.replace(/"/g, '""') + '"'
+          await db.exec(`DROP SCHEMA IF EXISTS ${quoted} CASCADE`)
+        }
+        if (shardSchemas.rows.length > 0) {
+          log.orez(`dropped ${shardSchemas.rows.length} stale shard schema(s)`)
+        }
+
+        // clear upstream replication tracking so zero-cache starts from a
+        // clean change stream baseline after full reset.
+        await db.exec(`TRUNCATE _orez._zero_changes`).catch(() => {})
+        await db.exec(`TRUNCATE _orez._zero_replication_slots`).catch(() => {})
+        await db
+          .exec(`ALTER SEQUENCE _orez._zero_watermark RESTART WITH 1`)
+          .catch(() => {})
+        log.orez('cleared upstream replication tracking state')
       }
 
       // always clean up replica file
@@ -331,12 +374,13 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
           DATABASE_URL: upstreamUrl,
           OREZ_PG_PORT: String(config.pgPort),
         })
-
-        // re-install change tracking on any tables created/modified by on-db-ready
-        log.debug.orez('re-installing change tracking after on-db-ready')
-        await installChangeTracking(db)
       }
 
+      // always re-install change tracking after a full reset so public table
+      // triggers reflect any schema changes introduced by restore.
+      log.debug.orez('re-installing change tracking after full reset')
+      await installChangeTracking(db)
+
       // restart zero-cache
       log.orez('starting zero-cache...')
       // use internal port when http proxy is enabled
@@ -350,7 +394,7 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
       zeroCacheProcess = result.process
       zeroEnv = result.env
 
-      await waitForZeroCache(zeroConfig)
+      await waitForZeroCache(zeroConfig, zeroCacheProcess, 60000, sqliteMode)
       log.orez(`zero state reset complete (${mode})`)
       log.zero(`ready ${port(config.zeroPort, 'magenta')}`)
     } catch (err: any) {
@@ -365,14 +409,22 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
     }
   }
 
-  // handle SIGUSR1 to reset zero state (sent by pg_restore)
+  // handle SIGUSR1 to reset zero state (sent by pg_restore after restore completes)
   if (!config.skipZeroCache) {
     process.on('SIGUSR1', () => {
-      log.orez('received SIGUSR1')
+      log.orez('received SIGUSR1 - full reset')
       resetZeroState('full').catch((err) => {
         log.orez(`SIGUSR1 reset failed: ${err?.message || err}`)
       })
     })
+
+    // handle SIGUSR2 to quiesce zero-cache (sent by pg_restore before restore starts)
+    process.on('SIGUSR2', () => {
+      log.orez('received SIGUSR2 - stopping zero-cache for restore')
+      killZeroCache().catch((err) => {
+        log.orez(`SIGUSR2 stop failed: ${err?.message || err}`)
+      })
+    })
   }
 
   const stop = async () => {
@@ -388,6 +440,9 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
     try {
       unlinkSync(pidFile)
     } catch {}
+    try {
+      unlinkSync(adminFile)
+    } catch {}
     log.debug.orez('stopped')
   }
 
@@ -402,6 +457,8 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
     httpLog,
     zeroEnv,
     restartZero: config.skipZeroCache ? undefined : restartZeroCache,
+    // stop zero-cache without restart (for pg_restore to safely modify schema)
+    stopZero: config.skipZeroCache ? undefined : killZeroCache,
     // cache-only reset: just replica file (fast, for minor sync issues)
     resetZero: config.skipZeroCache ? undefined : () => resetZeroState('cache-only'),
     // full reset: CVR/CDB + replica (for schema changes, used by pg_restore via SIGUSR1)
@@ -594,7 +651,14 @@ async function startZeroCache(
   const child = spawn(zeroCacheBin, [], {
     env,
     stdio: ['ignore', 'pipe', 'pipe'],
-  })
+  }) as ZeroChildProcess
+  child.__orezTail = []
+
+  const pushTail = (line: string) => {
+    const tail = child.__orezTail!
+    tail.push(line)
+    if (tail.length > 80) tail.splice(0, tail.length - 80)
+  }
 
   // detect log level from zero-cache output
   const detectLevel = (line: string, fallback: string): string => {
@@ -618,21 +682,28 @@ async function startZeroCache(
   child.stdout?.on('data', (data: Buffer) => {
     const lines = data.toString().trim().split('\n')
     for (const line of lines) {
-      log.debug.zero(line)
-      logStore?.push('zero', detectLevel(line, 'info'), line)
+      pushTail(`stdout: ${line}`)
+      const level = detectLevel(line, 'info')
+      if (level === 'warn' || level === 'error') log.zero(line)
+      else log.debug.zero(line)
+      logStore?.push('zero', level, line)
     }
   })
 
   child.stderr?.on('data', (data: Buffer) => {
     const lines = data.toString().trim().split('\n')
     for (const line of lines) {
-      log.debug.zero(line)
-      logStore?.push('zero', detectLevel(line, 'error'), line)
+      pushTail(`stderr: ${line}`)
+      const level = detectLevel(line, 'error')
+      if (level === 'warn' || level === 'error') log.zero(line)
+      else log.debug.zero(line)
+      logStore?.push('zero', level, line)
     }
   })
 
   child.on('exit', (code) => {
     if (code !== 0 && code !== null) {
+      pushTail(`exit: code ${code}`)
       log.zero(`exited with code ${code}`)
       logStore?.push('zero', 'error', `exited with code ${code}`)
     }
@@ -643,20 +714,48 @@ async function startZeroCache(
 
 async function waitForZeroCache(
   config: ZeroLiteConfig,
-  timeoutMs = 60000
+  zeroProcess?: ChildProcess | null,
+  timeoutMs = 60000,
+  sqliteMode: SqliteMode = resolveSqliteMode(config.disableWasmSqlite)
 ): Promise<void> {
   const start = Date.now()
   const url = `http://127.0.0.1:${config.zeroPort}/`
 
   while (Date.now() - start < timeoutMs) {
+    if (zeroProcess && zeroProcess.exitCode !== null) {
+      const tail = (zeroProcess as ZeroChildProcess).__orezTail
+      const details = tail?.length ? `\n${tail.slice(-20).join('\n')}` : ''
+      throw new Error(
+        `zero-cache exited with code ${zeroProcess.exitCode}${details}${nativeStartupDiagnostics(details, sqliteMode)}`
+      )
+    }
+
     try {
-      const res = await fetch(url)
-      if (res.ok) return
+      const controller = new AbortController()
+      const timer = setTimeout(() => controller.abort(), 1000)
+      const res = await fetch(url, { signal: controller.signal })
+      clearTimeout(timer)
+      // zero may return 404 on "/" while still being healthy.
+      if (res.ok || res.status === 404) return
     } catch {
       // not ready yet
     }
     await new Promise((r) => setTimeout(r, 500))
   }
 
-  log.zero('health check timed out, continuing anyway')
+  const tail = (zeroProcess as ZeroChildProcess | null | undefined)?.__orezTail
+  const details = tail?.length ? `\n${tail.slice(-20).join('\n')}` : ''
+  throw new Error(
+    `zero-cache health check timed out after ${timeoutMs}ms${details}${nativeStartupDiagnostics(details, sqliteMode)}`
+  )
+}
+
+function nativeStartupDiagnostics(details: string, sqliteMode: SqliteMode): string {
+  if (sqliteMode !== 'native') return ''
+  if (!details) return ''
+  if (!hasMissingNativeBinarySignature(details)) return ''
+
+  const check = inspectNativeSqliteBinary()
+  const instructions = formatNativeBootstrapInstructions(check)
+  return `\n\nnative sqlite startup diagnostics:\n${instructions}`
 }

package/src/integration/integration.test.ts

@@ -11,9 +11,21 @@ import { describe, expect, test, beforeAll, afterAll, beforeEach } from 'vitest'
 import WebSocket from 'ws'
 
 import { startZeroLite } from '../index.js'
+import { installChangeTracking } from '../replication/change-tracker.js'
+import { installAllowAllPermissions } from './test-permissions.js'
 
 import type { PGlite } from '@electric-sql/pglite'
 
+const SYNC_PROTOCOL_VERSION = 45
+
+function encodeSecProtocols(
+  initConnectionMessage: unknown,
+  authToken: string | undefined
+): string {
+  const payload = JSON.stringify({ initConnectionMessage, authToken })
+  return encodeURIComponent(Buffer.from(payload, 'utf-8').toString('base64'))
+}
+
 // simple async queue for collecting websocket messages
 class Queue<T> {
   private items: T[] = []
@@ -57,6 +69,7 @@ describe('orez integration', { timeout: 120000 }, () => {
   let zeroPort: number
   let pgPort: number
   let shutdown: () => Promise<void>
+  let restartZero: (() => Promise<void>) | undefined
   let dataDir: string
 
   beforeAll(async () => {
@@ -77,6 +90,7 @@ describe('orez integration', { timeout: 120000 }, () => {
     zeroPort = result.zeroPort
     pgPort = result.pgPort
     shutdown = result.stop
+    restartZero = result.restartZero
 
     console.log(`[test] orez started, creating tables`)
 
@@ -93,6 +107,22 @@ describe('orez integration', { timeout: 120000 }, () => {
         foo_id TEXT
       );
     `)
+    const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
+    if (pubName) {
+      const quotedPub = '"' + pubName.replace(/"/g, '""') + '"'
+      await db
+        .exec(`ALTER PUBLICATION ${quotedPub} ADD TABLE "public"."foo"`)
+        .catch(() => {})
+      await db
+        .exec(`ALTER PUBLICATION ${quotedPub} ADD TABLE "public"."bar"`)
+        .catch(() => {})
+      await installChangeTracking(db)
+    }
+    await installAllowAllPermissions(db, ['foo', 'bar'])
+    if (restartZero) {
+      await restartZero()
+    }
+    await ensureClientGroup(zeroPort, 'test-cg')
 
     console.log(`[test] tables created, waiting for zero-cache`)
     // wait for zero-cache to be ready
@@ -116,9 +146,14 @@ describe('orez integration', { timeout: 120000 }, () => {
   })
 
   test('zero-cache starts and accepts websocket connections', async () => {
+    const secProtocol = encodeSecProtocols(
+      ['initConnection', { desiredQueriesPatch: [] }],
+      undefined
+    )
     const ws = new WebSocket(
-      `ws://localhost:${zeroPort}/sync/v4/connect` +
-        `?clientGroupID=test-cg&clientID=test-client&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`
+      `ws://localhost:${zeroPort}/sync/v${SYNC_PROTOCOL_VERSION}/connect` +
+        `?clientGroupID=test-cg&clientID=test-client&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`,
+      secProtocol
     )
 
     const connected = new Promise<void>((resolve, reject) => {
@@ -323,26 +358,25 @@ describe('orez integration', { timeout: 120000 }, () => {
     downstream: Queue<unknown>,
     query: Record<string, unknown>
   ): WebSocket {
+    const secProtocol = encodeSecProtocols(
+      [
+        'initConnection',
+        {
+          desiredQueriesPatch: [{ op: 'put', hash: 'q1', ast: query }],
+        },
+      ],
+      undefined
+    )
     const ws = new WebSocket(
-      `ws://localhost:${port}/sync/v4/connect` +
-        `?clientGroupID=test-cg-${Date.now()}&clientID=test-client&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`
+      `ws://localhost:${port}/sync/v${SYNC_PROTOCOL_VERSION}/connect` +
+        `?clientGroupID=test-cg&clientID=test-client&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`,
+      secProtocol
     )
 
     ws.on('message', (data) => {
       downstream.enqueue(JSON.parse(data.toString()))
     })
 
-    ws.on('open', () => {
-      ws.send(
-        JSON.stringify([
-          'initConnection',
-          {
-            desiredQueriesPatch: [{ op: 'put', hash: 'q1', ast: query }],
-          },
-        ])
-      )
-    })
-
     return ws
   }
 
@@ -423,3 +457,40 @@ async function waitForZero(port: number, timeoutMs = 30000) {
   }
   throw new Error(`zero-cache not ready on port ${port} after ${timeoutMs}ms`)
 }
+
+async function ensureClientGroup(port: number, clientGroupID: string): Promise<void> {
+  const secProtocol = encodeSecProtocols(
+    ['initConnection', { desiredQueriesPatch: [], clientSchema: { tables: {} } }],
+    undefined
+  )
+  await new Promise<void>((resolve, reject) => {
+    const ws = new WebSocket(
+      `ws://localhost:${port}/sync/v${SYNC_PROTOCOL_VERSION}/connect` +
+        `?clientGroupID=${clientGroupID}&clientID=test-client&wsid=ws-bootstrap&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`,
+      secProtocol
+    )
+
+    const timer = setTimeout(() => {
+      try {
+        ws.close()
+      } catch {}
+      reject(new Error('client-group bootstrap timeout'))
+    }, 7000)
+
+    ws.once('message', (data) => {
+      clearTimeout(timer)
+      const msg = JSON.parse(data.toString())
+      const isError = Array.isArray(msg) && msg[0] === 'error'
+      ws.close()
+      if (isError) {
+        reject(new Error(`client-group bootstrap failed: ${JSON.stringify(msg)}`))
+      } else {
+        resolve()
+      }
+    })
+    ws.once('error', (err) => {
+      clearTimeout(timer)
+      reject(err)
+    })
+  })
+}

package/src/integration/native-binary.guard.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, test } from 'vitest'
+
+import {
+  formatNativeBootstrapInstructions,
+  inspectNativeSqliteBinary,
+} from '../sqlite-mode/native-binary.js'
+
+describe('native sqlite binary guard', () => {
+  test('better_sqlite3.node is present before native integration tests', () => {
+    const check = inspectNativeSqliteBinary()
+    expect(check.found, formatNativeBootstrapInstructions(check)).toBe(true)
+  })
+})

package/src/integration/native-startup.test.ts

@@ -0,0 +1,44 @@
+import { rmSync } from 'node:fs'
+
+import { afterAll, beforeAll, describe, expect, test } from 'vitest'
+
+import { startZeroLite } from '../index.js'
+
+describe('native sqlite startup integration', { timeout: 120_000 }, () => {
+  let shutdown: (() => Promise<void>) | undefined
+  let zeroPort = 0
+  let dataDir = ''
+
+  beforeAll(async () => {
+    const basePort = 29000 + Math.floor(Math.random() * 1000)
+    dataDir = `.orez-native-startup-test-${Date.now()}`
+
+    const started = await startZeroLite({
+      pgPort: basePort,
+      zeroPort: basePort + 100,
+      dataDir,
+      logLevel: 'warn',
+      skipZeroCache: false,
+      disableWasmSqlite: true,
+    })
+
+    shutdown = started.stop
+    zeroPort = started.zeroPort
+  })
+
+  afterAll(async () => {
+    if (shutdown) await shutdown()
+    if (dataDir) {
+      try {
+        rmSync(dataDir, { recursive: true, force: true })
+      } catch {
+        // ignore cleanup failures in test teardown
+      }
+    }
+  })
+
+  test('zero-cache responds in native mode', async () => {
+    const response = await fetch(`http://127.0.0.1:${zeroPort}/`)
+    expect([200, 404]).toContain(response.status)
+  })
+})