orez 0.0.48 → 0.0.49

package/src/integration/integration.test.ts

@@ -1,82 +1,19 @@
 /**
- * integration test for zero-cache sync pipeline.
+ * integration test adapted from zero-cache's integration.pg.test.ts
  *
- * validates: pglite → change tracking → replication protocol →
- * zero-cache → websocket poke messages to clients.
+ * validates the full sync pipeline: pglite → change tracking → replication
+ * protocol → zero-cache → websocket poke messages to clients.
  *
- * uses orez's startZeroLite() with beforeZero to set up tables
- * before zero-cache starts its initial sync. deploys ANYONE_CAN
- * permissions after zero-cache creates its schema tables.
+ * uses orez's startZeroLite() instead of real postgres + manual zero-cache.
  */
 
-import { describe, expect, test, beforeAll, afterAll } from 'vitest'
+import { describe, expect, test, beforeAll, afterAll, beforeEach } from 'vitest'
 import WebSocket from 'ws'
 
 import { startZeroLite } from '../index.js'
 
 import type { PGlite } from '@electric-sql/pglite'
 
-// encode initConnectionMessage + authToken for sec-websocket-protocol header
-// mirrors @rocicorp/zero's encodeSecProtocols
-function encodeSecProtocol(
-  initConnectionMessage: unknown,
-  authToken: string = ''
-): string {
-  const protocols = { initConnectionMessage, authToken }
-  const bytes = new TextEncoder().encode(JSON.stringify(protocols))
-  const s = Array.from(bytes, (byte: number) => String.fromCharCode(byte)).join('')
-  return encodeURIComponent(btoa(s))
-}
-
-// zero v0.25 requires clientSchema for new client groups
-const clientSchema = {
-  tables: {
-    foo: {
-      columns: {
-        id: { type: 'string' },
-        value: { type: 'string' },
-        num: { type: 'number' },
-      },
-      primaryKey: ['id'],
-    },
-    bar: {
-      columns: {
-        id: { type: 'string' },
-        foo_id: { type: 'string' },
-      },
-      primaryKey: ['id'],
-    },
-  },
-}
-
-// ANYONE_CAN permissions — empty AND condition = always true
-const anyoneCanPermissions = JSON.stringify({
-  tables: {
-    foo: {
-      row: {
-        select: [['allow', { type: 'and', conditions: [] }]],
-        insert: [['allow', { type: 'and', conditions: [] }]],
-        update: {
-          preMutation: [['allow', { type: 'and', conditions: [] }]],
-          postMutation: [['allow', { type: 'and', conditions: [] }]],
-        },
-        delete: [['allow', { type: 'and', conditions: [] }]],
-      },
-    },
-    bar: {
-      row: {
-        select: [['allow', { type: 'and', conditions: [] }]],
-        insert: [['allow', { type: 'and', conditions: [] }]],
-        update: {
-          preMutation: [['allow', { type: 'and', conditions: [] }]],
-          postMutation: [['allow', { type: 'and', conditions: [] }]],
-        },
-        delete: [['allow', { type: 'and', conditions: [] }]],
-      },
-    },
-  },
-})
-
 // simple async queue for collecting websocket messages
 class Queue<T> {
   private items: T[] = []
@@ -118,61 +55,49 @@ class Queue<T> {
 describe('orez integration', { timeout: 120000 }, () => {
   let db: PGlite
   let zeroPort: number
+  let pgPort: number
   let shutdown: () => Promise<void>
   let dataDir: string
 
   beforeAll(async () => {
     const testPgPort = 23000 + Math.floor(Math.random() * 1000)
-    const testZeroPort = testPgPort + 1000
+    const testZeroPort = testPgPort + 100
 
     dataDir = `.orez-integration-test-${Date.now()}`
+    console.log(`[test] starting orez on pg:${testPgPort} zero:${testZeroPort}`)
     const result = await startZeroLite({
       pgPort: testPgPort,
       zeroPort: testZeroPort,
       dataDir,
       logLevel: 'info',
       skipZeroCache: false,
-      beforeZero: async (pglite) => {
-        await pglite.exec(`
-          CREATE TABLE IF NOT EXISTS foo (
-            id TEXT PRIMARY KEY,
-            value TEXT,
-            num INTEGER
-          );
-          CREATE TABLE IF NOT EXISTS bar (
-            id TEXT PRIMARY KEY,
-            foo_id TEXT
-          );
-        `)
-        // insert test data before zero-cache starts so initial sync includes it
-        await pglite.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
-          'seed1',
-          'hello',
-          42,
-        ])
-        await pglite.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
-          'seed2',
-          'world',
-          99,
-        ])
-      },
     })
 
     db = result.db
     zeroPort = result.zeroPort
+    pgPort = result.pgPort
     shutdown = result.stop
 
-    await waitForZero(zeroPort, 90000)
+    console.log(`[test] orez started, creating tables`)
 
-    // deploy ANYONE_CAN permissions after zero-cache creates its schema tables
-    await db.query(
-      `INSERT INTO zero.permissions (lock, hash, permissions)
-       VALUES (true, $1, $2)
-       ON CONFLICT (lock) DO UPDATE SET hash = $1, permissions = $2`,
-      ['integration-test', anyoneCanPermissions]
-    )
-    // wait for permissions to replicate to zero-cache's sqlite replica
-    await new Promise((r) => setTimeout(r, 3000))
+    // create test tables
+    await db.exec(`
+      CREATE TABLE IF NOT EXISTS foo (
+        id TEXT PRIMARY KEY,
+        value TEXT,
+        num INTEGER
+      );
+
+      CREATE TABLE IF NOT EXISTS bar (
+        id TEXT PRIMARY KEY,
+        foo_id TEXT
+      );
+    `)
+
+    console.log(`[test] tables created, waiting for zero-cache`)
+    // wait for zero-cache to be ready
+    await waitForZero(zeroPort, 90000)
+    console.log(`[test] zero-cache ready`)
   }, 120000)
 
   afterAll(async () => {
@@ -185,27 +110,96 @@ describe('orez integration', { timeout: 120000 }, () => {
     }
   })
 
+  beforeEach(async () => {
+    // clean tables between tests
+    await db.exec(`DELETE FROM foo; DELETE FROM bar;`)
+  })
+
+  test('zero-cache starts and accepts websocket connections', async () => {
+    const ws = new WebSocket(
+      `ws://localhost:${zeroPort}/sync/v4/connect` +
+        `?clientGroupID=test-cg&clientID=test-client&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`
+    )
+
+    const connected = new Promise<void>((resolve, reject) => {
+      ws.on('open', resolve)
+      ws.on('error', reject)
+      setTimeout(() => reject(new Error('ws connect timeout')), 5000)
+    })
+
+    await connected
+
+    const firstMessage = await new Promise<unknown>((resolve) => {
+      ws.on('message', (data) => {
+        resolve(JSON.parse(data.toString()))
+      })
+    })
+
+    expect(firstMessage).toMatchObject(['connected', { wsid: 'ws1' }])
+
+    ws.close()
+  })
+
   test('initial sync delivers existing rows via poke', async () => {
+    // insert data before connecting
+    await db.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
+      'row1',
+      'hello',
+      42,
+    ])
+
     const downstream = new Queue<unknown>()
     const ws = connectAndSubscribe(zeroPort, downstream, {
       table: 'foo',
       orderBy: [['id', 'asc']],
     })
 
-    const poke = await waitForPokePart(downstream, 15000)
-    const ids = poke.rowsPatch
-      .filter((r: any) => r.op === 'put' && r.tableName === 'foo')
-      .map((r: any) => r.value.id)
-      .sort()
+    // drain until we get a pokePart with rowsPatch containing our data
+    const poke = await waitForPokePart(downstream, 30000)
+    expect(poke.rowsPatch).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          op: 'put',
+          tableName: 'foo',
+          value: expect.objectContaining({
+            id: 'row1',
+            value: 'hello',
+          }),
+        }),
+      ])
+    )
 
-    expect(ids).toContain('seed1')
-    expect(ids).toContain('seed2')
+    ws.close()
+  })
+
+  test('live replication: insert triggers poke', async () => {
+    const downstream = new Queue<unknown>()
+    const ws = connectAndSubscribe(zeroPort, downstream, {
+      table: 'foo',
+      orderBy: [['id', 'asc']],
+    })
+
+    // drain initial connection + sync pokes
+    await drainInitialPokes(downstream)
+
+    // now insert data - this should trigger a replication poke
+    await db.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
+      'live-row',
+      'live-value',
+      99,
+    ])
+
+    // wait for the replication poke
+    const poke = await waitForPokePart(downstream, 30000)
     expect(poke.rowsPatch).toEqual(
       expect.arrayContaining([
         expect.objectContaining({
           op: 'put',
           tableName: 'foo',
-          value: expect.objectContaining({ id: 'seed1', value: 'hello' }),
+          value: expect.objectContaining({
+            id: 'live-row',
+            value: 'live-value',
+          }),
         }),
       ])
     )
@@ -213,57 +207,165 @@ describe('orez integration', { timeout: 120000 }, () => {
     ws.close()
   })
 
-  test('initial sync delivers correct values for all columns', async () => {
+  test('live replication: update triggers poke', async () => {
+    // insert initial data
+    await db.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
+      'upd-row',
+      'original',
+      1,
+    ])
+
     const downstream = new Queue<unknown>()
     const ws = connectAndSubscribe(zeroPort, downstream, {
       table: 'foo',
       orderBy: [['id', 'asc']],
     })
 
-    const poke = await waitForPokePart(downstream, 15000)
-    const row = poke.rowsPatch.find(
-      (r: any) => r.op === 'put' && r.tableName === 'foo' && r.value.id === 'seed2'
+    await drainInitialPokes(downstream)
+
+    // update the row
+    await db.query(`UPDATE foo SET value = $1, num = $2 WHERE id = $3`, [
+      'updated',
+      2,
+      'upd-row',
+    ])
+
+    const poke = await waitForPokePart(downstream, 30000)
+    expect(poke.rowsPatch).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          op: 'put',
+          tableName: 'foo',
+          value: expect.objectContaining({
+            id: 'upd-row',
+            value: 'updated',
+          }),
+        }),
+      ])
     )
 
-    expect(row).toBeDefined()
-    expect(row.value).toEqual({ id: 'seed2', value: 'world', num: 99 })
+    ws.close()
+  })
+
+  test('live replication: delete triggers poke', async () => {
+    await db.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
+      'del-row',
+      'to-delete',
+      1,
+    ])
+
+    const downstream = new Queue<unknown>()
+    const ws = connectAndSubscribe(zeroPort, downstream, {
+      table: 'foo',
+      orderBy: [['id', 'asc']],
+    })
+
+    await drainInitialPokes(downstream)
+
+    // delete the row
+    await db.query(`DELETE FROM foo WHERE id = $1`, ['del-row'])
+
+    const poke = await waitForPokePart(downstream, 30000)
+    expect(poke.rowsPatch).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          op: 'del',
+          tableName: 'foo',
+        }),
+      ])
+    )
 
     ws.close()
   })
 
-  // --- helpers ---
+  test('concurrent inserts all replicate', async () => {
+    const downstream = new Queue<unknown>()
+    const ws = connectAndSubscribe(zeroPort, downstream, {
+      table: 'foo',
+      orderBy: [['id', 'asc']],
+    })
 
-  let clientCounter = 0
+    await drainInitialPokes(downstream)
+
+    // insert 5 rows concurrently
+    await Promise.all(
+      Array.from({ length: 5 }, (_, i) =>
+        db.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
+          `concurrent-${i}`,
+          `value-${i}`,
+          i,
+        ])
+      )
+    )
+
+    // collect all poke parts within a window
+    const allRows = await collectPokeRows(downstream, 30000)
+    const ids = allRows
+      .filter((r: any) => r.op === 'put' && r.tableName === 'foo')
+      .map((r: any) => r.value.id)
+      .sort()
+
+    expect(ids).toEqual([
+      'concurrent-0',
+      'concurrent-1',
+      'concurrent-2',
+      'concurrent-3',
+      'concurrent-4',
+    ])
+
+    ws.close()
+  })
+
+  // --- helpers ---
 
   function connectAndSubscribe(
     port: number,
     downstream: Queue<unknown>,
     query: Record<string, unknown>
   ): WebSocket {
-    const cid = `test-${++clientCounter}-${Date.now()}`
-    const wsid = `ws-${clientCounter}-${Date.now()}`
-    const initMsg = [
-      'initConnection',
-      {
-        desiredQueriesPatch: [{ op: 'put', hash: 'q1', ast: query }],
-        clientSchema,
-      },
-    ]
-    const secProtocol = encodeSecProtocol(initMsg)
-
     const ws = new WebSocket(
-      `ws://localhost:${port}/sync/v45/connect` +
-        `?clientGroupID=${cid}&clientID=${cid}-c&wsid=${wsid}&ts=${Date.now()}&lmid=0`,
-      [secProtocol]
+      `ws://localhost:${port}/sync/v4/connect` +
+        `?clientGroupID=test-cg-${Date.now()}&clientID=test-client&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`
     )
 
     ws.on('message', (data) => {
       downstream.enqueue(JSON.parse(data.toString()))
     })
 
+    ws.on('open', () => {
+      ws.send(
+        JSON.stringify([
+          'initConnection',
+          {
+            desiredQueriesPatch: [{ op: 'put', hash: 'q1', ast: query }],
+          },
+        ])
+      )
+    })
+
     return ws
   }
 
+  async function drainInitialPokes(downstream: Queue<unknown>) {
+    // drain messages until we've seen the initial data sync complete
+    // pattern: connected → pokeStart/End → pokeStart/pokePart(queries)/pokeEnd → pokeStart/pokePart(data)/pokeEnd
+    let settled = false
+    const timeout = Date.now() + 30000
+
+    while (!settled && Date.now() < timeout) {
+      const msg = (await downstream.dequeue('timeout' as any, 3000)) as any
+      if (msg === 'timeout') {
+        settled = true
+      } else if (Array.isArray(msg) && msg[0] === 'pokeEnd') {
+        // after a pokeEnd, check if another poke comes quickly
+        const next = (await downstream.dequeue('timeout' as any, 2000)) as any
+        if (next === 'timeout') {
+          settled = true
+        }
+      }
+    }
+  }
+
   async function waitForPokePart(
     downstream: Queue<unknown>,
     timeoutMs = 10000
@@ -279,6 +381,35 @@ describe('orez integration', { timeout: 120000 }, () => {
     }
     throw new Error('timed out waiting for pokePart')
   }
+
+  async function collectPokeRows(
+    downstream: Queue<unknown>,
+    windowMs = 5000
+  ): Promise<any[]> {
+    const rows: any[] = []
+    const deadline = Date.now() + windowMs
+    // first wait for the pokePart with data
+    while (Date.now() < deadline) {
+      const remaining = Math.max(1000, deadline - Date.now())
+      const msg = (await downstream.dequeue('timeout' as any, remaining)) as any
+      if (msg === 'timeout') break
+      if (Array.isArray(msg) && msg[0] === 'pokePart' && msg[1]?.rowsPatch) {
+        rows.push(...msg[1].rowsPatch)
+        // check if more poke parts come quickly
+        const more = (await downstream.dequeue('timeout' as any, 2000)) as any
+        if (
+          more !== 'timeout' &&
+          Array.isArray(more) &&
+          more[0] === 'pokePart' &&
+          more[1]?.rowsPatch
+        ) {
+          rows.push(...more[1].rowsPatch)
+        }
+        break
+      }
+    }
+    return rows
+  }
 })
 
 async function waitForZero(port: number, timeoutMs = 30000) {

package/src/log.ts

@@ -25,17 +25,6 @@ export function setLogLevel(level: LogLevel) {
   currentLevel = level
 }
 
-type LogListener = (source: string, level: LogLevel, msg: string) => void
-const listeners: LogListener[] = []
-
-export function addLogListener(fn: LogListener) {
-  listeners.push(fn)
-  return () => {
-    const idx = listeners.indexOf(fn)
-    if (idx !== -1) listeners.splice(idx, 1)
-  }
-}
-
 function prefix(label: string, color: string): string {
   return `${BOLD}${color}[${label}]${RESET}`
 }
@@ -48,12 +37,6 @@ export function port(n: number, color: keyof typeof COLORS): string {
 function makeLogger(label: string, color: string, level: LogLevel = 'info') {
   const p = prefix(label, color)
   return (...args: unknown[]) => {
-    // always notify listeners (they capture everything for admin ui)
-    if (listeners.length > 0) {
-      const msg = args.map((a) => (typeof a === 'string' ? a : String(a))).join(' ')
-      for (const fn of listeners) fn(label, level, msg)
-    }
-    // only print to terminal if level passes filter
     if (LEVEL_PRIORITY[level] <= LEVEL_PRIORITY[currentLevel]) {
       console.info(p, ...args)
     }
@@ -64,15 +47,8 @@ export const log = {
   orez: makeLogger('orez', COLORS.cyan, 'warn'),
   pglite: makeLogger('pglite', COLORS.green, 'warn'),
   proxy: makeLogger('pg-proxy', COLORS.yellow, 'warn'),
-  zero: makeLogger('zero', COLORS.magenta, 'warn'),
+  zero: makeLogger('zero-cache', COLORS.magenta, 'warn'),
   s3: makeLogger('orez/s3', COLORS.blue, 'warn'),
-  info: {
-    orez: makeLogger('orez', COLORS.cyan, 'info'),
-    pglite: makeLogger('pglite', COLORS.green, 'info'),
-    proxy: makeLogger('pg-proxy', COLORS.yellow, 'info'),
-    zero: makeLogger('zero', COLORS.magenta, 'info'),
-    s3: makeLogger('orez/s3', COLORS.blue, 'info'),
-  },
   debug: {
     orez: makeLogger('orez', COLORS.cyan, 'debug'),
     pglite: makeLogger('pglite', COLORS.green, 'debug'),

package/src/mutex.ts

@@ -1,8 +1,4 @@
-// simple mutex for serializing pglite access.
-// uses setImmediate/setTimeout between releases to prevent event loop
-// starvation when multiple connections queue up — without this, releasing
-// the mutex resolves the next waiter as a microtask, which causes a chain
-// of synchronous pglite executions that blocks all I/O processing.
+// simple mutex for serializing pglite access
 export class Mutex {
   private locked = false
   private queue: Array<() => void> = []
@@ -20,13 +16,7 @@ export class Mutex {
   release(): void {
     const next = this.queue.shift()
     if (next) {
-      // yield to event loop so I/O events (socket reads/writes) are processed
-      // before the next waiter acquires the mutex
-      if (typeof setImmediate !== 'undefined') {
-        setImmediate(next)
-      } else {
-        setTimeout(next, 0)
-      }
+      next()
     } else {
       this.locked = false
     }