orez 0.1.36 → 0.1.38

package/src/worker/embed-integration.test.ts

@@ -0,0 +1,321 @@
+/**
+ * integration test for zero-cache embedded mode.
+ *
+ * validates that zero-cache can run in-process with SINGLE_PROCESS=1,
+ * connected to PGlite via the TCP proxy. this is the same pipeline as
+ * the full integration test but without child_process.fork().
+ *
+ * test flow:
+ * 1. create PGlite instances (postgres, cvr, cdb)
+ * 2. start TCP proxy
+ * 3. start zero-cache in-process via startZeroCacheEmbed()
+ * 4. connect WebSocket client and verify sync
+ */
+
+import { mkdirSync } from 'node:fs'
+import { resolve } from 'node:path'
+
+import { describe, test, expect, beforeAll, afterAll } from 'vitest'
+import WebSocket from 'ws'
+
+import { getConfig, getConnectionString } from '../config.js'
+import {
+  ensureTablesInPublications,
+  installAllowAllPermissions,
+} from '../integration/test-permissions.js'
+import { startPgProxy } from '../pg-proxy.js'
+import { createPGliteInstances, type PGliteInstances } from '../pglite-manager.js'
+import { installChangeTracking } from '../replication/change-tracker.js'
+import { startZeroCacheEmbed, type ZeroCacheEmbed } from './zero-cache-embed.js'
+
+import type { PGlite } from '@electric-sql/pglite'
+
+const SYNC_PROTOCOL_VERSION = 49
+
+function encodeSecProtocols(
+  initConnectionMessage: unknown,
+  authToken: string | undefined
+): string {
+  const payload = JSON.stringify({ initConnectionMessage, authToken })
+  return encodeURIComponent(Buffer.from(payload, 'utf-8').toString('base64'))
+}
+
+describe('zero-cache embed integration', { timeout: 120000 }, () => {
+  let db: PGlite
+  let instances: PGliteInstances
+  let pgServer: ReturnType<Awaited<ReturnType<typeof startPgProxy>>>
+  let embed: ZeroCacheEmbed
+  let zeroPort: number
+  let pgPort: number
+  let dataDir: string
+
+  beforeAll(async () => {
+    // use random ports to avoid conflicts with other tests
+    pgPort = 24000 + Math.floor(Math.random() * 1000)
+    zeroPort = pgPort + 100
+
+    dataDir = `.orez-embed-test-${Date.now()}`
+
+    const config = getConfig({
+      pgPort,
+      zeroPort,
+      dataDir,
+      logLevel: 'info',
+      useWorkerThreads: false,
+      singleDb: false,
+    })
+
+    mkdirSync(dataDir, { recursive: true })
+
+    // create PGlite instances
+    instances = await createPGliteInstances(config)
+    db = instances.postgres
+
+    // create test table
+    await db.exec(`
+        CREATE TABLE IF NOT EXISTS foo (
+          id TEXT PRIMARY KEY,
+          value TEXT,
+          num INTEGER
+        )
+      `)
+
+    // set up publications
+    const pubName = `orez_zero_public`
+    process.env.ZERO_APP_PUBLICATIONS = pubName
+    await db.exec(`CREATE PUBLICATION "${pubName}"`).catch(() => {})
+    await db
+      .exec(`ALTER PUBLICATION "${pubName}" ADD TABLE "public"."foo"`)
+      .catch(() => {})
+
+    // install change tracking
+    await installChangeTracking(db)
+
+    // install allow-all permissions for test
+    await installAllowAllPermissions(db, ['foo'])
+    await ensureTablesInPublications(db, ['foo'])
+
+    // start TCP proxy
+    pgServer = await startPgProxy(instances, config)
+
+    // start zero-cache in-process
+    const upstreamDb = getConnectionString(config, 'postgres')
+    const cvrDb = getConnectionString(config, 'zero_cvr')
+    const changeDb = getConnectionString(config, 'zero_cdb')
+    const replicaFile = resolve(dataDir, 'zero-replica.db')
+
+    console.log(`[embed-test] starting in-process zero-cache on port ${zeroPort}`)
+    console.log(`[embed-test] upstream: ${upstreamDb}`)
+
+    embed = await startZeroCacheEmbed({
+      pglite: db,
+      upstreamDb,
+      cvrDb,
+      changeDb,
+      replicaFile,
+      port: zeroPort,
+      publications: [pubName],
+      env: {
+        ZERO_LOG_LEVEL: 'info',
+      },
+    })
+
+    console.log(`[embed-test] zero-cache ready on port ${embed.port}`)
+
+    // wait for HTTP health check
+    await waitForZero(zeroPort, 30000)
+    console.log(`[embed-test] health check passed`)
+  }, 120000)
+
+  afterAll(async () => {
+    if (embed) await embed.stop()
+    if (pgServer) pgServer.close()
+    if (instances) {
+      await instances.postgres.close().catch(() => {})
+      await instances.cvr.close().catch(() => {})
+      await instances.cdb.close().catch(() => {})
+    }
+    if (dataDir) {
+      const { rmSync } = await import('node:fs')
+      try {
+        rmSync(dataDir, { recursive: true, force: true })
+      } catch {}
+    }
+  })
+
+  test('zero-cache is ready', () => {
+    expect(embed.ready).toBe(true)
+    expect(embed.port).toBe(zeroPort)
+  })
+
+  test('accepts WebSocket connections', async () => {
+    const cg = `test-cg-${Date.now()}`
+    const cid = `test-client-${Date.now()}`
+    const secProtocol = encodeSecProtocols(
+      [
+        'initConnection',
+        {
+          desiredQueriesPatch: [],
+          clientSchema: {
+            tables: {
+              foo: {
+                columns: {
+                  id: { type: 'string' },
+                  value: { type: 'string' },
+                  num: { type: 'number' },
+                },
+                primaryKey: ['id'],
+              },
+            },
+          },
+        },
+      ],
+      undefined
+    )
+    const ws = new WebSocket(
+      `ws://localhost:${zeroPort}/sync/v${SYNC_PROTOCOL_VERSION}/connect` +
+        `?clientGroupID=${cg}&clientID=${cid}&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`,
+      secProtocol
+    )
+
+    // collect messages — attach listener before open to catch everything
+    const messages: unknown[] = []
+    ws.on('message', (data) => {
+      messages.push(JSON.parse(data.toString()))
+    })
+
+    const connected = new Promise<void>((resolve, reject) => {
+      ws.on('open', resolve)
+      ws.on('error', reject)
+      setTimeout(() => reject(new Error('ws connect timeout')), 10000)
+    })
+
+    await connected
+
+    // wait for messages to arrive
+    const deadline = Date.now() + 10000
+    while (
+      Date.now() < deadline &&
+      !messages.some((m) => Array.isArray(m) && m[0] === 'connected')
+    ) {
+      await new Promise((r) => setTimeout(r, 100))
+    }
+
+    const connectedMsg = messages.find((m) => Array.isArray(m) && m[0] === 'connected')
+    expect(connectedMsg).toMatchObject(['connected', { wsid: 'ws1' }])
+    ws.close()
+  })
+
+  test('live replication: insert triggers poke', async () => {
+    // insert data
+    await db.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
+      'embed-row',
+      'hello-embed',
+      42,
+    ])
+
+    // wait for replication to deliver
+    await waitForReplicationCatchup(db)
+    await new Promise((r) => setTimeout(r, 1000))
+
+    // connect and subscribe
+    const downstream: unknown[] = []
+    const ws = connectAndSubscribe(zeroPort, downstream)
+
+    // wait for poke with our data
+    const deadline = Date.now() + 30000
+    let found = false
+    while (Date.now() < deadline && !found) {
+      await new Promise((r) => setTimeout(r, 500))
+      for (const msg of downstream) {
+        if (Array.isArray(msg) && msg[0] === 'pokePart' && msg[1]?.rowsPatch) {
+          for (const patch of msg[1].rowsPatch) {
+            if (
+              patch.op === 'put' &&
+              patch.tableName === 'foo' &&
+              patch.value?.id === 'embed-row'
+            ) {
+              found = true
+              break
+            }
+          }
+        }
+      }
+    }
+
+    ws.close()
+    expect(found).toBe(true)
+  })
+})
+
+function connectAndSubscribe(port: number, downstream: unknown[]): WebSocket {
+  const cg = `test-cg-${Date.now()}`
+  const cid = `test-client-${Date.now()}`
+  const secProtocol = encodeSecProtocols(
+    [
+      'initConnection',
+      {
+        desiredQueriesPatch: [
+          {
+            op: 'put',
+            hash: 'q1',
+            ast: {
+              table: 'foo',
+              orderBy: [['id', 'asc']],
+            },
+          },
+        ],
+        clientSchema: {
+          tables: {
+            foo: {
+              columns: {
+                id: { type: 'string' },
+                value: { type: 'string' },
+                num: { type: 'number' },
+              },
+              primaryKey: ['id'],
+            },
+          },
+        },
+      },
+    ],
+    undefined
+  )
+  const ws = new WebSocket(
+    `ws://localhost:${port}/sync/v${SYNC_PROTOCOL_VERSION}/connect` +
+      `?clientGroupID=${cg}&clientID=${cid}&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`,
+    secProtocol
+  )
+
+  ws.on('message', (data) => {
+    downstream.push(JSON.parse(data.toString()))
+  })
+
+  return ws
+}
+
+async function waitForReplicationCatchup(
+  pglite: PGlite,
+  timeoutMs = 15000
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    const result = await pglite.query<{ count: string }>(
+      `SELECT count(*)::text as count FROM _orez.changes`
+    )
+    if (Number(result.rows[0]?.count) === 0) return
+    await new Promise((r) => setTimeout(r, 100))
+  }
+}
+
+async function waitForZero(port: number, timeoutMs = 30000) {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    try {
+      const res = await fetch(`http://localhost:${port}/`)
+      if (res.ok || res.status === 404) return
+    } catch {}
+    await new Promise((r) => setTimeout(r, 500))
+  }
+  throw new Error(`zero-cache not ready on port ${port} after ${timeoutMs}ms`)
+}

package/src/worker/index.ts

@@ -0,0 +1,138 @@
+// NOTE THIS IS NOT OREZ NODE THIS IS NOT A GOOD REFERENCE BECAUSE ITS OUR EARLY GUESS AT WHAT COULD WORK
+// DO NOT STUDY THIS, THE OTHER STUFF IN SRC IS WHERE YOU EANT TO LOOK
+
+/**
+ * orez/worker: embeddable PGlite + change tracking.
+ *
+ * runs without Node.js dependencies — works in CF Workers, browsers,
+ * vitest, bun, deno. provides the PGlite database layer with change
+ * tracking and replication encoding that zero-cache needs.
+ *
+ * usage:
+ *   import { createOrezWorker } from 'orez/worker'
+ *
+ *   const orez = await createOrezWorker({
+ *     pgliteOptions: { dataDir: 'memory://' },
+ *   })
+ *   await orez.exec('CREATE TABLE foo (id TEXT PRIMARY KEY, name TEXT)')
+ *   await orez.installChangeTracking()
+ *   await orez.query('INSERT INTO foo VALUES ($1, $2)', ['1', 'bar'])
+ *   const changes = await orez.getChangesSince(0)
+ */
+
+import { Mutex } from '../mutex.js'
+import {
+  installChangeTracking,
+  getChangesSince,
+  getCurrentWatermark,
+  purgeConsumedChanges,
+} from '../replication/change-tracker.js'
+import { handleStartReplication } from '../replication/handler.js'
+
+import type { ChangeRecord } from '../replication/change-tracker.js'
+import type { ReplicationWriter } from '../replication/handler.js'
+import type { OrezWorkerOptions, OrezWorker } from './types.js'
+import type { PGlite, Results } from '@electric-sql/pglite'
+
+export type { OrezWorkerOptions, OrezWorker } from './types.js'
+export type { ChangeRecord } from '../replication/change-tracker.js'
+export type { ReplicationWriter } from '../replication/handler.js'
+
+/**
+ * create an orez worker instance.
+ *
+ * accepts either a pre-created PGlite instance or PGliteOptions to
+ * create one. installs the _orez schema and change tracking infrastructure.
+ */
+export async function createOrezWorker(opts: OrezWorkerOptions): Promise<OrezWorker> {
+  let db: PGlite
+  let ownsInstance: boolean
+
+  if (opts.pglite) {
+    db = opts.pglite
+    ownsInstance = false
+  } else if (opts.pgliteOptions) {
+    // dynamic import so PGlite isn't required at module load time.
+    // this lets the worker module be imported in environments where
+    // PGlite is provided externally (CF Workers with custom WASM).
+    const { PGlite: PGliteCtor } = await import('@electric-sql/pglite')
+    db = new PGliteCtor(opts.pgliteOptions)
+    await db.waitReady
+    ownsInstance = true
+  } else {
+    throw new Error('orez/worker: provide either pglite or pgliteOptions')
+  }
+
+  const mutex = new Mutex()
+
+  // set up publication env if provided (change-tracker reads this)
+  if (opts.publications?.length) {
+    // change-tracker reads ZERO_APP_PUBLICATIONS to decide which tables to track.
+    // in non-Node environments globalThis may not have process.env, so we
+    // set it defensively.
+    if (typeof globalThis !== 'undefined') {
+      ;(globalThis as any).process ??= {}
+      ;(globalThis as any).process.env ??= {}
+      ;(globalThis as any).process.env.ZERO_APP_PUBLICATIONS = opts.publications.join(',')
+    }
+  }
+
+  // install core schema (plpgsql, _orez schema)
+  await db.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
+
+  // install change tracking (creates _orez schema, tables, trigger function)
+  await installChangeTracking(db)
+
+  const worker: OrezWorker = {
+    get db() {
+      return db
+    },
+
+    get mutex() {
+      return mutex
+    },
+
+    get ownsInstance() {
+      return ownsInstance
+    },
+
+    async query<T extends Record<string, unknown> = Record<string, unknown>>(
+      sql: string,
+      params?: unknown[]
+    ): Promise<Results<T>> {
+      return db.query<T>(sql, params)
+    },
+
+    async exec(sql: string): Promise<void> {
+      await db.exec(sql)
+    },
+
+    async installChangeTracking(): Promise<void> {
+      await installChangeTracking(db)
+    },
+
+    async getChangesSince(watermark: number, limit?: number): Promise<ChangeRecord[]> {
+      return getChangesSince(db, watermark, limit)
+    },
+
+    async getCurrentWatermark(): Promise<number> {
+      return getCurrentWatermark(db)
+    },
+
+    async purgeChanges(watermark: number): Promise<number> {
+      return purgeConsumedChanges(db, watermark)
+    },
+
+    async startReplication(writer: ReplicationWriter): Promise<void> {
+      await handleStartReplication('START_REPLICATION', writer, db, mutex)
+    },
+
+    async close(): Promise<void> {
+      if (ownsInstance && !db.closed) {
+        await db.close()
+      }
+    },
+  }
+
+  return worker
+}

package/src/worker/shims/fastify.test.ts

@@ -0,0 +1,255 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+
+import Fastify, { type FastifyShim } from './fastify.js'
+
+describe('Fastify shim', () => {
+  let app: FastifyShim
+  let origGlobal: unknown
+
+  beforeEach(() => {
+    origGlobal = (globalThis as any).__orez_fastify_instance
+    app = Fastify()
+  })
+
+  afterEach(() => {
+    if (origGlobal !== undefined) {
+      ;(globalThis as any).__orez_fastify_instance = origGlobal
+    } else {
+      delete (globalThis as any).__orez_fastify_instance
+    }
+  })
+
+  describe('constructor', () => {
+    it('creates an instance', () => {
+      expect(app).toBeDefined()
+      expect(app.server).toBeDefined()
+    })
+
+    it('registers itself on globalThis', () => {
+      expect((globalThis as any).__orez_fastify_instance).toBe(app)
+    })
+  })
+
+  describe('route registration', () => {
+    it('registers GET routes', async () => {
+      app.get('/', (_req, reply) => reply.send('ok'))
+      const result = await app.inject({ method: 'GET', url: '/' })
+      expect(result.statusCode).toBe(200)
+      expect(result.body).toBe('ok')
+    })
+
+    it('registers POST routes', async () => {
+      app.post('/data', (_req, reply) => reply.send('created'))
+      const result = await app.inject({ method: 'POST', url: '/data' })
+      expect(result.statusCode).toBe(200)
+      expect(result.body).toBe('created')
+    })
+
+    it('registers PUT routes', async () => {
+      app.put('/item', (_req, reply) => {
+        reply.code(200).send('updated')
+      })
+      const result = await app.inject({ method: 'PUT', url: '/item' })
+      expect(result.statusCode).toBe(200)
+      expect(result.body).toBe('updated')
+    })
+
+    it('registers DELETE routes', async () => {
+      app.delete('/item', (_req, reply) => {
+        reply.code(204).send('')
+      })
+      const result = await app.inject({ method: 'DELETE', url: '/item' })
+      expect(result.statusCode).toBe(204)
+    })
+  })
+
+  describe('inject()', () => {
+    it('returns 404 for unregistered routes', async () => {
+      const result = await app.inject({ method: 'GET', url: '/nope' })
+      expect(result.statusCode).toBe(404)
+      expect(result.body).toBe('Not Found')
+    })
+
+    it('passes request headers to handler', async () => {
+      let capturedHeaders: Record<string, string | undefined> = {}
+      app.get('/headers', (req, reply) => {
+        capturedHeaders = req.headers
+        reply.send('ok')
+      })
+      await app.inject({
+        method: 'GET',
+        url: '/headers',
+        headers: { 'x-custom': 'test-value' },
+      })
+      expect(capturedHeaders['x-custom']).toBe('test-value')
+    })
+
+    it('passes query parameters', async () => {
+      let capturedQuery: Record<string, string> = {}
+      app.get('/search', (req, reply) => {
+        capturedQuery = req.query || {}
+        reply.send('ok')
+      })
+      await app.inject({ method: 'GET', url: '/search?q=hello&page=2' })
+      expect(capturedQuery.q).toBe('hello')
+      expect(capturedQuery.page).toBe('2')
+    })
+
+    it('passes parsed JSON body', async () => {
+      let capturedBody: unknown
+      app.post('/json', (req, reply) => {
+        capturedBody = req.body
+        reply.send('ok')
+      })
+      await app.inject({
+        method: 'POST',
+        url: '/json',
+        payload: '{"name":"test"}',
+      })
+      expect(capturedBody).toEqual({ name: 'test' })
+    })
+
+    it('passes raw string body if not JSON', async () => {
+      let capturedBody: unknown
+      app.post('/raw', (req, reply) => {
+        capturedBody = req.body
+        reply.send('ok')
+      })
+      await app.inject({
+        method: 'POST',
+        url: '/raw',
+        payload: 'not json',
+      })
+      expect(capturedBody).toBe('not json')
+    })
+
+    it('reply.code() sets status code', async () => {
+      app.get('/created', (_req, reply) => {
+        reply.code(201).send('done')
+      })
+      const result = await app.inject({ method: 'GET', url: '/created' })
+      expect(result.statusCode).toBe(201)
+    })
+
+    it('reply.header() sets response headers', async () => {
+      app.get('/custom', (_req, reply) => {
+        reply.header('X-Custom', 'value').send('ok')
+      })
+      const result = await app.inject({ method: 'GET', url: '/custom' })
+      expect(result.headers['x-custom']).toBe('value')
+    })
+
+    it('reply.type() sets content-type', async () => {
+      app.get('/typed', (_req, reply) => {
+        reply.type('text/html').send('<h1>hi</h1>')
+      })
+      const result = await app.inject({ method: 'GET', url: '/typed' })
+      expect(result.headers['content-type']).toBe('text/html')
+    })
+
+    it('auto-serializes object responses as JSON', async () => {
+      app.get('/obj', (_req, reply) => {
+        reply.send({ foo: 'bar' })
+      })
+      const result = await app.inject({ method: 'GET', url: '/obj' })
+      expect(result.headers['content-type']).toBe('application/json')
+      expect(JSON.parse(result.body)).toEqual({ foo: 'bar' })
+    })
+
+    it('uses handler return value if reply.send() not called', async () => {
+      app.get('/return', () => 'returned')
+      const result = await app.inject({ method: 'GET', url: '/return' })
+      expect(result.body).toBe('returned')
+    })
+
+    it('returns 500 on handler error', async () => {
+      app.get('/boom', () => {
+        throw new Error('handler error')
+      })
+      const result = await app.inject({ method: 'GET', url: '/boom' })
+      expect(result.statusCode).toBe(500)
+    })
+
+    it('handles async handlers', async () => {
+      app.get('/async', async (_req, reply) => {
+        await new Promise((r) => setTimeout(r, 5))
+        reply.send('async ok')
+      })
+      const result = await app.inject({ method: 'GET', url: '/async' })
+      expect(result.statusCode).toBe(200)
+      expect(result.body).toBe('async ok')
+    })
+
+    it('is case-insensitive on method matching', async () => {
+      app.get('/test', (_req, reply) => reply.send('ok'))
+      const result = await app.inject({ method: 'get', url: '/test' })
+      expect(result.statusCode).toBe(200)
+    })
+  })
+
+  describe('lifecycle', () => {
+    it('listen() resolves to an address string', async () => {
+      const addr = await app.listen({ host: '::', port: 4848 })
+      expect(typeof addr).toBe('string')
+    })
+
+    it('ready() resolves', async () => {
+      await expect(app.ready()).resolves.toBeUndefined()
+    })
+
+    it('close() resolves', async () => {
+      await expect(app.close()).resolves.toBeUndefined()
+    })
+
+    it('register() returns this for chaining', () => {
+      const result = app.register(() => {})
+      expect(result).toBe(app)
+    })
+  })
+
+  describe('FakeHttpServer', () => {
+    it('has address() method', () => {
+      const addr = app.server.address()
+      expect(addr).toHaveProperty('address')
+      expect(addr).toHaveProperty('port')
+    })
+
+    it('supports onMessageType for EventEmitter IPC', () => {
+      let received: unknown = null
+      let receivedHandle: unknown = null
+
+      app.server.onMessageType('handoff', (msg: unknown, handle?: unknown) => {
+        received = msg
+        receivedHandle = handle
+      })
+
+      const payload = { message: { url: '/test' }, head: new Uint8Array(0) }
+      const fakeSocket = { accept: () => {} }
+
+      app.server.emit('message', ['handoff', payload], fakeSocket)
+
+      expect(received).toEqual(payload)
+      expect(receivedHandle).toBe(fakeSocket)
+    })
+
+    it('onMessageType ignores non-matching types', () => {
+      let called = false
+      app.server.onMessageType('handoff', () => {
+        called = true
+      })
+
+      app.server.emit('message', ['ready', { ready: true }])
+      expect(called).toBe(false)
+    })
+
+    it('onMessageType ignores non-array messages', () => {
+      let called = false
+      app.server.onMessageType('handoff', () => {
+        called = true
+      })
+
+      app.server.emit('message', 'not an array')
+      expect(called).toBe(false)
+    })
+  })
+})