orez 0.1.36 → 0.1.38

package/src/worker/shims/ws.ts

@@ -0,0 +1,367 @@
+// NOTE THIS IS NOT OREZ NODE THIS IS NOT A GOOD REFERENCE BECAUSE ITS OUR EARLY GUESS AT WHAT COULD WORK
+// DO NOT STUDY THIS, THE OTHER STUFF IN SRC IS WHERE YOU EANT TO LOOK
+
+/**
+ * ws (websocket) shim for cloudflare workers.
+ *
+ * wraps CF Workers WebSocket (from WebSocketPair) to implement the
+ * ws npm package API that zero-cache uses. enables bundler aliasing
+ * so zero-cache's WebSocket handling works with CF durable WebSockets.
+ *
+ * usage with bundler alias:
+ *   alias: { 'ws': './src/worker/shims/ws.js' }
+ */
+
+import EventEmitter from 'node:events'
+import { Duplex } from 'node:stream'
+
+// -- readyState constants --
+const CONNECTING = 0
+const OPEN = 1
+const CLOSING = 2
+const CLOSED = 3
+
+// -- CF WebSocket interface (minimal) --
+interface CFWebSocket {
+  send(data: string | ArrayBuffer | ArrayBufferView): void
+  close(code?: number, reason?: string): void
+  addEventListener(type: string, handler: (event: any) => void): void
+  removeEventListener(type: string, handler: (event: any) => void): void
+  readyState: number
+  accept?(): void
+}
+
+// -- WebSocket shim --
+// wraps a CF WebSocket to match the ws package WebSocket API
+
+class WebSocket extends EventEmitter {
+  static readonly CONNECTING = CONNECTING
+  static readonly OPEN = OPEN
+  static readonly CLOSING = CLOSING
+  static readonly CLOSED = CLOSED
+
+  readonly CONNECTING = CONNECTING
+  readonly OPEN = OPEN
+  readonly CLOSING = CLOSING
+  readonly CLOSED = CLOSED
+
+  #ws!: CFWebSocket
+  #url: string
+  #listeners = new Map<string, (event: any) => void>()
+
+  constructor(urlOrSocket: string | CFWebSocket, _protocols?: unknown, _opts?: unknown) {
+    super()
+
+    if (typeof urlOrSocket === 'string') {
+      this.#url = urlOrSocket
+      // check for in-process connections (fastify shim uses port 0 or 1)
+      const parsedUrl = new URL(urlOrSocket, 'http://localhost')
+      const isInProcess =
+        parsedUrl.port === '0' ||
+        parsedUrl.port === '1' ||
+        parsedUrl.hostname === 'localhost'
+
+      if (isInProcess) {
+        // in-process: connect via fastify server's handoff mechanism
+        const fastifyInstance = (globalThis as any).__orez_fastify_instance
+        if (fastifyInstance?.server) {
+          // create paired message channels for bidirectional communication
+          // the client-side WS (this) and serverWs are cross-linked so
+          // ping/pong, messages, and close propagate between them
+          const clientSide = this
+          const serverWs: any = {
+            readyState: 1,
+            _listeners: {} as Record<string, Function[]>,
+            send: (data: string | ArrayBuffer) => {
+              // deliver to client side
+              queueMicrotask(() => clientSide.emit('message', data))
+            },
+            close: (code?: number, reason?: string) => {
+              serverWs.readyState = 3
+              queueMicrotask(() => clientSide.emit('close', code || 1000, reason || ''))
+            },
+            ping: () => {
+              // ping from server → deliver 'ping' event to client
+              // (expectPingsForLiveness listens for 'ping', not 'pong')
+              queueMicrotask(() => clientSide.emit('ping'))
+            },
+            addEventListener: (type: string, handler: Function) => {
+              if (!serverWs._listeners[type]) serverWs._listeners[type] = []
+              serverWs._listeners[type].push(handler)
+            },
+            removeEventListener: (type: string, handler: Function) => {
+              const arr = serverWs._listeners[type]
+              if (arr) {
+                const idx = arr.indexOf(handler)
+                if (idx >= 0) arr.splice(idx, 1)
+              }
+            },
+          }
+
+          this.#ws = {
+            accept: () => {},
+            send: (data: string | ArrayBuffer) => {
+              // deliver to server side
+              const handlers = serverWs._listeners['message'] || []
+              for (const h of handlers) h({ data })
+            },
+            close: (code?: number, reason?: string) => {
+              const handlers = serverWs._listeners['close'] || []
+              for (const h of handlers) h({ code, reason })
+            },
+            addEventListener: () => {},
+            removeEventListener: () => {},
+            get readyState() {
+              return 1
+            },
+          } as CFWebSocket
+
+          // emit handoff to fastify server
+          const path = parsedUrl.pathname + parsedUrl.search
+          queueMicrotask(() => {
+            fastifyInstance.server.emit(
+              'message',
+              [
+                'handoff',
+                {
+                  message: { url: path, headers: {}, method: 'GET' },
+                  head: new Uint8Array(0),
+                },
+              ],
+              serverWs
+            )
+            this.emit('open')
+          })
+        } else {
+          // no fastify instance — emit close immediately
+          queueMicrotask(() => this.emit('close', 1006, 'no fastify server'))
+        }
+      } else if (typeof globalThis.WebSocket === 'function') {
+        // real outbound WebSocket for external connections
+        const nativeWs = new globalThis.WebSocket(urlOrSocket) as any
+        this.#ws = {
+          accept: () => {},
+          send: (data: string | ArrayBuffer) => nativeWs.send(data),
+          close: (code?: number, reason?: string) => nativeWs.close(code, reason),
+          addEventListener: (type: string, handler: (event: any) => void) =>
+            nativeWs.addEventListener(type, handler),
+          removeEventListener: (type: string, handler: (event: any) => void) =>
+            nativeWs.removeEventListener(type, handler),
+          get readyState() {
+            return nativeWs.readyState
+          },
+        } as CFWebSocket
+        nativeWs.addEventListener('open', () => this.emit('open'))
+        nativeWs.addEventListener('message', (ev: MessageEvent) =>
+          this.emit('message', ev.data)
+        )
+        nativeWs.addEventListener('close', (ev: CloseEvent) =>
+          this.emit('close', ev.code, ev.reason)
+        )
+        nativeWs.addEventListener('error', (ev: Event) =>
+          this.emit('error', new Error('WebSocket error'))
+        )
+      } else {
+        throw new Error(
+          'ws shim: outbound WebSocket connections not yet supported. ' +
+            'use the CF Workers fetch API for outbound WebSocket.'
+        )
+      }
+    } else {
+      this.#ws = urlOrSocket
+      this.#url = ''
+      this.#setupListeners()
+    }
+  }
+
+  get url(): string {
+    return this.#url
+  }
+
+  get readyState(): number {
+    return this.#ws.readyState
+  }
+
+  send(
+    data: string | Buffer | ArrayBuffer | ArrayBufferView,
+    cb?: (err?: Error) => void
+  ): void {
+    try {
+      if (typeof data === 'string') {
+        this.#ws.send(data)
+      } else if (Buffer.isBuffer(data)) {
+        this.#ws.send(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
+      } else if (data instanceof ArrayBuffer) {
+        this.#ws.send(data)
+      } else if (ArrayBuffer.isView(data)) {
+        this.#ws.send(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
+      } else {
+        this.#ws.send(String(data))
+      }
+      cb?.()
+    } catch (err) {
+      cb?.(err as Error)
+    }
+  }
+
+  close(code?: number, reason?: string): void {
+    try {
+      this.#ws.close(code, reason)
+    } catch {
+      // socket may already be closed
+    }
+  }
+
+  terminate(): void {
+    // real ws.terminate() destroys the socket without a close frame.
+    // CF Workers don't expose raw socket destroy, so close with 1000.
+    this.close(1000)
+  }
+
+  ping(_data?: unknown, _mask?: boolean, _cb?: () => void): void {
+    // forward ping to underlying socket if it supports it (in-process pairs)
+    if (typeof (this.#ws as any).ping === 'function') {
+      ;(this.#ws as any).ping()
+    }
+    // also emit pong locally (CF WebSockets handle ping/pong at platform level)
+    this.emit('pong')
+  }
+
+  // standard EventTarget-style addEventListener (used by Connection)
+  addEventListener(type: string, handler: (event: any) => void): void {
+    // wrap to emit EventEmitter-style
+    this.on(type, handler)
+  }
+
+  removeEventListener(type: string, handler: (event: any) => void): void {
+    this.off(type, handler)
+  }
+
+  #setupListeners(): void {
+    // match ws npm package event signatures:
+    //   message: (data: Buffer|string, isBinary: boolean)
+    //   close: (code: number, reason: string)
+    //   error: (err: Error)
+    const onMessage = (event: any) => {
+      const data = event.data
+      this.emit('message', data, typeof data !== 'string')
+    }
+    const onClose = (event: any) => {
+      this.emit('close', event.code ?? 1000, event.reason ?? '')
+    }
+    const onError = (event: any) => {
+      this.emit('error', event.error ?? new Error(event.message ?? 'WebSocket error'))
+    }
+    const onOpen = () => {
+      this.emit('open')
+    }
+
+    this.#ws.addEventListener('message', onMessage)
+    this.#ws.addEventListener('close', onClose)
+    this.#ws.addEventListener('error', onError)
+    this.#ws.addEventListener('open', onOpen)
+
+    this.#listeners.set('message', onMessage)
+    this.#listeners.set('close', onClose)
+    this.#listeners.set('error', onError)
+    this.#listeners.set('open', onOpen)
+  }
+}
+
+// -- WebSocketServer shim --
+// zero-cache uses WebSocketServer with { noServer: true } for handleUpgrade
+
+class WebSocketServer extends EventEmitter {
+  constructor(_opts?: { noServer?: boolean }) {
+    super()
+  }
+
+  close(cb?: (err?: Error) => void): void {
+    // no-op — browser embed has no real server to close
+    cb?.()
+  }
+
+  /**
+   * handle a WebSocket upgrade. on CF Workers the upgrade is already done
+   * (WebSocketPair), so this just wraps the CF WebSocket in our shim.
+   *
+   * @param message - the HTTP request (IncomingMessage-like object)
+   * @param socket - the underlying socket (CF WebSocket on CF Workers)
+   * @param head - upgrade head buffer
+   * @param callback - receives the wrapped WebSocket
+   */
+  handleUpgrade(
+    _message: unknown,
+    socket: CFWebSocket | unknown,
+    _head: unknown,
+    callback: (ws: WebSocket) => void
+  ): void {
+    // wrap the CF WebSocket in our shim
+    const ws = new WebSocket(socket as CFWebSocket)
+    callback(ws)
+  }
+}
+
+// -- createWebSocketStream --
+// creates a Node.js Duplex stream from a WebSocket.
+// used by zero-cache's Connection class for streaming messages.
+
+function createWebSocketStream(
+  ws: WebSocket,
+  _opts?: { decodeStrings?: boolean }
+): Duplex {
+  const duplex = new Duplex({
+    objectMode: false,
+    decodeStrings: false,
+
+    read() {
+      // data is pushed from ws message events
+    },
+
+    write(
+      chunk: Buffer | string,
+      _encoding: string,
+      callback: (err?: Error | null) => void
+    ) {
+      try {
+        ws.send(typeof chunk === 'string' ? chunk : chunk.toString(), callback)
+      } catch (err) {
+        callback(err as Error)
+      }
+    },
+
+    destroy(err: Error | null, callback: (err?: Error | null) => void) {
+      ws.close()
+      callback(err)
+    },
+  })
+
+  // pipe ws messages into the readable side
+  ws.on('message', (event: any) => {
+    const data = event?.data ?? event
+    if (typeof data === 'string') {
+      duplex.push(data)
+    } else if (data instanceof ArrayBuffer) {
+      duplex.push(Buffer.from(data))
+    } else if (ArrayBuffer.isView(data)) {
+      duplex.push(Buffer.from(data.buffer, data.byteOffset, data.byteLength))
+    } else {
+      duplex.push(String(data))
+    }
+  })
+
+  ws.on('close', () => {
+    duplex.push(null) // signal end of readable
+    duplex.destroy()
+  })
+
+  ws.on('error', (event: any) => {
+    const err = event?.error ?? new Error(event?.message ?? 'WebSocket error')
+    duplex.destroy(err)
+  })
+
+  return duplex
+}
+
+export default WebSocket
+export { WebSocket, WebSocketServer, createWebSocketStream }

package/src/worker/types.ts

@@ -0,0 +1,75 @@
+/**
+ * orez/worker types.
+ *
+ * interfaces for the embeddable orez worker that runs PGlite + change
+ * tracking without Node.js dependencies. designed for CF Workers/DO
+ * but usable in any JS runtime (browser, vitest, bun, deno).
+ */
+
+import type { Mutex } from '../mutex.js'
+import type { ChangeRecord } from '../replication/change-tracker.js'
+import type { ReplicationWriter } from '../replication/handler.js'
+import type { PGlite, PGliteOptions, Results } from '@electric-sql/pglite'
+
+/** options for creating an orez worker */
+export interface OrezWorkerOptions {
+  /**
+   * pre-created PGlite instance. when provided, orez wraps it
+   * without managing its lifecycle (caller is responsible for closing).
+   */
+  pglite?: PGlite
+
+  /**
+   * PGlite constructor options. used when `pglite` is not provided.
+   * in CF Workers, pass wasmModule/fsBundle/loadDataDir here.
+   */
+  pgliteOptions?: PGliteOptions
+
+  /** publication names to track. defaults to all public tables. */
+  publications?: string[]
+
+  /** log level (default: 'warn') */
+  logLevel?: 'error' | 'warn' | 'info' | 'debug'
+}
+
+/** the orez worker instance */
+export interface OrezWorker {
+  /** the underlying PGlite instance */
+  readonly db: PGlite
+
+  /** mutex for serializing PGlite access */
+  readonly mutex: Mutex
+
+  /** run a parameterized query */
+  query<T extends Record<string, unknown> = Record<string, unknown>>(
+    sql: string,
+    params?: unknown[]
+  ): Promise<Results<T>>
+
+  /** execute raw SQL (DDL, multi-statement) */
+  exec(sql: string): Promise<void>
+
+  /** install/reinstall change tracking triggers on all published tables */
+  installChangeTracking(): Promise<void>
+
+  /** get changes since a watermark */
+  getChangesSince(watermark: number, limit?: number): Promise<ChangeRecord[]>
+
+  /** get current watermark value */
+  getCurrentWatermark(): Promise<number>
+
+  /** purge consumed changes up to watermark */
+  purgeChanges(watermark: number): Promise<number>
+
+  /**
+   * start streaming replication to a writer.
+   * runs until the writer is closed or the worker is shut down.
+   */
+  startReplication(writer: ReplicationWriter): Promise<void>
+
+  /** whether this worker owns the PGlite instance (manages its lifecycle) */
+  readonly ownsInstance: boolean
+
+  /** close the worker. if ownsInstance, also closes PGlite. */
+  close(): Promise<void>
+}

package/src/worker/worker-integration.test.ts

@@ -0,0 +1,223 @@
+/**
+ * integration test for orez/worker.
+ *
+ * tests the full pipeline available without zero-cache:
+ * PGlite → change tracking → replication encoding → InProcessWriter
+ *
+ * mirrors the existing integration test patterns but uses the worker
+ * API instead of startZeroLite(). validates that the worker entry
+ * point produces the same replication stream that zero-cache expects.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+
+import { InProcessWriter } from '../replication/handler.js'
+import { resetReplicationState, signalReplicationChange } from '../replication/handler.js'
+import { createOrezWorker } from './index'
+
+import type { OrezWorker } from './types'
+
+// extract pgoutput message types from CopyData(XLogData(...)) buffers
+function extractPayloadTypes(buf: Uint8Array): number[] {
+  const types: number[] = []
+  let pos = 0
+  while (pos < buf.length) {
+    if (buf[pos] !== 0x64) break // CopyData
+    const view = new DataView(buf.buffer, buf.byteOffset + pos + 1)
+    const len = view.getInt32(0)
+    // XLogData starts at pos+5, payload type at pos+5+1+8+8+8 = pos+30
+    if (pos + 30 < buf.length) {
+      types.push(buf[pos + 30])
+    }
+    pos += 1 + len
+  }
+  return types
+}
+
+describe('orez/worker integration', { timeout: 30000 }, () => {
+  let worker: OrezWorker
+
+  beforeEach(async () => {
+    resetReplicationState()
+    worker = await createOrezWorker({
+      pgliteOptions: { dataDir: 'memory://' },
+    })
+    // create test table
+    await worker.exec(`
+      CREATE TABLE public.foo (
+        id TEXT PRIMARY KEY,
+        value TEXT,
+        num INTEGER
+      )
+    `)
+    // reinstall triggers after table creation
+    await worker.installChangeTracking()
+  })
+
+  afterEach(async () => {
+    resetReplicationState()
+    await worker.close()
+  })
+
+  it('change tracking captures insert/update/delete cycle', async () => {
+    await worker.query('INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)', [
+      'row1',
+      'hello',
+      42,
+    ])
+    await worker.query('UPDATE foo SET value = $1 WHERE id = $2', ['updated', 'row1'])
+    await worker.query('DELETE FROM foo WHERE id = $1', ['row1'])
+
+    const changes = await worker.getChangesSince(0)
+    expect(changes).toHaveLength(3)
+    expect(changes[0].op).toBe('INSERT')
+    expect(changes[0].row_data).toMatchObject({ id: 'row1', value: 'hello', num: 42 })
+    expect(changes[1].op).toBe('UPDATE')
+    expect(changes[1].row_data).toMatchObject({ id: 'row1', value: 'updated' })
+    expect(changes[1].old_data).toMatchObject({ id: 'row1', value: 'hello' })
+    expect(changes[2].op).toBe('DELETE')
+    expect(changes[2].old_data).toMatchObject({ id: 'row1', value: 'updated' })
+  })
+
+  it('InProcessWriter receives pgoutput stream from replication', async () => {
+    const received: Uint8Array[] = []
+    const writer = new InProcessWriter((data) => received.push(new Uint8Array(data)))
+
+    // start replication in background
+    const replPromise = worker.startReplication(writer)
+
+    // wait for handler to set up
+    await new Promise((r) => setTimeout(r, 200))
+
+    // insert data
+    await worker.query('INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)', [
+      'streamed-1',
+      'live',
+      99,
+    ])
+    signalReplicationChange()
+
+    // wait for replication to deliver
+    await new Promise((r) => setTimeout(r, 1500))
+
+    // close writer to stop replication
+    writer.close()
+    await replPromise.catch(() => {}) // may reject on close
+
+    // should have received pgoutput messages
+    expect(received.length).toBeGreaterThan(0)
+
+    // extract message types from all received buffers
+    const allTypes = received.flatMap(extractPayloadTypes)
+
+    // first message is CopyBothResponse (0x57), then pgoutput messages
+    // check we got the core pgoutput types
+    expect(allTypes).toContain(0x42) // BEGIN
+    expect(allTypes).toContain(0x52) // RELATION
+    expect(allTypes).toContain(0x49) // INSERT
+    expect(allTypes).toContain(0x43) // COMMIT
+  })
+
+  it('watermarks advance monotonically', async () => {
+    const watermarks: number[] = []
+
+    for (let i = 0; i < 5; i++) {
+      await worker.query('INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)', [
+        `wm-${i}`,
+        `val-${i}`,
+        i,
+      ])
+      watermarks.push(await worker.getCurrentWatermark())
+    }
+
+    // each watermark should be strictly greater than the previous
+    for (let i = 1; i < watermarks.length; i++) {
+      expect(watermarks[i]).toBeGreaterThan(watermarks[i - 1])
+    }
+
+    // getChangesSince with earlier watermark returns only newer changes
+    const midpoint = watermarks[2]
+    const laterChanges = await worker.getChangesSince(midpoint)
+    expect(laterChanges).toHaveLength(2)
+    expect(laterChanges[0].row_data).toMatchObject({ id: 'wm-3' })
+    expect(laterChanges[1].row_data).toMatchObject({ id: 'wm-4' })
+  })
+
+  it('purge removes consumed changes', async () => {
+    for (let i = 0; i < 10; i++) {
+      await worker.query('INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)', [
+        `purge-${i}`,
+        'x',
+        i,
+      ])
+    }
+
+    const allChanges = await worker.getChangesSince(0)
+    expect(allChanges).toHaveLength(10)
+
+    // purge first 7
+    const purged = await worker.purgeChanges(allChanges[6].watermark)
+    expect(purged).toBe(7)
+
+    // only 3 remain
+    const remaining = await worker.getChangesSince(0)
+    expect(remaining).toHaveLength(3)
+  })
+
+  it('multiple tables each get tracked', async () => {
+    await worker.exec(`
+      CREATE TABLE public.bar (
+        id TEXT PRIMARY KEY,
+        foo_id TEXT
+      )
+    `)
+    await worker.installChangeTracking()
+
+    await worker.query('INSERT INTO foo VALUES ($1, $2, $3)', ['f1', 'a', 1])
+    await worker.query('INSERT INTO bar VALUES ($1, $2)', ['b1', 'f1'])
+
+    const changes = await worker.getChangesSince(0)
+    expect(changes).toHaveLength(2)
+    expect(changes[0].table_name).toBe('public.foo')
+    expect(changes[1].table_name).toBe('public.bar')
+  })
+
+  it('replication stream contains correct data for multiple operations', async () => {
+    const received: Uint8Array[] = []
+    const writer = new InProcessWriter((data) => received.push(new Uint8Array(data)))
+
+    const replPromise = worker.startReplication(writer)
+    await new Promise((r) => setTimeout(r, 200))
+
+    // insert
+    await worker.query('INSERT INTO foo VALUES ($1, $2, $3)', ['r1', 'initial', 1])
+    signalReplicationChange()
+    await new Promise((r) => setTimeout(r, 800))
+
+    // update
+    await worker.query('UPDATE foo SET value = $1 WHERE id = $2', ['modified', 'r1'])
+    signalReplicationChange()
+    await new Promise((r) => setTimeout(r, 800))
+
+    // delete
+    await worker.query('DELETE FROM foo WHERE id = $1', ['r1'])
+    signalReplicationChange()
+    await new Promise((r) => setTimeout(r, 800))
+
+    writer.close()
+    await replPromise.catch(() => {})
+
+    const allTypes = received.flatMap(extractPayloadTypes)
+
+    // should have all three operation types
+    expect(allTypes).toContain(0x49) // INSERT
+    expect(allTypes).toContain(0x55) // UPDATE
+    expect(allTypes).toContain(0x44) // DELETE
+
+    // should have BEGIN/COMMIT pairs
+    const begins = allTypes.filter((t) => t === 0x42).length
+    const commits = allTypes.filter((t) => t === 0x43).length
+    expect(begins).toBe(commits)
+    expect(begins).toBeGreaterThanOrEqual(1)
+  })
+})