orez 0.1.43 → 0.1.45

package/src/pglite-web-worker.ts

@@ -0,0 +1,152 @@
+/**
+ * PGlite Web Worker — browser equivalent of pglite-worker-thread.ts.
+ *
+ * runs a single PGlite instance in a Web Worker. receives commands via
+ * postMessage, executes on PGlite, sends results back. ArrayBuffers
+ * are transferred (not copied) for execProtocolRaw.
+ *
+ * message protocol (same as pglite-worker-thread.ts):
+ *   init: { type: 'init', dataDir, name, withExtensions, pgliteOptions }
+ *   → { type: 'ready' }
+ *
+ *   execProtocolRaw: { type: 'execProtocolRaw', id, data: ArrayBuffer, options }
+ *   → { type: 'result', id, data: ArrayBuffer }
+ *
+ *   query: { type: 'query', id, sql, params }
+ *   → { type: 'result', id, rows, affectedRows }
+ *
+ *   exec: { type: 'exec', id, sql }
+ *   → { type: 'result', id, results: [{ affectedRows }] }
+ *
+ *   listen/unlisten/close: same as pglite-worker-thread.ts
+ */
+
+// NOTE: this file is meant to be bundled with PGlite as external
+// the consumer provides PGlite via importScripts or ESM import
+
+declare const self: any
+
+const listeners = new Map<number, () => Promise<void>>()
+let db: any // PGlite instance — type depends on how it's loaded
+
+self.onmessage = async (ev: MessageEvent) => {
+  const msg = ev.data
+  if (!msg || typeof msg !== 'object') return
+  const { type, id } = msg
+
+  try {
+    switch (type) {
+      case 'init': {
+        // dynamically import PGlite (external, provided by consumer's bundler)
+        const { PGlite } = await import('@electric-sql/pglite')
+        db = new PGlite({
+          dataDir: msg.dataDir || 'idb://orez-pglite',
+          relaxedDurability: true,
+          initialMemory: 32 * 1024 * 1024,
+          startParams: [
+            '--single',
+            '-F',
+            '-O',
+            '-j',
+            '-c',
+            'search_path=public',
+            '-c',
+            'exit_on_error=false',
+            '-c',
+            'log_checkpoints=false',
+            '-c',
+            'shared_buffers=4MB',
+            '-c',
+            'wal_buffers=1MB',
+          ],
+          ...(msg.pgliteOptions || {}),
+          // extensions loaded by consumer if needed
+        })
+        await db.waitReady
+
+        // tune postgres internals — modest values for embedded use
+        await db.exec(`
+          SET work_mem = '4MB';
+          SET maintenance_work_mem = '16MB';
+          SET effective_cache_size = '64MB';
+          SET random_page_cost = 1.1;
+          SET jit = off;
+        `)
+
+        self.postMessage({ type: 'ready' })
+        break
+      }
+
+      case 'execProtocolRaw': {
+        const input = new Uint8Array(msg.data as ArrayBuffer)
+        const result = await db.execProtocolRaw(input, msg.options)
+        const buf = new ArrayBuffer(result.byteLength)
+        new Uint8Array(buf).set(result)
+        self.postMessage({ type: 'result', id, data: buf }, [buf])
+        break
+      }
+
+      case 'query': {
+        const result = await db.query(msg.sql, msg.params)
+        self.postMessage({
+          type: 'result',
+          id,
+          rows: result.rows,
+          affectedRows: result.affectedRows,
+        })
+        break
+      }
+
+      case 'exec': {
+        const result = await db.exec(msg.sql)
+        const results = result.map((r: any) => ({ affectedRows: r.affectedRows ?? 0 }))
+        self.postMessage({ type: 'result', id, results })
+        break
+      }
+
+      case 'listen': {
+        const unsub = await db.listen(msg.channel, (payload: string) => {
+          self.postMessage({ type: 'notification', channel: msg.channel, payload })
+        })
+        listeners.set(id, unsub)
+        self.postMessage({ type: 'result', id })
+        break
+      }
+
+      case 'unlisten': {
+        const unsub = listeners.get(msg.listenId)
+        if (unsub) {
+          await unsub()
+          listeners.delete(msg.listenId)
+        }
+        self.postMessage({ type: 'result', id })
+        break
+      }
+
+      case 'close': {
+        for (const unsub of listeners.values()) {
+          await unsub().catch(() => {})
+        }
+        listeners.clear()
+        await db.close()
+        self.postMessage({ type: 'result', id })
+        break
+      }
+
+      default:
+        self.postMessage({
+          type: 'error',
+          id,
+          message: `unknown message type: ${type}`,
+        })
+    }
+  } catch (err: unknown) {
+    const error = err as { message?: string; code?: string }
+    self.postMessage({
+      type: 'error',
+      id,
+      message: error?.message || String(err),
+      code: error?.code,
+    })
+  }
+}

package/src/pglite-worker-thread.ts

@@ -38,42 +38,84 @@ const listeners = new Map<number, () => Promise<void>>()
 
 let db: PGlite
 
+const PGLITE_BASE_FLAGS = [
+  '--single',
+  '-F',
+  '-O',
+  '-j',
+  '-c',
+  'search_path=public',
+  '-c',
+  'exit_on_error=false',
+  '-c',
+  'log_checkpoints=false',
+]
+
+const ZERO_START_PARAMS = [
+  ...PGLITE_BASE_FLAGS,
+  '-c',
+  'shared_buffers=128kB',
+  '-c',
+  'wal_buffers=64kB',
+  '-c',
+  'work_mem=64kB',
+  '-c',
+  'maintenance_work_mem=1MB',
+  '-c',
+  'temp_buffers=800kB',
+]
+
 async function init() {
   const { dataDir: _userDataDir, debug: _dbg, ...userOpts } = config.pgliteOptions || {}
+  const isMain = config.withExtensions
 
   db = new PGlite({
     dataDir: config.dataDir,
     debug: config.debug,
     relaxedDurability: true,
-    ...userOpts,
-    extensions: config.withExtensions
-      ? userOpts.extensions || {
-          vector,
-          pg_trgm,
-          pgcrypto,
-          uuid_ossp,
-          citext,
-          hstore,
-          ltree,
-          fuzzystrmatch,
-          btree_gin,
-          btree_gist,
-          cube,
-          earthdistance,
+    initialMemory: isMain ? 32 * 1024 * 1024 : 16 * 1024 * 1024,
+    ...(isMain ? {} : { startParams: ZERO_START_PARAMS }),
+    ...(isMain
+      ? {
+          startParams: [
+            ...PGLITE_BASE_FLAGS,
+            '-c',
+            'shared_buffers=4MB',
+            '-c',
+            'wal_buffers=1MB',
+          ],
+          ...userOpts,
+          extensions: userOpts.extensions || {
+            vector,
+            pg_trgm,
+            pgcrypto,
+            uuid_ossp,
+            citext,
+            hstore,
+            ltree,
+            fuzzystrmatch,
+            btree_gin,
+            btree_gist,
+            cube,
+            earthdistance,
+          },
         }
-      : {},
+      : { extensions: {} }),
   } as any)
 
   await db.waitReady
 
-  // tune postgres internals
-  await db.exec(`
-    SET work_mem = '64MB';
-    SET maintenance_work_mem = '128MB';
-    SET effective_cache_size = '512MB';
-    SET random_page_cost = 1.1;
-    SET jit = off;
-  `)
+  if (isMain) {
+    await db.exec(`
+      SET work_mem = '4MB';
+      SET maintenance_work_mem = '16MB';
+      SET effective_cache_size = '64MB';
+      SET random_page_cost = 1.1;
+      SET jit = off;
+    `)
+  } else {
+    await db.exec(`SET jit = off;`)
+  }
 
   port.postMessage({ type: 'ready' })
 }

package/src/recovery.ts

@@ -100,9 +100,9 @@ export async function recoverFromCdcCorruption(ctx: RecoveryContext): Promise<vo
 
   // clear upstream replication tracking
   const db = instances.postgres
-  await db.exec(`TRUNCATE _orez.changes`).catch(() => {})
+  await db.exec(`TRUNCATE _orez._zero_changes`).catch(() => {})
   await db.exec(`TRUNCATE _orez._zero_replication_slots`).catch(() => {})
-  await db.exec(`ALTER SEQUENCE _orez.watermark RESTART WITH 1`).catch(() => {})
+  await db.exec(`ALTER SEQUENCE _orez._zero_watermark RESTART WITH 1`).catch(() => {})
 
   // drop stale shard schemas
   const shardSchemas = await db.query<{ schemaname: string }>(

package/src/replication/change-tracker.test.ts

@@ -151,7 +151,7 @@ describe('change-tracker', () => {
     try {
       await db.exec(`CREATE PUBLICATION "zero_scope"`)
       await installChangeTracking(db) // reinstall picks up publication scope
-      await db.exec(`TRUNCATE _orez.changes`)
+      await db.exec(`TRUNCATE _orez._zero_changes`)
 
       await db.exec(`INSERT INTO public.items (name, value) VALUES ('x', 1)`)
       const changes = await getChangesSince(db, 0)

package/src/replication/change-tracker.ts

@@ -17,10 +17,10 @@ export async function installChangeTracking(db: PGlite): Promise<void> {
   // create changes table and watermark sequence
   // watermark is the primary key - monotonically increasing, no separate id needed
   await db.exec(`
-    CREATE SEQUENCE IF NOT EXISTS _orez.watermark;
+    CREATE SEQUENCE IF NOT EXISTS _orez._zero_watermark;
 
-    CREATE TABLE IF NOT EXISTS _orez.changes (
-      watermark BIGINT NOT NULL DEFAULT nextval('_orez.watermark') PRIMARY KEY,
+    CREATE TABLE IF NOT EXISTS _orez._zero_changes (
+      watermark BIGINT NOT NULL DEFAULT nextval('_orez._zero_watermark') PRIMARY KEY,
       table_name TEXT NOT NULL,
       op TEXT NOT NULL CHECK (op IN ('INSERT', 'UPDATE', 'DELETE')),
       row_data JSONB,
@@ -47,7 +47,7 @@ export async function installChangeTracking(db: PGlite): Promise<void> {
     CREATE OR REPLACE FUNCTION public._zero_track_change() RETURNS TRIGGER AS $$
     BEGIN
       IF TG_OP = 'DELETE' THEN
-        INSERT INTO _orez.changes (table_name, op, old_data)
+        INSERT INTO _orez._zero_changes (table_name, op, old_data)
         VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'DELETE', to_jsonb(OLD));
         RETURN OLD;
       ELSIF TG_OP = 'UPDATE' THEN
@@ -55,11 +55,11 @@ export async function installChangeTracking(db: PGlite): Promise<void> {
         IF to_jsonb(NEW) = to_jsonb(OLD) THEN
           RETURN NEW;
         END IF;
-        INSERT INTO _orez.changes (table_name, op, row_data, old_data)
+        INSERT INTO _orez._zero_changes (table_name, op, row_data, old_data)
         VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'UPDATE', to_jsonb(NEW), to_jsonb(OLD));
         RETURN NEW;
       ELSE
-        INSERT INTO _orez.changes (table_name, op, row_data)
+        INSERT INTO _orez._zero_changes (table_name, op, row_data)
         VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'INSERT', to_jsonb(NEW));
         RETURN NEW;
       END IF;
@@ -228,7 +228,7 @@ export async function getChangesSince(
   limit = 50000
 ): Promise<ChangeRecord[]> {
   const result = await db.query<ChangeRecord>(
-    'SELECT watermark, table_name, op, row_data, old_data FROM _orez.changes WHERE watermark > $1 ORDER BY watermark LIMIT $2',
+    'SELECT watermark, table_name, op, row_data, old_data FROM _orez._zero_changes WHERE watermark > $1 ORDER BY watermark LIMIT $2',
     [watermark, limit]
   )
   return result.rows
@@ -239,14 +239,14 @@ export async function purgeConsumedChanges(
   watermark: number
 ): Promise<number> {
   const result = await db.exec(
-    `DELETE FROM _orez.changes WHERE watermark <= ${Number(watermark)}`
+    `DELETE FROM _orez._zero_changes WHERE watermark <= ${Number(watermark)}`
   )
   return result[0]?.affectedRows ?? 0
 }
 
 export async function getCurrentWatermark(db: PGlite): Promise<number> {
   const result = await db.query<{ last_value: string; is_called: boolean }>(
-    'SELECT last_value, is_called FROM _orez.watermark'
+    'SELECT last_value, is_called FROM _orez._zero_watermark'
   )
   const { last_value, is_called } = result.rows[0]
   if (!is_called) return 0

package/src/replication/handler.ts

@@ -32,6 +32,32 @@ import {
 import type { Mutex } from '../mutex.js'
 import type { PGlite } from '@electric-sql/pglite'
 
+// types pglite can't replicate — excluded from change tracking columns
+const UNSUPPORTED_TYPES = new Set(['tsvector', 'tsquery', 'USER-DEFINED'])
+
+// pg data_type string → wire protocol oid mapping
+const PG_DATA_TYPE_OIDS: Record<string, number> = {
+  boolean: 16,
+  bytea: 17,
+  bigint: 20,
+  smallint: 21,
+  integer: 23,
+  text: 25,
+  json: 114,
+  real: 700,
+  'double precision': 701,
+  character: 1042,
+  'character varying': 1043,
+  date: 1082,
+  'time without time zone': 1083,
+  'timestamp without time zone': 1114,
+  'timestamp with time zone': 1184,
+  'time with time zone': 1266,
+  numeric: 1700,
+  uuid: 2950,
+  jsonb: 3802,
+}
+
 export interface ReplicationWriter {
   write(data: Uint8Array): void
   readonly closed?: boolean
@@ -393,7 +419,7 @@ export async function handleStartReplication(
         const all = await db.query<{ tablename: string }>(
           `SELECT tablename FROM pg_tables
            WHERE schemaname = 'public'
-             AND tablename NOT IN ('migrations', 'changes')
+             AND tablename NOT IN ('migrations', '_zero_changes')
              AND tablename NOT LIKE '_zero_%'`
         )
         tables = all.rows
@@ -403,7 +429,7 @@ export async function handleStartReplication(
       const ddlParts: string[] = [
         `CREATE OR REPLACE FUNCTION public._zero_notify_change() RETURNS TRIGGER AS $$
         BEGIN
-          PERFORM pg_notify('changes', TG_TABLE_NAME);
+          PERFORM pg_notify('_zero_changes', TG_TABLE_NAME);
           RETURN NULL;
         END;
         $$ LANGUAGE plpgsql;`,
@@ -499,28 +525,6 @@ export async function handleStartReplication(
           }
           keys.add(row.column_name)
         } else {
-          const UNSUPPORTED_TYPES = new Set(['tsvector', 'tsquery', 'USER-DEFINED'])
-          const PG_DATA_TYPE_OIDS: Record<string, number> = {
-            boolean: 16,
-            bytea: 17,
-            bigint: 20,
-            smallint: 21,
-            integer: 23,
-            text: 25,
-            json: 114,
-            real: 700,
-            'double precision': 701,
-            character: 1042,
-            'character varying': 1043,
-            date: 1082,
-            'time without time zone': 1083,
-            'timestamp without time zone': 1114,
-            'timestamp with time zone': 1184,
-            'time with time zone': 1266,
-            numeric: 1700,
-            uuid: 2950,
-            jsonb: 3802,
-          }
           if (row.data_type && UNSUPPORTED_TYPES.has(row.data_type)) {
             let cols = excludedColumns.get(key)
             if (!cols) {
@@ -612,8 +616,8 @@ export async function handleStartReplication(
   // also set up LISTEN as secondary signal
   let unsubscribe: (() => Promise<void>) | null = null
   try {
-    unsubscribe = await db.listen('changes', wakeup)
-    log.debug.proxy('replication: listening for changes notifications')
+    unsubscribe = await db.listen('_zero_changes', wakeup)
+    log.debug.proxy('replication: listening for _zero_changes notifications')
   } catch {
     log.debug.proxy('replication: LISTEN not available')
   }
@@ -962,6 +966,13 @@ async function streamChanges(
   for (const msg of messages) {
     writer.write(msg)
   }
+
+  // hook for arch instrumentation (soot-arch sq-write events)
+  const hook = (globalThis as any).__orez_on_repl_commit
+  if (hook) {
+    const tables = new Set(changes.map((c) => c.table_name))
+    hook({ changes: changes.length, tables: [...tables], txId })
+  }
 }
 
 function normalizeShardClientsRow(

package/src/worker/browser-build-config.test.ts

@@ -24,7 +24,7 @@ describe('browser build config', () => {
     it('includes Node.js polyfills', () => {
       const aliases = getBrowserAliases()
       expect(aliases['node:events']).toBe('events')
-      expect(aliases['node:stream']).toBe('stream-browserify')
+      expect(aliases['node:stream']).toBe('orez/worker/shims/stream-browser')
       expect(aliases['node:path']).toBe('path-browserify')
     })
 

package/src/worker/browser-build-config.ts

@@ -42,7 +42,10 @@
 export function getBrowserAliases(): Record<string, string> {
   return {
     // -- orez shims for zero-cache dependencies --
-    postgres: 'orez/worker/shims/postgres',
+    // postgres-browser uses the real postgres package with MessagePort transport
+    // to pg-proxy-browser, matching orez-node's wire protocol architecture.
+    // falls back to old PGlite-wrapping shim if postgres-browser isn't available.
+    postgres: 'orez/worker/shims/postgres-browser',
     '@rocicorp/zero-sqlite3': 'orez/worker/shims/sqlite',
     fastify: 'orez/worker/shims/fastify',
     ws: 'orez/worker/shims/ws',
@@ -55,7 +58,7 @@ export function getBrowserAliases(): Record<string, string> {
     'node:process': 'process/browser',
     'node:crypto': 'orez/worker/shims/node-stub',
     'crypto-browserify': 'orez/worker/shims/node-stub',
-    'node:stream': 'stream-browserify',
+    'node:stream': 'orez/worker/shims/stream-browser',
     'node:path': 'path-browserify',
     'node:os': 'os-browserify/browser',
 

package/src/worker/browser-embed.ts

@@ -171,9 +171,9 @@ export async function startZeroCacheEmbedBrowser(
     ...((globalThis as any).process.env as Record<string, string>),
     SINGLE_PROCESS: '1',
     NODE_ENV: 'development',
-    ZERO_UPSTREAM_DB: 'pglite://in-process',
-    ZERO_CVR_DB: 'pglite://in-process',
-    ZERO_CHANGE_DB: 'pglite://in-process',
+    ZERO_UPSTREAM_DB: 'pglite://localhost/postgres',
+    ZERO_CVR_DB: 'pglite://localhost/zero_cvr',
+    ZERO_CHANGE_DB: 'pglite://localhost/zero_cdb',
     ZERO_REPLICA_FILE: ':browser-sqlite:',
     ZERO_PORT: '0',
     ZERO_APP_ID: appId,
@@ -263,40 +263,43 @@ export async function startZeroCacheEmbedBrowser(
     },
 
     async handleWebSocket(ws: WsLike, url = '/', headers?: Record<string, string>) {
-      // lazily resolve fastifyInstance — ZeroDispatcher is constructed AFTER the
-      // ready signal fires, so __orez_fastify_instance may not be set yet.
-      // the dispatcher's instance is the one with the WS handoff handler,
-      // and it's always the LAST Fastify() call. we need to wait for it.
-      //
-      // check: the dispatcher's server will have 'message' listeners (from
-      // installWebSocketHandoff). poll until we find an instance with listeners.
+      // lazily resolve fastify instances — ZeroDispatcher is constructed AFTER the
+      // ready signal fires, so instances may not all be registered yet.
+      // poll until we have instances with ws routes (tryHandoff will check).
+      let instances: any[] = []
       for (let i = 0; i < 100; i++) {
+        instances = (globalThis as any).__orez_fastify_instances || []
         fastifyInstance = (globalThis as any).__orez_fastify_instance
-        // the dispatcher's server has 2+ 'message' listeners:
-        //   1. from FastifyShim#installWsHandoffHandler (every instance has this)
-        //   2. from ZeroDispatcher's installWebSocketHandoff (only the dispatcher has this)
-        // wait for >= 2 to ensure we have the dispatcher's instance
-        if (fastifyInstance?.server?.listenerCount?.('message') >= 2) break
+        // wait until we have at least one instance with 2+ message listeners
+        // (the dispatcher's instance has both the shim handler + installWebSocketHandoff)
+        if (instances.some((inst: any) => inst?.server?.listenerCount?.('message') >= 2))
+          break
         await new Promise((r) => setTimeout(r, 50))
       }
-      if (!isReady || !fastifyInstance?.server) return
-
-      const message = {
-        url,
-        headers: headers || {},
-        method: 'GET',
+      if (!isReady) return
+
+      const handoffMsg = {
+        message: {
+          url,
+          headers: headers || {},
+          method: 'GET',
+        },
+        head: new Uint8Array(0),
       }
 
-      // emit handoff
+      // try all fastify instances via tryHandoff, stop at first match
+      let handled = false
+      for (const inst of instances) {
+        if (inst?.tryHandoff?.(handoffMsg, ws)) {
+          handled = true
+          break
+        }
+      }
 
-      // feed the WebSocket into zero-cache's handoff mechanism.
-      // the fastify shim's server is an EventEmitter with onMessageType.
-      // installWebSocketHandoff (non-Server branch) listens for "handoff".
-      fastifyInstance.server.emit(
-        'message',
-        ['handoff', { message, head: new Uint8Array(0) }],
-        ws // the WebSocket-like object as sendHandle
-      )
+      // fallback: emit directly on the last instance's server
+      if (!handled && fastifyInstance?.server) {
+        fastifyInstance.server.emit('message', ['handoff', handoffMsg], ws)
+      }
     },
 
     async handleHttp(request: HttpRequest): Promise<HttpResponse> {

package/src/worker/shims/fastify.ts

@@ -124,32 +124,41 @@ class FastifyShim {
   // and call the handler with the socket.
   #installWsHandoffHandler() {
     this.server.onMessageType('handoff', (msg: any, socket?: any) => {
-      if (!socket || !msg?.message?.url) return
-      const url = msg.message.url
-      const parsedUrl = new URL(url, 'http://localhost')
-      const pathname = parsedUrl.pathname
-
-      for (const route of this.#wsRoutes) {
-        if (route.pattern.test(pathname)) {
-          const req = {
-            url,
-            headers: msg.message.headers || {},
-            method: msg.message.method || 'GET',
-          }
-          // wrap socket through handleUpgrade so it gets the full WS API
-          // (ping, on, once, terminate, etc.) needed by zero-cache's streamOut
-          this.websocketServer.handleUpgrade(
-            req,
-            socket,
-            Buffer.from(new Uint8Array(0)),
-            (ws: any) => {
-              route.handler(ws, req)
-            }
-          )
-          return
+      this.tryHandoff(msg, socket)
+    })
+  }
+
+  // try to match a handoff message against registered websocket routes.
+  // returns true if a route matched, false otherwise.
+  // this is public so callers (ws shim, browser-embed) can iterate
+  // all fastify instances and stop at the first match.
+  tryHandoff(msg: any, socket?: any): boolean {
+    if (!socket || !msg?.message?.url) return false
+    const url = msg.message.url
+    const parsedUrl = new URL(url, 'http://localhost')
+    const pathname = parsedUrl.pathname
+
+    for (const route of this.#wsRoutes) {
+      if (route.pattern.test(pathname)) {
+        const req = {
+          url,
+          headers: msg.message.headers || {},
+          method: msg.message.method || 'GET',
         }
+        // wrap socket through handleUpgrade so it gets the full WS API
+        // (ping, on, once, terminate, etc.) needed by zero-cache's streamOut
+        this.websocketServer.handleUpgrade(
+          req,
+          socket,
+          Buffer.from(new Uint8Array(0)),
+          (ws: any) => {
+            route.handler(ws, req)
+          }
+        )
+        return true
       }
-    })
+    }
+    return false
   }
 
   // route registration — supports optional { websocket: true } option
@@ -286,6 +295,10 @@ function Fastify(_opts?: unknown): FastifyShim {
   // always overwrite — the ZeroDispatcher (which has the WS handoff routes)
   // is created LAST, so the final instance is the one handleWebSocket needs.
   ;(globalThis as any).__orez_fastify_instance = instance
+  // track all instances so callers can try handoff against each one
+  ;(globalThis as any).__orez_fastify_instances =
+    (globalThis as any).__orez_fastify_instances || []
+  ;(globalThis as any).__orez_fastify_instances.push(instance)
   return instance
 }