orez 0.1.43 → 0.1.45

package/src/pg-proxy.ts

@@ -45,24 +45,45 @@ const SCHEMA_CACHE_TTL_MS = 30_000
 // performance tracking
 const proxyStats = { totalWaitMs: 0, totalExecMs: 0, count: 0, batches: 0 }
 
-function isCacheableQuery(query: string): boolean {
-  const q = query.trimStart().toLowerCase()
-  return (
-    (q.includes('information_schema.') ||
-      q.includes('pg_catalog.') ||
-      q.includes('pg_tables') ||
-      q.includes('pg_namespace') ||
-      q.includes('pg_class') ||
-      q.includes('pg_attribute') ||
-      q.includes('pg_type') ||
-      q.includes('pg_publication')) &&
-    !q.startsWith('insert') &&
-    !q.startsWith('update') &&
-    !q.startsWith('delete') &&
-    !q.startsWith('create') &&
-    !q.startsWith('alter') &&
-    !q.startsWith('drop')
-  )
+// query classification helpers — operate on pre-normalized (trimmed+lowercased) query strings
+const SCHEMA_QUERY_MARKERS = [
+  'information_schema.',
+  'pg_catalog.',
+  'pg_tables',
+  'pg_namespace',
+  'pg_class',
+  'pg_attribute',
+  'pg_type',
+  'pg_publication',
+]
+const WRITE_PREFIXES = ['insert', 'update', 'delete', 'copy', 'truncate']
+const DDL_PREFIXES = ['create', 'alter', 'drop']
+const MUTATING_PREFIXES = [...WRITE_PREFIXES, ...DDL_PREFIXES]
+
+function isCacheableNormalized(q: string): boolean {
+  // fast-fail: mutating queries are never cacheable
+  for (const p of MUTATING_PREFIXES) {
+    if (q.startsWith(p)) return false
+  }
+  // check if it touches schema/catalog tables
+  for (const marker of SCHEMA_QUERY_MARKERS) {
+    if (q.includes(marker)) return true
+  }
+  return false
+}
+
+function isWriteNormalized(q: string): boolean {
+  for (const p of WRITE_PREFIXES) {
+    if (q.startsWith(p)) return true
+  }
+  return false
+}
+
+function isDDLNormalized(q: string): boolean {
+  for (const p of DDL_PREFIXES) {
+    if (q.startsWith(p)) return true
+  }
+  return false
 }
 
 function extractQueryText(data: Uint8Array): string | null {
@@ -795,10 +816,13 @@ export async function startPgProxy(
           // intercept and rewrite queries
           data = interceptQuery(data)
 
-          // cache Simple Query (0x51) schema queries
+          // normalize query once for all classification checks
           const isSimpleQuery = msgType === 0x51
           const queryText = isSimpleQuery ? extractQueryText(data) : null
-          const cacheable = queryText && isCacheableQuery(queryText)
+          const queryNorm = queryText ? queryText.trimStart().toLowerCase() : null
+          const cacheable = queryNorm && isCacheableNormalized(queryNorm)
+
+          // cache Simple Query schema queries
           if (cacheable) {
             const cached = schemaQueryCache.get(queryText!)
             if (cached && Date.now() < cached.expiresAt) {
@@ -861,15 +885,8 @@ export async function startPgProxy(
             }
           } else {
             result = await execute()
-            if (isSimpleQuery && queryText) {
-              const q = queryText.trimStart().toLowerCase()
-              if (
-                q.startsWith('create') ||
-                q.startsWith('alter') ||
-                q.startsWith('drop')
-              ) {
-                invalidateSchemaCache()
-              }
+            if (queryNorm && isDDLNormalized(queryNorm)) {
+              invalidateSchemaCache()
             }
           }
 
@@ -877,17 +894,8 @@ export async function startPgProxy(
           result = stripResponseMessages(result, stripRfq)
 
           // signal replication handler on postgres writes for instant sync
-          if (dbName === 'postgres' && isSimpleQuery && queryText) {
-            const q = queryText.trimStart().toLowerCase()
-            if (
-              q.startsWith('insert') ||
-              q.startsWith('update') ||
-              q.startsWith('delete') ||
-              q.startsWith('copy') ||
-              q.startsWith('truncate')
-            ) {
-              signalReplicationChange()
-            }
+          if (dbName === 'postgres' && queryNorm && isWriteNormalized(queryNorm)) {
+            signalReplicationChange()
           }
 
           return result

package/src/pglite-ipc.ts

@@ -26,13 +26,14 @@ interface PendingRequest {
 const WRITE_PREFIXES = ['insert', 'update', 'delete', 'copy', 'truncate']
 // shard-internal tables that the replication handler filters out.
 // signaling for these just causes spurious wakeups + mutex contention.
-const SHARD_INTERNAL_TABLES = ['"replicas"', '"mutations"', '"replicationState"']
+// pre-lowercased so we don't call toLowerCase() per iteration
+const SHARD_INTERNAL_TABLES = ['"replicas"', '"mutations"', '"replicationstate"']
 function isReplicatedWrite(sql: string): boolean {
   const q = sql.trimStart().toLowerCase()
   if (!WRITE_PREFIXES.some((p) => q.startsWith(p))) return false
   // skip shard-internal writes (zero-cache manages these, not replicated)
   for (const t of SHARD_INTERNAL_TABLES) {
-    if (q.includes(t.toLowerCase())) return false
+    if (q.includes(t)) return false
   }
   return true
 }

package/src/pglite-manager.ts

@@ -78,6 +78,70 @@ export interface PGliteInstances {
   cdb: PGlite
 }
 
+// shared setup extracted from the 4 factory functions below
+
+/** migrate old single-instance pgdata dir to the new pgdata-postgres layout */
+function migrateDataDir(config: ZeroLiteConfig): void {
+  const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
+  if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
+    const oldDataPath = resolve(config.dataDir, 'pgdata')
+    const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
+    if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
+      renameSync(oldDataPath, newDataPath)
+      log.debug.pglite('migrated pgdata → pgdata-postgres')
+    }
+  }
+}
+
+/** create publication if ZERO_APP_PUBLICATIONS is set and publication doesn't exist */
+async function ensurePublication(db: {
+  exec(sql: string): Promise<any>
+  query<T>(sql: string, params?: any[]): Promise<{ rows: T[] }>
+}): Promise<void> {
+  await db.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
+
+  const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
+  if (pubName) {
+    const pubs = await db.query<{ count: string }>(
+      `SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
+      [pubName]
+    )
+    if (Number(pubs.rows[0].count) === 0) {
+      const quoted = '"' + pubName.replace(/"/g, '""') + '"'
+      await db.exec(`CREATE PUBLICATION ${quoted}`)
+    }
+  }
+}
+
+// pglite startParams replaces defaults, so always include required flags
+const PGLITE_BASE_FLAGS = [
+  '--single',
+  '-F',
+  '-O',
+  '-j',
+  '-c',
+  'search_path=public',
+  '-c',
+  'exit_on_error=false',
+  '-c',
+  'log_checkpoints=false',
+]
+
+// cvr/cdb are just zero-cache bookkeeping — minimal fixed memory
+const ZERO_START_PARAMS = [
+  ...PGLITE_BASE_FLAGS,
+  '-c',
+  'shared_buffers=128kB',
+  '-c',
+  'wal_buffers=64kB',
+  '-c',
+  'work_mem=64kB',
+  '-c',
+  'maintenance_work_mem=1MB',
+  '-c',
+  'temp_buffers=800kB',
+]
+
 // create a single pglite instance with given dataDir suffix
 async function createInstance(
   config: ZeroLiteConfig,
@@ -103,42 +167,57 @@ async function createInstance(
 
   log.debug.pglite(`creating ${name} instance at ${dataPath}`)
 
+  const isMain = withExtensions
+
   try {
     const db = new PGlite({
       dataDir: dataPath,
       debug: config.logLevel === 'debug' ? 1 : 0,
       relaxedDurability: true,
-      ...userOpts,
-      extensions: withExtensions
-        ? userOpts.extensions || {
-            vector,
-            pg_trgm,
-            pgcrypto,
-            uuid_ossp,
-            citext,
-            hstore,
-            ltree,
-            fuzzystrmatch,
-            btree_gin,
-            btree_gist,
-            cube,
-            earthdistance,
+      initialMemory: isMain ? 32 * 1024 * 1024 : 16 * 1024 * 1024,
+      ...(isMain ? {} : { startParams: ZERO_START_PARAMS }),
+      // main instance: user overrides via pgliteOptions, zero instances: fixed
+      ...(isMain
+        ? {
+            startParams: [
+              ...PGLITE_BASE_FLAGS,
+              '-c',
+              'shared_buffers=4MB',
+              '-c',
+              'wal_buffers=1MB',
+            ],
+            ...userOpts,
+            extensions: userOpts.extensions || {
+              vector,
+              pg_trgm,
+              pgcrypto,
+              uuid_ossp,
+              citext,
+              hstore,
+              ltree,
+              fuzzystrmatch,
+              btree_gin,
+              btree_gist,
+              cube,
+              earthdistance,
+            },
           }
-        : {},
+        : { extensions: {} }),
     })
 
     await db.waitReady
 
-    // tune postgres internals for throughput over durability.
-    // pglite defaults are conservative for embedded use — these settings
-    // match what a real postgres would use for a dev/test workload.
-    await db.exec(`
-      SET work_mem = '64MB';
-      SET maintenance_work_mem = '128MB';
-      SET effective_cache_size = '512MB';
-      SET random_page_cost = 1.1;
-      SET jit = off;
-    `)
+    if (isMain) {
+      await db.exec(`
+        SET work_mem = '4MB';
+        SET maintenance_work_mem = '16MB';
+        SET effective_cache_size = '64MB';
+        SET random_page_cost = 1.1;
+        SET jit = off;
+      `)
+    } else {
+      await db.exec(`SET jit = off;`)
+    }
 
     log.debug.pglite(`${name} ready`)
     return db
@@ -174,40 +253,15 @@ async function createInstance(
 export async function createPGliteInstances(
   config: ZeroLiteConfig
 ): Promise<PGliteInstances> {
-  // migrate from old single-instance layout (pgdata → pgdata-postgres)
-  const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
-  if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
-    const oldDataPath = resolve(config.dataDir, 'pgdata')
-    const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
-    if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
-      renameSync(oldDataPath, newDataPath)
-      log.debug.pglite('migrated pgdata → pgdata-postgres')
-    }
-  }
+  migrateDataDir(config)
 
-  // create all 3 instances in parallel (only postgres needs app extensions)
   const [postgres, cvr, cdb] = await Promise.all([
     createInstance(config, 'postgres', true),
     createInstance(config, 'cvr', false),
     createInstance(config, 'cdb', false),
   ])
 
-  // postgres-specific setup
-  await postgres.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
-
-  // create publication only when explicitly configured
-  const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
-  if (pubName) {
-    const pubs = await postgres.query<{ count: string }>(
-      `SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
-      [pubName]
-    )
-    if (Number(pubs.rows[0].count) === 0) {
-      const quoted = '"' + pubName.replace(/"/g, '""') + '"'
-      await postgres.exec(`CREATE PUBLICATION ${quoted}`)
-    }
-  }
-
+  await ensurePublication(postgres)
   return { postgres, cvr, cdb }
 }
 
@@ -221,17 +275,9 @@ export async function createPGliteInstances(
 export async function createPGliteWorkerInstances(
   config: ZeroLiteConfig
 ): Promise<PGliteInstances> {
-  // migrate from old single-instance layout (pgdata → pgdata-postgres)
-  const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
-  if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
-    const oldDataPath = resolve(config.dataDir, 'pgdata')
-    const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
-    if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
-      renameSync(oldDataPath, newDataPath)
-      log.debug.pglite('migrated pgdata → pgdata-postgres')
-    }
-  }
+  migrateDataDir(config)
 
+  const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
   const useMemory =
     typeof pgliteDataDir === 'string' && pgliteDataDir.startsWith('memory://')
   const {
@@ -259,32 +305,15 @@ export async function createPGliteWorkerInstances(
 
   log.pglite('starting worker threads for postgres, cvr, cdb')
 
-  // create all 3 worker proxies in parallel
   const pgProxy = new PGliteWorkerProxy(makeWorkerConfig('postgres', true))
   const cvrProxy = new PGliteWorkerProxy(makeWorkerConfig('cvr', false))
   const cdbProxy = new PGliteWorkerProxy(makeWorkerConfig('cdb', false))
 
   await Promise.all([pgProxy.waitReady, cvrProxy.waitReady, cdbProxy.waitReady])
-
   log.pglite('all worker threads ready')
 
-  // postgres-specific setup
-  await pgProxy.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
+  await ensurePublication(pgProxy)
 
-  // create publication only when explicitly configured
-  const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
-  if (pubName) {
-    const pubs = await pgProxy.query<{ count: string }>(
-      `SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
-      [pubName]
-    )
-    if (Number(pubs.rows[0].count) === 0) {
-      const quoted = '"' + pubName.replace(/"/g, '""') + '"'
-      await pgProxy.exec(`CREATE PUBLICATION ${quoted}`)
-    }
-  }
-
-  // cast to PGlite — our proxy implements the same interface surface
   return {
     postgres: pgProxy as unknown as PGlite,
     cvr: cvrProxy as unknown as PGlite,
@@ -302,36 +331,11 @@ export async function createPGliteWorkerInstances(
 export async function createSinglePGliteInstance(
   config: ZeroLiteConfig
 ): Promise<PGliteInstances> {
-  // migrate from old single-instance layout (pgdata → pgdata-postgres)
-  const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
-  if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
-    const oldDataPath = resolve(config.dataDir, 'pgdata')
-    const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
-    if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
-      renameSync(oldDataPath, newDataPath)
-      log.debug.pglite('migrated pgdata → pgdata-postgres')
-    }
-  }
-
+  migrateDataDir(config)
   log.pglite('starting single shared pglite instance')
 
   const db = await createInstance(config, 'postgres', true)
-
-  // postgres-specific setup
-  await db.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
-
-  // create publication only when explicitly configured
-  const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
-  if (pubName) {
-    const pubs = await db.query<{ count: string }>(
-      `SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
-      [pubName]
-    )
-    if (Number(pubs.rows[0].count) === 0) {
-      const quoted = '"' + pubName.replace(/"/g, '""') + '"'
-      await db.exec(`CREATE PUBLICATION ${quoted}`)
-    }
-  }
+  await ensurePublication(db)
 
   // same instance for all three — pg-proxy detects this and shares a mutex
   return { postgres: db, cvr: db, cdb: db }
@@ -343,17 +347,9 @@ export async function createSinglePGliteInstance(
 export async function createSinglePGliteWorkerInstance(
   config: ZeroLiteConfig
 ): Promise<PGliteInstances> {
-  // migrate from old single-instance layout (pgdata → pgdata-postgres)
-  const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
-  if (!pgliteDataDir || !String(pgliteDataDir).startsWith('memory://')) {
-    const oldDataPath = resolve(config.dataDir, 'pgdata')
-    const newDataPath = resolve(config.dataDir, 'pgdata-postgres')
-    if (existsSync(oldDataPath) && !existsSync(newDataPath)) {
-      renameSync(oldDataPath, newDataPath)
-      log.debug.pglite('migrated pgdata → pgdata-postgres')
-    }
-  }
+  migrateDataDir(config)
 
+  const pgliteDataDir = (config.pgliteOptions as Record<string, any>)?.dataDir
   const useMemory =
     typeof pgliteDataDir === 'string' && pgliteDataDir.startsWith('memory://')
   const {
@@ -383,21 +379,7 @@ export async function createSinglePGliteWorkerInstance(
   await proxy.waitReady
   log.pglite('single worker thread ready')
 
-  // postgres-specific setup
-  await proxy.exec('CREATE EXTENSION IF NOT EXISTS plpgsql')
-
-  // create publication only when explicitly configured
-  const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
-  if (pubName) {
-    const pubs = await proxy.query<{ count: string }>(
-      `SELECT count(*) as count FROM pg_publication WHERE pubname = $1`,
-      [pubName]
-    )
-    if (Number(pubs.rows[0].count) === 0) {
-      const quoted = '"' + pubName.replace(/"/g, '""') + '"'
-      await proxy.exec(`CREATE PUBLICATION ${quoted}`)
-    }
-  }
+  await ensurePublication(proxy)
 
   const db = proxy as unknown as PGlite
   return { postgres: db, cvr: db, cdb: db }

package/src/pglite-web-proxy.ts

@@ -0,0 +1,180 @@
+/**
+ * PGlite Web Worker proxy — browser equivalent of pglite-ipc.ts.
+ *
+ * runs in the zero-cache worker, proxies calls to a Web Worker
+ * running the actual PGlite instance. mirrors PGliteWorkerProxy
+ * from pglite-ipc.ts but uses Web Worker postMessage instead of
+ * node worker_threads.
+ *
+ * ArrayBuffers are transferred (not copied) for execProtocolRaw
+ * to keep IPC overhead near-zero for wire protocol data.
+ */
+
+import { signalReplicationChange } from './replication/handler.js'
+
+interface PendingRequest {
+  resolve: (value: any) => void
+  reject: (error: Error) => void
+}
+
+const WRITE_PREFIXES = ['insert', 'update', 'delete', 'copy', 'truncate']
+const SHARD_INTERNAL_TABLES = ['"replicas"', '"mutations"', '"replicationState"']
+
+function isReplicatedWrite(sql: string): boolean {
+  const q = sql.trimStart().toLowerCase()
+  if (!WRITE_PREFIXES.some((p) => q.startsWith(p))) return false
+  for (const t of SHARD_INTERNAL_TABLES) {
+    if (q.includes(t.toLowerCase())) return false
+  }
+  return true
+}
+
+export class PGliteWebProxy {
+  private worker: Worker
+  private pending = new Map<number, PendingRequest>()
+  private nextId = 1
+  private notificationCallbacks = new Map<string, Set<(payload: string) => void>>()
+  readonly name: string
+
+  readonly waitReady: Promise<void>
+
+  // PGlite compat flags
+  closed = false
+  ready = false
+
+  constructor(worker: Worker, name: string) {
+    this.name = name
+    this.worker = worker
+
+    let onReady: () => void
+    this.waitReady = new Promise<void>((resolveReady, rejectReady) => {
+      onReady = () => {
+        this.ready = true
+        resolveReady()
+      }
+
+      const onMessage = (ev: MessageEvent) => {
+        const msg = ev.data
+        if (msg?.type === 'ready') {
+          this.worker.removeEventListener('message', onMessage)
+          this.installMessageHandler()
+          onReady()
+        } else if (msg?.type === 'error' && msg.id === 0) {
+          rejectReady(new Error(msg.message))
+        }
+      }
+
+      this.worker.addEventListener('message', onMessage)
+      this.worker.addEventListener('error', (ev) => {
+        rejectReady(new Error(String(ev)))
+      })
+    })
+  }
+
+  private installMessageHandler() {
+    this.worker.addEventListener('message', (ev: MessageEvent) => {
+      const msg = ev.data
+      if (!msg || typeof msg !== 'object') return
+
+      if (msg.type === 'notification') {
+        const callbacks = this.notificationCallbacks.get(msg.channel)
+        if (callbacks) {
+          for (const cb of callbacks) {
+            try {
+              cb(msg.payload)
+            } catch {}
+          }
+        }
+        return
+      }
+
+      const req = this.pending.get(msg.id)
+      if (!req) return
+      this.pending.delete(msg.id)
+
+      if (msg.type === 'error') {
+        const err = new Error(msg.message) as Error & { code?: string }
+        if (msg.code) err.code = msg.code
+        req.reject(err)
+      } else {
+        req.resolve(msg)
+      }
+    })
+  }
+
+  private send(msg: Record<string, unknown>, transfer?: Transferable[]): Promise<any> {
+    const id = this.nextId++
+    msg.id = id
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject })
+      if (transfer?.length) {
+        this.worker.postMessage(msg, transfer)
+      } else {
+        this.worker.postMessage(msg)
+      }
+    })
+  }
+
+  async execProtocolRaw(
+    data: Uint8Array,
+    options?: { syncToFs?: boolean; throwOnError?: boolean }
+  ): Promise<Uint8Array> {
+    // copy to a transferable buffer then transfer
+    const buf = new ArrayBuffer(data.byteLength)
+    new Uint8Array(buf).set(data)
+    const result = await this.send({ type: 'execProtocolRaw', data: buf, options }, [buf])
+    return new Uint8Array(result.data)
+  }
+
+  async query<T = any>(
+    sql: string,
+    params?: any[]
+  ): Promise<{ rows: T[]; affectedRows?: number }> {
+    const result = await this.send({ type: 'query', sql, params })
+    // signal replication after writes on postgres instance (like orez-node's PGliteWorkerProxy)
+    if (this.name === 'postgres' && isReplicatedWrite(sql)) {
+      signalReplicationChange()
+    }
+    return { rows: result.rows ?? [], affectedRows: result.affectedRows }
+  }
+
+  async exec(sql: string): Promise<{ affectedRows?: number }[]> {
+    const result = await this.send({ type: 'exec', sql })
+    if (this.name === 'postgres' && isReplicatedWrite(sql)) {
+      signalReplicationChange()
+    }
+    return result.results ?? []
+  }
+
+  async listen(
+    channel: string,
+    callback: (payload: string) => void
+  ): Promise<() => Promise<void>> {
+    let callbacks = this.notificationCallbacks.get(channel)
+    if (!callbacks) {
+      callbacks = new Set()
+      this.notificationCallbacks.set(channel, callbacks)
+    }
+    callbacks.add(callback)
+
+    const result = await this.send({ type: 'listen', channel })
+    const listenId = result.id
+
+    return async () => {
+      callbacks!.delete(callback)
+      if (callbacks!.size === 0) {
+        this.notificationCallbacks.delete(channel)
+      }
+      await this.send({ type: 'unlisten', listenId }).catch(() => {})
+    }
+  }
+
+  async close(): Promise<void> {
+    this.closed = true
+    this.ready = false
+    try {
+      await this.send({ type: 'close' })
+    } catch {}
+    this.worker.terminate()
+  }
+}