orez 0.2.24 → 0.2.26

package/src/cf-do/wrangler.toml

@@ -0,0 +1,11 @@
+name = "zero-do"
+main = "worker.ts"
+compatibility_date = "2025-04-01"
+
+[[durable_objects.bindings]]
+name = "ZERO_DO"
+class_name = "ZeroDO"
+
+[[migrations]]
+tag = "v1"
+new_sqlite_classes = ["ZeroDO"]

package/src/config.ts

@@ -30,6 +30,8 @@ export interface ZeroLiteConfig {
   // storage controls
   checkpointIntervalMs: number // WAL checkpoint interval (default: 5min)
   maxLogFileSize: number // log rotation threshold in bytes (default: 2MB)
+  /** DO backend URL — replaces PGlite with Durable Object SQLite */
+  doBackendUrl?: string
   disableDiskLogs: boolean // skip writing logs to disk (default: false)
   // lifecycle hooks
   onDbReady?: Hook // after db+proxy ready, before zero-cache
@@ -103,6 +105,8 @@ export interface OrezConfig {
   maxLogFileSize?: number
   /** disable writing logs to disk (default: false) */
   disableDiskLogs?: boolean
+  /** DO backend URL — replaces PGlite with Durable Object SQLite */
+  doBackendUrl?: string
 }
 
 /** type-safe helper for orez.config.ts */
@@ -135,6 +139,7 @@ export function getConfig(overrides: Partial<ZeroLiteConfig> = {}): ZeroLiteConf
     checkpointIntervalMs: overrides.checkpointIntervalMs ?? 5 * 60 * 1000,
     maxLogFileSize: overrides.maxLogFileSize ?? 2 * 1024 * 1024,
     disableDiskLogs: overrides.disableDiskLogs ?? false,
+    doBackendUrl: overrides.doBackendUrl ?? process.env.DO_BACKEND_URL,
     onDbReady: overrides.onDbReady,
     onHealthy: overrides.onHealthy,
   }

package/src/do-sql-tracking.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from 'vitest'
+
+import { trackedChangeRow } from './do-sql-tracking.js'
+
+describe('trackedChangeRow', () => {
+  it('keeps only table columns and strips internal returning expressions', () => {
+    expect(
+      trackedChangeRow(
+        {
+          id: 't1',
+          body: 'hello',
+          __orez_returning_1: 'HELLO',
+          extra: 'client-visible only',
+        },
+        { rowColumns: ['id', 'body'] }
+      )
+    ).toEqual({ id: 't1', body: 'hello' })
+  })
+})

package/src/do-sql-tracking.ts

@@ -0,0 +1,19 @@
+export const RETURNING_INTERNAL_PREFIX = '__orez_returning_'
+
+export interface TrackedRowFilter {
+  rowColumns?: string[]
+}
+
+export function trackedChangeRow(
+  row: Record<string, unknown>,
+  track: TrackedRowFilter
+): Record<string, unknown> {
+  const allowed = track.rowColumns ? new Set(track.rowColumns) : null
+  const out: Record<string, unknown> = {}
+  for (const [key, value] of Object.entries(row)) {
+    if (key.startsWith(RETURNING_INTERNAL_PREFIX)) continue
+    if (allowed && !allowed.has(key)) continue
+    out[key] = value
+  }
+  return out
+}

package/src/index.ts

@@ -7,6 +7,7 @@
  */
 
 import { spawn, spawnSync, type ChildProcess } from 'node:child_process'
+import { randomUUID } from 'node:crypto'
 import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'
 import { resolve } from 'node:path'
 
@@ -24,6 +25,7 @@ import {
 } from './child-process.js'
 import { getConfig, getConnectionString } from './config.js'
 import { log, port, setLogLevel, setLogStore } from './log.js'
+import { DoBackend } from './pg-proxy-do-backend.js'
 import { startPgProxy } from './pg-proxy.js'
 import {
   createPGliteInstances,
@@ -60,6 +62,17 @@ import type { PGlite } from '@electric-sql/pglite'
 
 type ZeroChildProcess = ChildProcess & { __orezTail?: string[] }
 
+function ensureDoBackendNamespace(dataDir: string): string {
+  const marker = resolve(dataDir, 'do-backend-namespace')
+  if (existsSync(marker)) {
+    const existing = readFileSync(marker, 'utf8').trim()
+    if (existing) return existing
+  }
+  const next = randomUUID()
+  writeFileSync(marker, `${next}\n`)
+  return next
+}
+
 function resolveNodeBinary(): string {
   const explicitNode = process.env.NODE
   if (explicitNode && existsSync(explicitNode)) {
@@ -319,22 +332,60 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
   // single-db mode uses one instance for all databases (lighter for constrained envs).
   // otherwise, separate instances for postgres, zero_cvr, zero_cdb with optional
   // worker threads for non-blocking WASM execution.
-  const instances = config.singleDb
-    ? config.useWorkerThreads
-      ? await createSinglePGliteWorkerInstance(config)
-      : await createSinglePGliteInstance(config)
-    : config.useWorkerThreads
-      ? await createPGliteWorkerInstances(config)
-      : await createPGliteInstances(config)
-  const db = instances.postgres
-
-  // periodic WAL checkpoint to prevent pg_wal/ from growing unboundedly
-  const stopCheckpoint =
-    config.checkpointIntervalMs > 0
-      ? startPeriodicCheckpoint(instances, config.checkpointIntervalMs)
-      : () => {}
-
-  // config-based publications take precedence over env var
+
+  // ── DO backend path (replaces PGlite) ──────────────────────────────
+  let instances: any, db: any, stopCheckpoint: any
+  let migrationsApplied = 0
+  let isDoBackend = false
+
+  if (config.doBackendUrl) {
+    isDoBackend = true
+    log.orez(`using DO backend: ${config.doBackendUrl}`)
+    const backendUrl = config.doBackendUrl.replace(/\/+$/, '')
+    const doNamespace = ensureDoBackendNamespace(config.dataDir)
+    const doInstances = {
+      postgres: new DoBackend(backendUrl, 'postgres', doNamespace),
+      cvr: new DoBackend(backendUrl, 'zero_cvr', doNamespace),
+      cdb: new DoBackend(backendUrl, 'zero_cdb', doNamespace),
+      postgresReplicas: [],
+    }
+    await Promise.all([
+      doInstances.postgres.waitReady,
+      doInstances.cvr.waitReady,
+      doInstances.cdb.waitReady,
+    ])
+    instances = doInstances
+    db = doInstances.postgres
+    stopCheckpoint = () => {}
+  } else {
+    // ── PGlite backend (default) ────────────────────────────────────────────
+    instances = config.singleDb
+      ? config.useWorkerThreads
+        ? await createSinglePGliteWorkerInstance(config)
+        : await createSinglePGliteInstance(config)
+      : config.useWorkerThreads
+        ? await createPGliteWorkerInstances(config)
+        : await createPGliteInstances(config)
+    db = instances.postgres
+
+    // periodic WAL checkpoint
+    stopCheckpoint =
+      config.checkpointIntervalMs > 0
+        ? startPeriodicCheckpoint(instances, config.checkpointIntervalMs)
+        : () => {}
+
+    // config-based publications
+    if (config.zeroPublications && !process.env.ZERO_APP_PUBLICATIONS) {
+      process.env.ZERO_APP_PUBLICATIONS = config.zeroPublications
+    }
+
+    // run migrations & change tracking
+    migrationsApplied = await runMigrations(db, config)
+    log.debug.orez('installing change tracking')
+    await installChangeTracking(db)
+  }
+
+  // shared: publications config
   if (config.zeroPublications && !process.env.ZERO_APP_PUBLICATIONS) {
     process.env.ZERO_APP_PUBLICATIONS = config.zeroPublications
   }
@@ -343,14 +394,11 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
     log.debug.orez(`using managed publication: ${managedPub.names.join(', ')}`)
   }
 
-  // run migrations (on postgres instance only)
-  const migrationsApplied = await runMigrations(db, config)
+  // sync publications. for DO backend this goes through the TCP proxy, which
+  // rewrites the catalog queries and forwards CREATE PUBLICATION / ALTER PUBLICATION
+  // as no-ops or DO-native equivalents (PGlite still owns the real path).
   await syncManagedPublications(db, managedPub.names, managedPub.managedByOrez)
 
-  // install change tracking (on postgres instance only)
-  log.debug.orez('installing change tracking')
-  await installChangeTracking(db)
-
   // start tcp proxy (routes connections to correct instance by database name)
   const pgServer = await startPgProxy(instances, config)
 
@@ -375,17 +423,17 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
       OREZ_PG_PORT: String(config.pgPort),
     })
 
-    // re-sync publication membership after on-db-ready.
-    // for orez-managed publications, add any new public tables.
-    // for user-managed publications, ensure the publication isn't empty
-    // (app migrations may have created the pub with tables, but if orez
-    // pre-created an empty pub, the migration may have skipped adding tables).
+    // re-sync publication membership
     await syncManagedPublications(db, managedPub.names, managedPub.managedByOrez)
     await ensurePublicationHasTables(db, managedPub.names)
     log.debug.orez('re-installing change tracking after on-db-ready')
     await installChangeTracking(db)
   }
 
+  if (isDoBackend) {
+    await installChangeTracking(db)
+  }
+
   // write the ready marker so external orchestrators (e.g. CI scripts that
   // currently `wait:ports 6434`) can wait for orez to be fully initialized.
   // important: the pg port is bound earlier (startPgProxy above) so that
@@ -624,7 +672,7 @@ export async function startZeroLite(overrides: Partial<ZeroLiteConfig> = {}) {
 
         // remove stale zero shard schemas from upstream; these can outlive CVR/CDB
         // and cause dispatcher errors after full reset.
-        const shardSchemas = await db.query<{ schemaname: string }>(
+        const shardSchemas = await db.query(
           `SELECT DISTINCT schemaname
            FROM pg_tables
            WHERE tablename IN ('clients', 'replicas', 'mutations')

package/src/pg-proxy-browser.ts

@@ -656,7 +656,7 @@ function messagePortToDuplexWithInject(port: MessagePort): {
         chunk.byteOffset,
         chunk.byteOffset + chunk.byteLength
       ) as ArrayBuffer
-      port.postMessage(buf, [buf])
+      port.postMessage(buf)
     },
     close() {
       port.close()
@@ -668,7 +668,7 @@ function messagePortToDuplexWithInject(port: MessagePort): {
       data.byteOffset,
       data.byteOffset + data.byteLength
     ) as ArrayBuffer
-    port.postMessage(buf, [buf])
+    port.postMessage(buf)
   }
 
   const injectMessage = (data: Uint8Array) => {
@@ -714,12 +714,12 @@ function messagePortToDuplex(port: MessagePort): {
       if (isDebugWire() && _globalWriteCount <= 200) {
         console.debug(`[pg-proxy-ws-write] #${_globalWriteCount} len=${chunk.byteLength}`)
       }
-      // transfer the ArrayBuffer for zero-copy
+      // CF workerd MessagePort does not support transfer lists.
       const buf = chunk.buffer.slice(
         chunk.byteOffset,
         chunk.byteOffset + chunk.byteLength
       ) as ArrayBuffer
-      port.postMessage(buf, [buf])
+      port.postMessage(buf)
     },
     close() {
       port.close()
@@ -733,7 +733,7 @@ function messagePortToDuplex(port: MessagePort): {
       data.byteOffset,
       data.byteOffset + data.byteLength
     ) as ArrayBuffer
-    port.postMessage(buf, [buf])
+    port.postMessage(buf)
   }
 
   return { duplex: { readable, writable }, rawWrite }
@@ -1020,7 +1020,7 @@ export async function createBrowserProxy(
       // cause issues if the caller still references the original data
       const copy = new Uint8Array(data.length)
       copy.set(data)
-      port.postMessage(copy.buffer, [copy.buffer])
+      port.postMessage(copy.buffer)
     }
 
     // step 1: send AuthenticationClearTextPassword (R, type=3) — ask for password