orez 0.2.24 → 0.2.26

package/src/worker/cf-patches.ts

@@ -5,9 +5,12 @@
  * can run in SINGLE_PROCESS mode on CF Workers where dynamic import()
  * doesn't work.
  *
- * two patches:
+ * five patches:
  * 1. worker-urls.js — replace file:// URLs with zero-worker:// identifiers
- * 2. processes.js — replace dynamic import() with static worker module lookup
+ * 2. server worker entrypoints — disable CLI auto-start blocks
+ * 3. processes.js — replace dynamic import() with static worker module lookup
+ * 4. write-worker-client.js — run zero-cache's replica writer in-process
+ * 5. pgsql-parser — embed libpg-query wasm bytes for Workers
  *
  * usage in a post-build script:
  *
@@ -24,7 +27,10 @@ export function patchZeroCacheForCF(nodeModulesPath: string): void {
   const zcBase = resolve(nodeModulesPath, '@rocicorp', 'zero', 'out', 'zero-cache', 'src')
 
   patchWorkerUrls(zcBase)
+  patchWorkerEntrypoints(zcBase)
   patchProcesses(zcBase)
+  patchWriteWorkerClient(zcBase)
+  patchPgsqlParserWasm(nodeModulesPath)
 }
 
 function patchWorkerUrls(zcBase: string): void {
@@ -50,11 +56,49 @@ export const CHANGE_STREAMER_URL = u("change-streamer");
 export const REAPER_URL = u("reaper");
 export const REPLICATOR_URL = u("replicator");
 export const SYNCER_URL = u("syncer");
+// write-worker is spawned via 'new Worker()' (node:worker_threads), not via
+// childWorker() — it uses its own URL → worker resolution path. we still expose
+// it here so write-worker-client.js can import it without throwing.
+export const WRITE_WORKER_URL = u("write-worker");
 `
   )
   console.log('[orez] patched zero-cache worker-urls.js')
 }
 
+function patchWorkerEntrypoints(zcBase: string): void {
+  const entrypoints = ['main', 'change-streamer', 'reaper', 'replicator', 'syncer']
+
+  for (const entrypoint of entrypoints) {
+    const entrypointPath = resolve(zcBase, 'server', `${entrypoint}.js`)
+    if (!existsSync(entrypointPath)) {
+      console.warn('[orez] zero-cache worker entrypoint not found at', entrypointPath)
+      continue
+    }
+
+    let code = readFileSync(entrypointPath, 'utf-8')
+    if (code.includes('orez-disable-autostart')) {
+      continue
+    }
+
+    const next = code.replace(
+      /if \(!singleProcessMode\(\)\) exitAfter\(\(\) => runWorker\(must\(parentWorker\), process\.env(?:, \.\.\.process\.argv\.slice\(2\))?\)\);/g,
+      '// orez-disable-autostart: childWorker invokes runWorker explicitly in CF embeds.'
+    )
+
+    if (next === code) {
+      console.warn(
+        `[orez] could not find auto-start guard in ${entrypoint}.js. ` +
+          'zero-cache version may have changed — check compatibility.'
+      )
+      continue
+    }
+
+    code = next
+    writeFileSync(entrypointPath, code)
+    console.log(`[orez] patched zero-cache ${entrypoint}.js (disabled auto-start)`)
+  }
+}
+
 function patchProcesses(zcBase: string): void {
   const processesPath = resolve(zcBase, 'types', 'processes.js')
   if (!existsSync(processesPath)) {
@@ -71,6 +115,9 @@ function patchProcesses(zcBase: string): void {
 
   // add static imports of all zero-cache worker modules at the top.
   // these are relative to processes.js location in @rocicorp/zero.
+  // NOTE: mutator.js and write-worker.js don't export a default `runWorker`
+  // (mutator runs via auto-run guard, write-worker spawns via node:worker_threads
+  // not via childWorker()), so they're not in the lookup table.
   const workerImports = `\
 // patched by orez for CF Workers (static imports replace dynamic import())
 import { default as __zc_main } from "../server/main.js";
@@ -95,10 +142,11 @@ const __zc_workers = {
   const staticLookup =
     '((async () => { ' +
     'const _name = moduleUrl.hostname || moduleUrl.pathname.split("/").pop()?.replace(".js",""); ' +
+    'if (process.env.OREZ_DEBUG_WIRE === "1" || globalThis.__OREZ_DEBUG_WIRE__ === true) console.debug("[orez-zc-worker] start", _name, args); ' +
     'const runWorker = __zc_workers[_name]; ' +
     'if (!runWorker) throw new Error("orez: unknown zero-cache worker: " + _name + " (available: " + Object.keys(__zc_workers).join(", ") + ")"); ' +
-    'return { default: runWorker }; ' +
-    '})()).then(async ({ default: runWorker })'
+    'return { default: runWorker, name: _name }; ' +
+    '})()).then(async ({ default: runWorker, name })'
 
   if (!code.includes(dynamicImportPattern)) {
     console.warn(
@@ -112,3 +160,225 @@ const __zc_workers = {
   writeFileSync(processesPath, code)
   console.log('[orez] patched zero-cache processes.js (static worker imports)')
 }
+
+function patchWriteWorkerClient(zcBase: string): void {
+  const clientPath = resolve(zcBase, 'services', 'replicator', 'write-worker-client.js')
+  if (!existsSync(clientPath)) {
+    console.warn('[orez] write-worker-client.js not found at', clientPath)
+    return
+  }
+
+  let code = readFileSync(clientPath, 'utf-8')
+  if (
+    code.includes('orez-inline-write-worker') &&
+    code.includes('__orez_zero_sqlite_role')
+  ) {
+    return
+  }
+
+  if (
+    !code.includes('orez-inline-write-worker') &&
+    !code.includes('import { Worker } from "node:worker_threads";')
+  ) {
+    console.warn(
+      '[orez] could not find node:worker_threads import in write-worker-client.js. ' +
+        'zero-cache version may have changed — check compatibility.'
+    )
+    return
+  }
+
+  writeFileSync(
+    clientPath,
+    `// patched by orez for CF Workers (orez-inline-write-worker)
+import { must } from "../../../../shared/src/must.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { createLogContext } from "../../server/logging.js";
+import { StatementRunner } from "../../db/statements.js";
+import { getSubscriptionState } from "./schema/replication-state.js";
+import { ChangeProcessor } from "./change-processor.js";
+
+function applyPragmas(db, pragmas) {
+  db.pragma(\`busy_timeout = \${pragmas.busyTimeout}\`);
+  db.pragma(\`analysis_limit = \${pragmas.analysisLimit}\`);
+  if (pragmas.walAutocheckpoint !== void 0) {
+    db.pragma(\`wal_autocheckpoint = \${pragmas.walAutocheckpoint}\`);
+  }
+}
+
+function createAPI(onWriteError) {
+  let db;
+  let runner;
+  let processor;
+  let mode;
+  let lc;
+
+  function createProcessor() {
+    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {
+      onWriteError(err instanceof Error ? err : new Error(String(err)));
+    });
+  }
+
+  return {
+    init(dbPath, cpMode, pragmas, logConfig) {
+      lc = createLogContext({ log: logConfig }, { worker: "write-worker" });
+      const previousRole = globalThis.__orez_zero_sqlite_role;
+      globalThis.__orez_zero_sqlite_role = "replica-writer";
+      try {
+        db = new Database(lc, dbPath);
+      } finally {
+        if (previousRole === void 0) {
+          delete globalThis.__orez_zero_sqlite_role;
+        } else {
+          globalThis.__orez_zero_sqlite_role = previousRole;
+        }
+      }
+      applyPragmas(db, pragmas);
+      runner = new StatementRunner(db);
+      mode = cpMode;
+      createProcessor();
+    },
+    getSubscriptionState() {
+      return getSubscriptionState(must(runner));
+    },
+    processMessage(downstream) {
+      return must(processor).processMessage(must(lc), downstream);
+    },
+    abort() {
+      must(processor).abort(must(lc));
+      createProcessor();
+    },
+    stop() {
+      db?.close();
+      db = void 0;
+      runner = void 0;
+      processor = void 0;
+    },
+  };
+}
+
+class ThreadWriteWorkerClient {
+  #api;
+  #errorHandler = () => {};
+  #writeError = null;
+
+  constructor() {
+    this.#api = createAPI((err) => {
+      this.#writeError = err;
+      this.#errorHandler(err);
+    });
+  }
+
+  async #call(method, args) {
+    if (this.#writeError) throw this.#writeError;
+    try {
+      const result = this.#api[method](...args);
+      if (this.#writeError) throw this.#writeError;
+      return result;
+    } catch (err) {
+      throw err instanceof Error ? err : new Error(String(err));
+    }
+  }
+
+  init(dbPath, mode, pragmas, logConfig) {
+    return this.#call("init", [dbPath, mode, pragmas, logConfig]);
+  }
+
+  getSubscriptionState() {
+    return this.#call("getSubscriptionState", []);
+  }
+
+  processMessage(downstream) {
+    return this.#call("processMessage", [downstream]);
+  }
+
+  abort() {
+    if (!this.#writeError) {
+      try {
+        this.#api.abort();
+      } catch {
+      }
+    }
+  }
+
+  async stop() {
+    await this.#call("stop", []);
+  }
+
+  onError(handler) {
+    this.#errorHandler = handler;
+  }
+}
+
+export { ThreadWriteWorkerClient, applyPragmas };
+`
+  )
+  console.log('[orez] patched zero-cache write-worker-client.js (inline writer)')
+}
+
+function patchPgsqlParserWasm(nodeModulesPath: string): void {
+  const parserIndexPath = resolve(nodeModulesPath, 'libpg-query', 'wasm', 'index.js')
+  const wasmPath = resolve(nodeModulesPath, 'libpg-query', 'wasm', 'libpg-query.wasm')
+
+  if (!existsSync(parserIndexPath) || !existsSync(wasmPath)) {
+    console.warn('[orez] libpg-query wasm files not found under', nodeModulesPath)
+    return
+  }
+
+  let code = readFileSync(parserIndexPath, 'utf-8')
+  if (
+    code.includes('orez-libpg-query-wasm-binary') &&
+    code.includes('__orezLibPgQueryInit')
+  ) {
+    return
+  }
+  if (code.includes('orez-libpg-query-wasm-binary')) {
+    console.warn(
+      '[orez] libpg-query wasm loader already patched with an older shape. ' +
+        'Reinstall libpg-query or restore node_modules before re-patching.'
+    )
+    return
+  }
+
+  const pattern = 'const initPromise = PgQueryModule().then((module) => {'
+  if (!code.includes(pattern)) {
+    console.warn(
+      '[orez] could not find PgQueryModule init in libpg-query wasm index. ' +
+        'pgsql-parser version may have changed — check compatibility.'
+    )
+    return
+  }
+
+  const wasmBase64 = readFileSync(wasmPath).toString('base64')
+  const replacement = `\
+// orez-libpg-query-wasm-binary: embed parser wasm for CF Workers.
+const __orezLibPgQueryWasmBase64 = '${wasmBase64}';
+function __orezLibPgQueryWasmBinary() {
+    const decode = globalThis.atob
+        ? globalThis.atob(__orezLibPgQueryWasmBase64)
+        : Buffer.from(__orezLibPgQueryWasmBase64, 'base64').toString('binary');
+    const bytes = new Uint8Array(decode.length);
+    for (let i = 0; i < decode.length; i++) bytes[i] = decode.charCodeAt(i);
+    return bytes;
+}
+try {
+    const g = globalThis;
+    if (g.self && !g.self.location) g.self.location = { href: 'https://orez.local/libpg-query.js' };
+    if (!g.location) g.location = { href: 'https://orez.local/libpg-query.js' };
+}
+catch {
+}
+const __orezLibPgQueryPreviousProcessType = globalThis.process?.type;
+if (globalThis.process && !globalThis.process.type) globalThis.process.type = 'renderer';
+const __orezLibPgQueryInit = PgQueryModule({ wasmBinary: __orezLibPgQueryWasmBinary() });
+if (globalThis.process && __orezLibPgQueryPreviousProcessType === undefined) {
+    delete globalThis.process.type;
+}
+else if (globalThis.process) {
+    globalThis.process.type = __orezLibPgQueryPreviousProcessType;
+}
+const initPromise = __orezLibPgQueryInit.then((module) => {`
+
+  code = code.replace(pattern, replacement)
+  writeFileSync(parserIndexPath, code)
+  console.log('[orez] patched libpg-query wasm loader (embedded wasm bytes)')
+}

package/src/worker/shims/node-stub.ts

@@ -64,9 +64,38 @@ export function platform() {
 export function tmpdir() {
   return '/tmp'
 }
+export function homedir() {
+  return '/tmp'
+}
 export function availableParallelism() {
   return 1
 }
+export function loadavg() {
+  return [0, 0, 0]
+}
+export function uptime() {
+  return 0
+}
+export function totalmem() {
+  return 128 * 1024 * 1024
+}
+export function freemem() {
+  return 64 * 1024 * 1024
+}
+export function cpus() {
+  return [{ model: 'worker', speed: 0 }]
+}
+export function networkInterfaces() {
+  return {
+    lo: [
+      {
+        address: '127.0.0.1',
+        family: 'IPv4',
+        internal: true,
+      },
+    ],
+  }
+}
 
 // stub for node:crypto
 export function timingSafeEqual(a: unknown, b: unknown) {
@@ -94,7 +123,23 @@ export class Session {
 
 // stub for node:v8
 export function getHeapStatistics() {
-  return { total_heap_size: 0, used_heap_size: 0 }
+  const heapSizeLimit = 128 * 1024 * 1024
+  return {
+    total_heap_size: 64 * 1024 * 1024,
+    total_heap_size_executable: 0,
+    total_physical_size: 64 * 1024 * 1024,
+    total_available_size: 64 * 1024 * 1024,
+    used_heap_size: 32 * 1024 * 1024,
+    heap_size_limit: heapSizeLimit,
+    malloced_memory: 0,
+    peak_malloced_memory: 0,
+    does_zap_garbage: 0,
+    number_of_native_contexts: 1,
+    number_of_detached_contexts: 0,
+    total_global_handles_size: 0,
+    used_global_handles_size: 0,
+    external_memory: 0,
+  }
 }
 
 // stub for node:zlib
@@ -197,9 +242,16 @@ export default {
   hostname,
   platform,
   tmpdir,
+  homedir,
   availableParallelism,
+  loadavg,
+  uptime,
+  totalmem,
+  freemem,
+  cpus,
   arch,
   release,
+  networkInterfaces,
   existsSync,
   readFileSync,
   writeFileSync,

package/src/worker/shims/oxfmt.ts

@@ -0,0 +1,3 @@
+export async function format(_file: string, content: string) {
+  return { code: content }
+}

package/src/worker/shims/postgres-socket.ts

@@ -151,7 +151,7 @@ class MessagePortSocket extends EventEmitter {
     copy.set(bytes)
 
     try {
-      this.port.postMessage(copy.buffer, [copy.buffer])
+      this.port.postMessage(copy.buffer)
     } catch (err) {
       queueMicrotask(() => this.emit('error', err))
       if (typeof encoding === 'function') encoding()

package/src/worker/shims/sqlite.test.ts

@@ -139,6 +139,7 @@ describe('Database', () => {
   })
 
   afterEach(() => {
+    db.close()
     mock._nativeDb.close()
   })
 
@@ -189,6 +190,7 @@ describe('Database.exec', () => {
   })
 
   afterEach(() => {
+    db.close()
     mock._nativeDb.close()
   })
 
@@ -229,6 +231,7 @@ describe('Database.prepare / Statement', () => {
   })
 
   afterEach(() => {
+    db.close()
     mock._nativeDb.close()
   })
 
@@ -552,6 +555,148 @@ describe('StatementRunner', () => {
   })
 })
 
+describe('DO snapshot transactions', () => {
+  let mock: SqlStorageLike & { _nativeDb: any; transactionSync: <T>(fn: () => T) => T }
+  let live: Database
+  let snapshot: Database
+
+  beforeEach(() => {
+    mock = createMockSqlStorage() as SqlStorageLike & {
+      _nativeDb: any
+      transactionSync: <T>(fn: () => T) => T
+    }
+    mock.transactionSync = (fn) => fn()
+    live = new Database(mock)
+    snapshot = new Database(mock)
+    live.exec('CREATE TABLE todo (id TEXT PRIMARY KEY, title TEXT, _0_version TEXT)')
+    live.prepare('INSERT INTO todo VALUES (?, ?, ?)').run('1', 'old', '01')
+  })
+
+  afterEach(() => {
+    live.close()
+    snapshot.close()
+    mock._nativeDb.close()
+  })
+
+  it('keeps BEGIN CONCURRENT reads stable until rollback', () => {
+    snapshot.prepare('BEGIN CONCURRENT').run()
+    expect(snapshot.prepare('SELECT title FROM todo WHERE id = ?').get('1')).toEqual({
+      title: 'old',
+    })
+
+    live
+      .prepare('UPDATE todo SET title = ?, _0_version = ? WHERE id = ?')
+      .run('new', '02', '1')
+
+    expect(live.prepare('SELECT title FROM todo WHERE id = ?').get('1')).toEqual({
+      title: 'new',
+    })
+    expect(snapshot.prepare('SELECT title FROM todo WHERE id = ?').get('1')).toEqual({
+      title: 'old',
+    })
+
+    snapshot.prepare('ROLLBACK').run()
+    expect(snapshot.prepare('SELECT title FROM todo WHERE id = ?').get('1')).toEqual({
+      title: 'new',
+    })
+  })
+
+  it('hides snapshot tables from sqlite catalog queries', () => {
+    snapshot.prepare('BEGIN CONCURRENT').run()
+
+    expect(
+      live
+        .prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name")
+        .all()
+    ).toEqual([{ name: 'todo' }])
+  })
+
+  it('does not rewrite table names inside string literals during snapshots', () => {
+    live.exec('CREATE TABLE log (id TEXT PRIMARY KEY, table_name TEXT, _0_version TEXT)')
+    live.prepare('INSERT INTO log VALUES (?, ?, ?)').run('1', 'todo', '01')
+    live.prepare('INSERT INTO log VALUES (?, ?, ?)').run('2', '"todo"', '02')
+
+    snapshot.prepare('BEGIN CONCURRENT').run()
+
+    expect(
+      snapshot.prepare("SELECT table_name FROM log WHERE table_name = 'todo'").get()
+    ).toEqual({ table_name: 'todo' })
+    expect(
+      snapshot.prepare('SELECT table_name FROM log WHERE table_name = ?').get('"todo"')
+    ).toEqual({ table_name: '"todo"' })
+  })
+
+  it('cleans inactive snapshot tables when opening a database', () => {
+    mock.exec('CREATE TABLE _orez_snapshot_1_todo AS SELECT * FROM todo')
+    expect(
+      mock
+        .exec("SELECT name FROM sqlite_master WHERE name = '_orez_snapshot_1_todo'")
+        .toArray()
+    ).toEqual([{ name: '_orez_snapshot_1_todo' }])
+
+    const reopened = new Database(mock)
+    try {
+      expect(
+        mock
+          .exec("SELECT name FROM sqlite_master WHERE name = '_orez_snapshot_1_todo'")
+          .toArray()
+      ).toEqual([])
+    } finally {
+      reopened.close()
+    }
+  })
+
+  it('does not remove another open connection snapshot', () => {
+    snapshot.prepare('BEGIN CONCURRENT').run()
+    const snapshotTables = mock
+      .exec("SELECT name FROM sqlite_master WHERE name LIKE '_orez_snapshot_%'")
+      .toArray()
+    expect(snapshotTables).toHaveLength(1)
+
+    const other = new Database(mock)
+    try {
+      live
+        .prepare('UPDATE todo SET title = ?, _0_version = ? WHERE id = ?')
+        .run('new', '02', '1')
+
+      expect(snapshot.prepare('SELECT title FROM todo WHERE id = ?').get('1')).toEqual({
+        title: 'old',
+      })
+    } finally {
+      other.close()
+    }
+  })
+
+  it('persists BEGIN CONCURRENT writes for the zero-cache replica writer', () => {
+    const globalObject = globalThis as any
+    const previousRole = globalObject.__orez_zero_sqlite_role
+    globalObject.__orez_zero_sqlite_role = 'replica-writer'
+    let writer: Database
+    try {
+      writer = new Database(mock)
+    } finally {
+      if (previousRole === undefined) {
+        delete globalObject.__orez_zero_sqlite_role
+      } else {
+        globalObject.__orez_zero_sqlite_role = previousRole
+      }
+    }
+
+    writer.prepare('BEGIN CONCURRENT').run()
+    writer.prepare('INSERT INTO todo VALUES (?, ?, ?)').run('2', 'writer row', '02')
+    writer.prepare('COMMIT').run()
+
+    expect(live.prepare('SELECT title FROM todo WHERE id = ?').get('2')).toEqual({
+      title: 'writer row',
+    })
+    expect(
+      live
+        .prepare("SELECT name FROM sqlite_master WHERE name LIKE '_orez_snapshot_%'")
+        .all()
+    ).toEqual([])
+  })
+})
+
 describe('StatementRunner: zero-cache replicator pattern', () => {
   let mock: SqlStorageLike & { _nativeDb: any }
   let db: Database