orez 0.2.26 → 0.2.29

package/src/worker/cf-patches.ts

@@ -1,384 +0,0 @@
-/**
- * zero-cache CF Workers patches.
- *
- * applies patches to @rocicorp/zero's internal files so zero-cache
- * can run in SINGLE_PROCESS mode on CF Workers where dynamic import()
- * doesn't work.
- *
- * five patches:
- * 1. worker-urls.js — replace file:// URLs with zero-worker:// identifiers
- * 2. server worker entrypoints — disable CLI auto-start blocks
- * 3. processes.js — replace dynamic import() with static worker module lookup
- * 4. write-worker-client.js — run zero-cache's replica writer in-process
- * 5. pgsql-parser — embed libpg-query wasm bytes for Workers
- *
- * usage in a post-build script:
- *
- *   import { patchZeroCacheForCF } from 'orez/worker/cf-patches'
- *   patchZeroCacheForCF('./node_modules')
- *
- * idempotent: safe to run multiple times.
- */
-
-import { readFileSync, writeFileSync, existsSync } from 'node:fs'
-import { resolve } from 'node:path'
-
-export function patchZeroCacheForCF(nodeModulesPath: string): void {
-  const zcBase = resolve(nodeModulesPath, '@rocicorp', 'zero', 'out', 'zero-cache', 'src')
-
-  patchWorkerUrls(zcBase)
-  patchWorkerEntrypoints(zcBase)
-  patchProcesses(zcBase)
-  patchWriteWorkerClient(zcBase)
-  patchPgsqlParserWasm(nodeModulesPath)
-}
-
-function patchWorkerUrls(zcBase: string): void {
-  const workerUrlsPath = resolve(zcBase, 'server', 'worker-urls.js')
-  if (!existsSync(workerUrlsPath)) {
-    console.warn('[orez] worker-urls.js not found at', workerUrlsPath)
-    return
-  }
-
-  const content = readFileSync(workerUrlsPath, 'utf-8')
-
-  // skip if already patched
-  if (content.includes('zero-worker://')) {
-    return
-  }
-
-  writeFileSync(
-    workerUrlsPath,
-    `// patched by orez for CF Workers (replaces file:// URLs with identifiers)
-const u = (n) => new URL("zero-worker://" + n);
-export const MAIN_URL = u("main");
-export const CHANGE_STREAMER_URL = u("change-streamer");
-export const REAPER_URL = u("reaper");
-export const REPLICATOR_URL = u("replicator");
-export const SYNCER_URL = u("syncer");
-// write-worker is spawned via 'new Worker()' (node:worker_threads), not via
-// childWorker() — it uses its own URL → worker resolution path. we still expose
-// it here so write-worker-client.js can import it without throwing.
-export const WRITE_WORKER_URL = u("write-worker");
-`
-  )
-  console.log('[orez] patched zero-cache worker-urls.js')
-}
-
-function patchWorkerEntrypoints(zcBase: string): void {
-  const entrypoints = ['main', 'change-streamer', 'reaper', 'replicator', 'syncer']
-
-  for (const entrypoint of entrypoints) {
-    const entrypointPath = resolve(zcBase, 'server', `${entrypoint}.js`)
-    if (!existsSync(entrypointPath)) {
-      console.warn('[orez] zero-cache worker entrypoint not found at', entrypointPath)
-      continue
-    }
-
-    let code = readFileSync(entrypointPath, 'utf-8')
-    if (code.includes('orez-disable-autostart')) {
-      continue
-    }
-
-    const next = code.replace(
-      /if \(!singleProcessMode\(\)\) exitAfter\(\(\) => runWorker\(must\(parentWorker\), process\.env(?:, \.\.\.process\.argv\.slice\(2\))?\)\);/g,
-      '// orez-disable-autostart: childWorker invokes runWorker explicitly in CF embeds.'
-    )
-
-    if (next === code) {
-      console.warn(
-        `[orez] could not find auto-start guard in ${entrypoint}.js. ` +
-          'zero-cache version may have changed — check compatibility.'
-      )
-      continue
-    }
-
-    code = next
-    writeFileSync(entrypointPath, code)
-    console.log(`[orez] patched zero-cache ${entrypoint}.js (disabled auto-start)`)
-  }
-}
-
-function patchProcesses(zcBase: string): void {
-  const processesPath = resolve(zcBase, 'types', 'processes.js')
-  if (!existsSync(processesPath)) {
-    console.warn('[orez] processes.js not found at', processesPath)
-    return
-  }
-
-  let code = readFileSync(processesPath, 'utf-8')
-
-  // skip if already patched
-  if (code.includes('__zc_workers')) {
-    return
-  }
-
-  // add static imports of all zero-cache worker modules at the top.
-  // these are relative to processes.js location in @rocicorp/zero.
-  // NOTE: mutator.js and write-worker.js don't export a default `runWorker`
-  // (mutator runs via auto-run guard, write-worker spawns via node:worker_threads
-  // not via childWorker()), so they're not in the lookup table.
-  const workerImports = `\
-// patched by orez for CF Workers (static imports replace dynamic import())
-import { default as __zc_main } from "../server/main.js";
-import { default as __zc_change_streamer } from "../server/change-streamer.js";
-import { default as __zc_reaper } from "../server/reaper.js";
-import { default as __zc_replicator } from "../server/replicator.js";
-import { default as __zc_syncer } from "../server/syncer.js";
-const __zc_workers = {
-  "main": __zc_main,
-  "change-streamer": __zc_change_streamer,
-  "reaper": __zc_reaper,
-  "replicator": __zc_replicator,
-  "syncer": __zc_syncer,
-};
-`
-
-  // replace the dynamic import in childWorker with a synchronous lookup.
-  // original: import(moduleUrl.href).then(async ({ default: runWorker }) => ...
-  // patched:  lookup __zc_workers by name, then continue as before
-  const dynamicImportPattern =
-    'import(moduleUrl.href).then(async ({ default: runWorker })'
-  const staticLookup =
-    '((async () => { ' +
-    'const _name = moduleUrl.hostname || moduleUrl.pathname.split("/").pop()?.replace(".js",""); ' +
-    'if (process.env.OREZ_DEBUG_WIRE === "1" || globalThis.__OREZ_DEBUG_WIRE__ === true) console.debug("[orez-zc-worker] start", _name, args); ' +
-    'const runWorker = __zc_workers[_name]; ' +
-    'if (!runWorker) throw new Error("orez: unknown zero-cache worker: " + _name + " (available: " + Object.keys(__zc_workers).join(", ") + ")"); ' +
-    'return { default: runWorker, name: _name }; ' +
-    '})()).then(async ({ default: runWorker, name })'
-
-  if (!code.includes(dynamicImportPattern)) {
-    console.warn(
-      '[orez] could not find dynamic import pattern in processes.js. ' +
-        'zero-cache version may have changed — check compatibility.'
-    )
-    return
-  }
-
-  code = workerImports + code.replace(dynamicImportPattern, staticLookup)
-  writeFileSync(processesPath, code)
-  console.log('[orez] patched zero-cache processes.js (static worker imports)')
-}
-
-function patchWriteWorkerClient(zcBase: string): void {
-  const clientPath = resolve(zcBase, 'services', 'replicator', 'write-worker-client.js')
-  if (!existsSync(clientPath)) {
-    console.warn('[orez] write-worker-client.js not found at', clientPath)
-    return
-  }
-
-  let code = readFileSync(clientPath, 'utf-8')
-  if (
-    code.includes('orez-inline-write-worker') &&
-    code.includes('__orez_zero_sqlite_role')
-  ) {
-    return
-  }
-
-  if (
-    !code.includes('orez-inline-write-worker') &&
-    !code.includes('import { Worker } from "node:worker_threads";')
-  ) {
-    console.warn(
-      '[orez] could not find node:worker_threads import in write-worker-client.js. ' +
-        'zero-cache version may have changed — check compatibility.'
-    )
-    return
-  }
-
-  writeFileSync(
-    clientPath,
-    `// patched by orez for CF Workers (orez-inline-write-worker)
-import { must } from "../../../../shared/src/must.js";
-import { Database } from "../../../../zqlite/src/db.js";
-import { createLogContext } from "../../server/logging.js";
-import { StatementRunner } from "../../db/statements.js";
-import { getSubscriptionState } from "./schema/replication-state.js";
-import { ChangeProcessor } from "./change-processor.js";
-
-function applyPragmas(db, pragmas) {
-  db.pragma(\`busy_timeout = \${pragmas.busyTimeout}\`);
-  db.pragma(\`analysis_limit = \${pragmas.analysisLimit}\`);
-  if (pragmas.walAutocheckpoint !== void 0) {
-    db.pragma(\`wal_autocheckpoint = \${pragmas.walAutocheckpoint}\`);
-  }
-}
-
-function createAPI(onWriteError) {
-  let db;
-  let runner;
-  let processor;
-  let mode;
-  let lc;
-
-  function createProcessor() {
-    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {
-      onWriteError(err instanceof Error ? err : new Error(String(err)));
-    });
-  }
-
-  return {
-    init(dbPath, cpMode, pragmas, logConfig) {
-      lc = createLogContext({ log: logConfig }, { worker: "write-worker" });
-      const previousRole = globalThis.__orez_zero_sqlite_role;
-      globalThis.__orez_zero_sqlite_role = "replica-writer";
-      try {
-        db = new Database(lc, dbPath);
-      } finally {
-        if (previousRole === void 0) {
-          delete globalThis.__orez_zero_sqlite_role;
-        } else {
-          globalThis.__orez_zero_sqlite_role = previousRole;
-        }
-      }
-      applyPragmas(db, pragmas);
-      runner = new StatementRunner(db);
-      mode = cpMode;
-      createProcessor();
-    },
-    getSubscriptionState() {
-      return getSubscriptionState(must(runner));
-    },
-    processMessage(downstream) {
-      return must(processor).processMessage(must(lc), downstream);
-    },
-    abort() {
-      must(processor).abort(must(lc));
-      createProcessor();
-    },
-    stop() {
-      db?.close();
-      db = void 0;
-      runner = void 0;
-      processor = void 0;
-    },
-  };
-}
-
-class ThreadWriteWorkerClient {
-  #api;
-  #errorHandler = () => {};
-  #writeError = null;
-
-  constructor() {
-    this.#api = createAPI((err) => {
-      this.#writeError = err;
-      this.#errorHandler(err);
-    });
-  }
-
-  async #call(method, args) {
-    if (this.#writeError) throw this.#writeError;
-    try {
-      const result = this.#api[method](...args);
-      if (this.#writeError) throw this.#writeError;
-      return result;
-    } catch (err) {
-      throw err instanceof Error ? err : new Error(String(err));
-    }
-  }
-
-  init(dbPath, mode, pragmas, logConfig) {
-    return this.#call("init", [dbPath, mode, pragmas, logConfig]);
-  }
-
-  getSubscriptionState() {
-    return this.#call("getSubscriptionState", []);
-  }
-
-  processMessage(downstream) {
-    return this.#call("processMessage", [downstream]);
-  }
-
-  abort() {
-    if (!this.#writeError) {
-      try {
-        this.#api.abort();
-      } catch {
-      }
-    }
-  }
-
-  async stop() {
-    await this.#call("stop", []);
-  }
-
-  onError(handler) {
-    this.#errorHandler = handler;
-  }
-}
-
-export { ThreadWriteWorkerClient, applyPragmas };
-`
-  )
-  console.log('[orez] patched zero-cache write-worker-client.js (inline writer)')
-}
-
-function patchPgsqlParserWasm(nodeModulesPath: string): void {
-  const parserIndexPath = resolve(nodeModulesPath, 'libpg-query', 'wasm', 'index.js')
-  const wasmPath = resolve(nodeModulesPath, 'libpg-query', 'wasm', 'libpg-query.wasm')
-
-  if (!existsSync(parserIndexPath) || !existsSync(wasmPath)) {
-    console.warn('[orez] libpg-query wasm files not found under', nodeModulesPath)
-    return
-  }
-
-  let code = readFileSync(parserIndexPath, 'utf-8')
-  if (
-    code.includes('orez-libpg-query-wasm-binary') &&
-    code.includes('__orezLibPgQueryInit')
-  ) {
-    return
-  }
-  if (code.includes('orez-libpg-query-wasm-binary')) {
-    console.warn(
-      '[orez] libpg-query wasm loader already patched with an older shape. ' +
-        'Reinstall libpg-query or restore node_modules before re-patching.'
-    )
-    return
-  }
-
-  const pattern = 'const initPromise = PgQueryModule().then((module) => {'
-  if (!code.includes(pattern)) {
-    console.warn(
-      '[orez] could not find PgQueryModule init in libpg-query wasm index. ' +
-        'pgsql-parser version may have changed — check compatibility.'
-    )
-    return
-  }
-
-  const wasmBase64 = readFileSync(wasmPath).toString('base64')
-  const replacement = `\
-// orez-libpg-query-wasm-binary: embed parser wasm for CF Workers.
-const __orezLibPgQueryWasmBase64 = '${wasmBase64}';
-function __orezLibPgQueryWasmBinary() {
-    const decode = globalThis.atob
-        ? globalThis.atob(__orezLibPgQueryWasmBase64)
-        : Buffer.from(__orezLibPgQueryWasmBase64, 'base64').toString('binary');
-    const bytes = new Uint8Array(decode.length);
-    for (let i = 0; i < decode.length; i++) bytes[i] = decode.charCodeAt(i);
-    return bytes;
-}
-try {
-    const g = globalThis;
-    if (g.self && !g.self.location) g.self.location = { href: 'https://orez.local/libpg-query.js' };
-    if (!g.location) g.location = { href: 'https://orez.local/libpg-query.js' };
-}
-catch {
-}
-const __orezLibPgQueryPreviousProcessType = globalThis.process?.type;
-if (globalThis.process && !globalThis.process.type) globalThis.process.type = 'renderer';
-const __orezLibPgQueryInit = PgQueryModule({ wasmBinary: __orezLibPgQueryWasmBinary() });
-if (globalThis.process && __orezLibPgQueryPreviousProcessType === undefined) {
-    delete globalThis.process.type;
-}
-else if (globalThis.process) {
-    globalThis.process.type = __orezLibPgQueryPreviousProcessType;
-}
-const initPromise = __orezLibPgQueryInit.then((module) => {`
-
-  code = code.replace(pattern, replacement)
-  writeFileSync(parserIndexPath, code)
-  console.log('[orez] patched libpg-query wasm loader (embedded wasm bytes)')
-}

package/src/worker/embed-integration.test.ts

@@ -1,321 +0,0 @@
-/**
- * integration test for zero-cache embedded mode.
- *
- * validates that zero-cache can run in-process with SINGLE_PROCESS=1,
- * connected to PGlite via the TCP proxy. this is the same pipeline as
- * the full integration test but without child_process.fork().
- *
- * test flow:
- * 1. create PGlite instances (postgres, cvr, cdb)
- * 2. start TCP proxy
- * 3. start zero-cache in-process via startZeroCacheEmbed()
- * 4. connect WebSocket client and verify sync
- */
-
-import { mkdirSync } from 'node:fs'
-import { resolve } from 'node:path'
-
-import { describe, test, expect, beforeAll, afterAll } from 'vitest'
-import WebSocket from 'ws'
-
-import { getConfig, getConnectionString } from '../config.js'
-import {
-  ensureTablesInPublications,
-  installAllowAllPermissions,
-} from '../integration/test-permissions.js'
-import { startPgProxy } from '../pg-proxy.js'
-import { createPGliteInstances, type PGliteInstances } from '../pglite-manager.js'
-import { installChangeTracking } from '../replication/change-tracker.js'
-import { startZeroCacheEmbed, type ZeroCacheEmbed } from './zero-cache-embed.js'
-
-import type { PGlite } from '@electric-sql/pglite'
-
-const SYNC_PROTOCOL_VERSION = 49
-
-function encodeSecProtocols(
-  initConnectionMessage: unknown,
-  authToken: string | undefined
-): string {
-  const payload = JSON.stringify({ initConnectionMessage, authToken })
-  return encodeURIComponent(Buffer.from(payload, 'utf-8').toString('base64'))
-}
-
-describe('zero-cache embed integration', { timeout: 120000 }, () => {
-  let db: PGlite
-  let instances: PGliteInstances
-  let pgServer: ReturnType<Awaited<ReturnType<typeof startPgProxy>>>
-  let embed: ZeroCacheEmbed
-  let zeroPort: number
-  let pgPort: number
-  let dataDir: string
-
-  beforeAll(async () => {
-    // use random ports to avoid conflicts with other tests
-    pgPort = 24000 + Math.floor(Math.random() * 1000)
-    zeroPort = pgPort + 100
-
-    dataDir = `.orez-embed-test-${Date.now()}`
-
-    const config = getConfig({
-      pgPort,
-      zeroPort,
-      dataDir,
-      logLevel: 'info',
-      useWorkerThreads: false,
-      singleDb: false,
-    })
-
-    mkdirSync(dataDir, { recursive: true })
-
-    // create PGlite instances
-    instances = await createPGliteInstances(config)
-    db = instances.postgres
-
-    // create test table
-    await db.exec(`
-        CREATE TABLE IF NOT EXISTS foo (
-          id TEXT PRIMARY KEY,
-          value TEXT,
-          num INTEGER
-        )
-      `)
-
-    // set up publications
-    const pubName = `orez_zero_public`
-    process.env.ZERO_APP_PUBLICATIONS = pubName
-    await db.exec(`CREATE PUBLICATION "${pubName}"`).catch(() => {})
-    await db
-      .exec(`ALTER PUBLICATION "${pubName}" ADD TABLE "public"."foo"`)
-      .catch(() => {})
-
-    // install change tracking
-    await installChangeTracking(db)
-
-    // install allow-all permissions for test
-    await installAllowAllPermissions(db, ['foo'])
-    await ensureTablesInPublications(db, ['foo'])
-
-    // start TCP proxy
-    pgServer = await startPgProxy(instances, config)
-
-    // start zero-cache in-process
-    const upstreamDb = getConnectionString(config, 'postgres')
-    const cvrDb = getConnectionString(config, 'zero_cvr')
-    const changeDb = getConnectionString(config, 'zero_cdb')
-    const replicaFile = resolve(dataDir, 'zero-replica.db')
-
-    console.log(`[embed-test] starting in-process zero-cache on port ${zeroPort}`)
-    console.log(`[embed-test] upstream: ${upstreamDb}`)
-
-    embed = await startZeroCacheEmbed({
-      pglite: db,
-      upstreamDb,
-      cvrDb,
-      changeDb,
-      replicaFile,
-      port: zeroPort,
-      publications: [pubName],
-      env: {
-        ZERO_LOG_LEVEL: 'info',
-      },
-    })
-
-    console.log(`[embed-test] zero-cache ready on port ${embed.port}`)
-
-    // wait for HTTP health check
-    await waitForZero(zeroPort, 30000)
-    console.log(`[embed-test] health check passed`)
-  }, 120000)
-
-  afterAll(async () => {
-    if (embed) await embed.stop()
-    if (pgServer) pgServer.close()
-    if (instances) {
-      await instances.postgres.close().catch(() => {})
-      await instances.cvr.close().catch(() => {})
-      await instances.cdb.close().catch(() => {})
-    }
-    if (dataDir) {
-      const { rmSync } = await import('node:fs')
-      try {
-        rmSync(dataDir, { recursive: true, force: true })
-      } catch {}
-    }
-  })
-
-  test('zero-cache is ready', () => {
-    expect(embed.ready).toBe(true)
-    expect(embed.port).toBe(zeroPort)
-  })
-
-  test('accepts WebSocket connections', async () => {
-    const cg = `test-cg-${Date.now()}`
-    const cid = `test-client-${Date.now()}`
-    const secProtocol = encodeSecProtocols(
-      [
-        'initConnection',
-        {
-          desiredQueriesPatch: [],
-          clientSchema: {
-            tables: {
-              foo: {
-                columns: {
-                  id: { type: 'string' },
-                  value: { type: 'string' },
-                  num: { type: 'number' },
-                },
-                primaryKey: ['id'],
-              },
-            },
-          },
-        },
-      ],
-      undefined
-    )
-    const ws = new WebSocket(
-      `ws://localhost:${zeroPort}/sync/v${SYNC_PROTOCOL_VERSION}/connect` +
-        `?clientGroupID=${cg}&clientID=${cid}&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`,
-      secProtocol
-    )
-
-    // collect messages — attach listener before open to catch everything
-    const messages: unknown[] = []
-    ws.on('message', (data) => {
-      messages.push(JSON.parse(data.toString()))
-    })
-
-    const connected = new Promise<void>((resolve, reject) => {
-      ws.on('open', resolve)
-      ws.on('error', reject)
-      setTimeout(() => reject(new Error('ws connect timeout')), 10000)
-    })
-
-    await connected
-
-    // wait for messages to arrive
-    const deadline = Date.now() + 10000
-    while (
-      Date.now() < deadline &&
-      !messages.some((m) => Array.isArray(m) && m[0] === 'connected')
-    ) {
-      await new Promise((r) => setTimeout(r, 100))
-    }
-
-    const connectedMsg = messages.find((m) => Array.isArray(m) && m[0] === 'connected')
-    expect(connectedMsg).toMatchObject(['connected', { wsid: 'ws1' }])
-    ws.close()
-  })
-
-  test('live replication: insert triggers poke', async () => {
-    // insert data
-    await db.query(`INSERT INTO foo (id, value, num) VALUES ($1, $2, $3)`, [
-      'embed-row',
-      'hello-embed',
-      42,
-    ])
-
-    // wait for replication to deliver
-    await waitForReplicationCatchup(db)
-    await new Promise((r) => setTimeout(r, 1000))
-
-    // connect and subscribe
-    const downstream: unknown[] = []
-    const ws = connectAndSubscribe(zeroPort, downstream)
-
-    // wait for poke with our data
-    const deadline = Date.now() + 30000
-    let found = false
-    while (Date.now() < deadline && !found) {
-      await new Promise((r) => setTimeout(r, 500))
-      for (const msg of downstream) {
-        if (Array.isArray(msg) && msg[0] === 'pokePart' && msg[1]?.rowsPatch) {
-          for (const patch of msg[1].rowsPatch) {
-            if (
-              patch.op === 'put' &&
-              patch.tableName === 'foo' &&
-              patch.value?.id === 'embed-row'
-            ) {
-              found = true
-              break
-            }
-          }
-        }
-      }
-    }
-
-    ws.close()
-    expect(found).toBe(true)
-  })
-})
-
-function connectAndSubscribe(port: number, downstream: unknown[]): WebSocket {
-  const cg = `test-cg-${Date.now()}`
-  const cid = `test-client-${Date.now()}`
-  const secProtocol = encodeSecProtocols(
-    [
-      'initConnection',
-      {
-        desiredQueriesPatch: [
-          {
-            op: 'put',
-            hash: 'q1',
-            ast: {
-              table: 'foo',
-              orderBy: [['id', 'asc']],
-            },
-          },
-        ],
-        clientSchema: {
-          tables: {
-            foo: {
-              columns: {
-                id: { type: 'string' },
-                value: { type: 'string' },
-                num: { type: 'number' },
-              },
-              primaryKey: ['id'],
-            },
-          },
-        },
-      },
-    ],
-    undefined
-  )
-  const ws = new WebSocket(
-    `ws://localhost:${port}/sync/v${SYNC_PROTOCOL_VERSION}/connect` +
-      `?clientGroupID=${cg}&clientID=${cid}&wsid=ws1&schemaVersion=1&baseCookie=&ts=${Date.now()}&lmid=0`,
-    secProtocol
-  )
-
-  ws.on('message', (data) => {
-    downstream.push(JSON.parse(data.toString()))
-  })
-
-  return ws
-}
-
-async function waitForReplicationCatchup(
-  pglite: PGlite,
-  timeoutMs = 15000
-): Promise<void> {
-  const deadline = Date.now() + timeoutMs
-  while (Date.now() < deadline) {
-    const result = await pglite.query<{ count: string }>(
-      `SELECT count(*)::text as count FROM _orez._zero_changes`
-    )
-    if (Number(result.rows[0]?.count) === 0) return
-    await new Promise((r) => setTimeout(r, 100))
-  }
-}
-
-async function waitForZero(port: number, timeoutMs = 30000) {
-  const deadline = Date.now() + timeoutMs
-  while (Date.now() < deadline) {
-    try {
-      const res = await fetch(`http://localhost:${port}/`)
-      if (res.ok || res.status === 404) return
-    } catch {}
-    await new Promise((r) => setTimeout(r, 500))
-  }
-  throw new Error(`zero-cache not ready on port ${port} after ${timeoutMs}ms`)
-}