orez 0.2.25 → 0.2.27

package/src/pg-sqlite-compiler/test/oracle.ts

@@ -0,0 +1,237 @@
+/**
+ * pgsqlite-backed oracle for compiler quality.
+ *
+ * Spawns a real pgsqlite server, sends PG SQL via the PG wire protocol,
+ * captures the result set. Compares against running the same query through
+ * our compiler + bun:sqlite. Equivalence at the result-set level is what we
+ * actually care about — pgsqlite is the oracle, not the spec.
+ *
+ * The pgsqlite binary path comes from `vendor/pgsqlite/.resolved-path`,
+ * populated by `scripts/pgsqlite/ensure.ts`. If empty, oracle tests should
+ * be marked `it.skip` so the suite still runs without pgsqlite installed.
+ */
+import { spawn, type ChildProcess } from 'node:child_process'
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { createConnection, createServer } from 'node:net'
+import { tmpdir } from 'node:os'
+import { resolve } from 'node:path'
+
+import Database from '@rocicorp/zero-sqlite3'
+
+const VENDOR_PATH_FILE = resolve(
+  import.meta.dirname,
+  '..',
+  '..',
+  '..',
+  'vendor',
+  'pgsqlite',
+  '.resolved-path'
+)
+
+export function pgsqliteBinPath(): string | null {
+  if (!existsSync(VENDOR_PATH_FILE)) return null
+  const path = readFileSync(VENDOR_PATH_FILE, 'utf-8').trim()
+  return path && existsSync(path) ? path : null
+}
+
+export const ORACLE_AVAILABLE = pgsqliteBinPath() !== null
+
+export interface OracleServer {
+  port: number
+  dbPath: string
+  stop(): Promise<void>
+}
+
+/**
+ * Pick a free TCP port by binding ephemeral and reading what we got.
+ * Then close immediately and hand the port to pgsqlite. There's a tiny TOCTOU
+ * window where another process could grab it before pgsqlite binds, but
+ * vitest's parallel test files all go through this helper, so the only races
+ * are against unrelated processes on the host — vanishingly unlikely in CI.
+ */
+async function getFreePort(): Promise<number> {
+  return new Promise((resolveFn, reject) => {
+    const server = createServer()
+    server.unref()
+    server.on('error', reject)
+    server.listen(0, '127.0.0.1', () => {
+      const addr = server.address()
+      if (!addr || typeof addr === 'string') {
+        server.close()
+        reject(new Error('failed to get free port'))
+        return
+      }
+      const port = addr.port
+      server.close(() => resolveFn(port))
+    })
+  })
+}
+
+async function probePort(port: number, timeoutMs: number): Promise<boolean> {
+  return new Promise((resolveFn) => {
+    const sock = createConnection({ host: '127.0.0.1', port })
+    const cleanup = (val: boolean) => {
+      try {
+        sock.destroy()
+      } catch {}
+      resolveFn(val)
+    }
+    const timer = setTimeout(() => cleanup(false), timeoutMs)
+    sock.once('connect', () => {
+      clearTimeout(timer)
+      cleanup(true)
+    })
+    sock.once('error', () => {
+      clearTimeout(timer)
+      cleanup(false)
+    })
+  })
+}
+
+/**
+ * Start a pgsqlite server on an OS-assigned ephemeral port with an ephemeral
+ * database. Throws on probe timeout. Caller is responsible for calling
+ * `stop()` in a `finally` / `afterAll` hook — `stop()` awaits child exit
+ * and cleans up the tempdir.
+ */
+export async function startPgsqliteServer(): Promise<OracleServer> {
+  const bin = pgsqliteBinPath()
+  if (!bin) throw new Error('pgsqlite binary not available')
+
+  const port = await getFreePort()
+  const tmpDir = mkdtempSync(resolve(tmpdir(), 'orez-oracle-'))
+  const dbPath = resolve(tmpDir, 'pg.db')
+
+  const proc: ChildProcess = spawn(
+    bin,
+    ['--port', String(port), '--database', dbPath, '--log-level', 'error'],
+    { stdio: ['ignore', 'pipe', 'pipe'] }
+  )
+
+  const exited: Promise<void> = new Promise((res) => proc.once('exit', () => res()))
+
+  // wait for "ready" — poll until the server accepts a connection or we time out
+  let ready = false
+  const start = Date.now()
+  while (Date.now() - start < 10_000) {
+    if (await probePort(port, 250)) {
+      ready = true
+      break
+    }
+    if (proc.exitCode !== null) {
+      throw new Error(`pgsqlite exited before becoming ready (code=${proc.exitCode})`)
+    }
+    await new Promise((r) => setTimeout(r, 50))
+  }
+  if (!ready) {
+    try {
+      proc.kill('SIGKILL')
+    } catch {}
+    try {
+      rmSync(tmpDir, { recursive: true, force: true })
+    } catch {}
+    throw new Error(`pgsqlite did not become ready on port ${port} within 10s`)
+  }
+
+  return {
+    port,
+    dbPath,
+    async stop() {
+      try {
+        proc.kill('SIGTERM')
+      } catch {}
+      // wait up to 2s for graceful exit, then SIGKILL
+      const killTimer = setTimeout(() => {
+        try {
+          proc.kill('SIGKILL')
+        } catch {}
+      }, 2_000)
+      try {
+        await exited
+      } finally {
+        clearTimeout(killTimer)
+      }
+      try {
+        rmSync(tmpDir, { recursive: true, force: true })
+      } catch {}
+    },
+  }
+}
+
+/**
+ * Helper: open a postgres.js client to a running pgsqlite server.
+ * Requires `postgres` to be available (we already ship it via @rocicorp/zero).
+ */
+export async function connectOracle(server: OracleServer): Promise<{
+  exec: (sql: string, params?: any[]) => Promise<any[]>
+  end: () => Promise<void>
+}> {
+  const { default: postgres } = await import('postgres')
+  const sql = postgres({
+    host: '127.0.0.1',
+    port: server.port,
+    user: 'oracle',
+    password: '',
+    database: 'main',
+    ssl: false,
+    max: 1,
+    idle_timeout: 5,
+    connect_timeout: 5,
+    fetch_types: false,
+    prepare: false,
+  })
+  return {
+    exec: async (s: string, params: any[] = []) => {
+      const rows = await sql.unsafe(s, params as any[])
+      return rows as any[]
+    },
+    end: () => sql.end({ timeout: 2 }).then(() => undefined),
+  }
+}
+
+/**
+ * Run the same query against pgsqlite (oracle) and our compiler+SQLite (under
+ * test). Return both result sets so the test can diff them however it wants.
+ *
+ * Uses freshly-spawned servers/dbs for each call — slow but isolated. For
+ * batches use the lower-level helpers above.
+ */
+export async function runOracleAndCompiler(
+  setupSql: string[],
+  pgSql: string,
+  params: any[] = []
+): Promise<{
+  oracle: any[]
+  ours: any[]
+}> {
+  const server = await startPgsqliteServer()
+  try {
+    const conn = await connectOracle(server)
+    try {
+      for (const s of setupSql) await conn.exec(s)
+      const oracle = await conn.exec(pgSql, params)
+
+      // ours: setup + query against fresh in-memory sqlite (after compile())
+      const { compile } = await import('../index.js')
+      const db = new Database(':memory:')
+      for (const s of setupSql) {
+        const { sql: translated } = compile(s)
+        db.exec(translated)
+      }
+      const { sql: translatedQuery } = compile(pgSql)
+      // postgres.js uses $1 params; sqlite uses ?
+      const sqliteSql = translatedQuery.replace(/\$(\d+)/g, '?')
+      const stmt = db.prepare(sqliteSql)
+      const ours = (
+        params.length > 0 ? stmt.all(...(params as any[])) : stmt.all()
+      ) as any[]
+      db.close()
+
+      return { oracle, ours }
+    } finally {
+      await conn.end()
+    }
+  } finally {
+    await server.stop()
+  }
+}

package/src/pg-sqlite-compiler/test/types.test.ts

@@ -0,0 +1,109 @@
+import Database from '@rocicorp/zero-sqlite3'
+/**
+ * Snapshot tests for the types pass. Each case asserts that the emitted
+ * SQLite SQL has the expected normalized type name AND that the result
+ * actually executes against a fresh @rocicorp/zero-sqlite3 in-memory db.
+ */
+import { describe, expect, it } from 'vitest'
+
+import { compile } from '../index.js'
+
+function compilesAndRuns(pgSql: string): { sql: string } {
+  const { sql, warnings } = compile(pgSql)
+  expect(warnings).toEqual([])
+  const db = new Database(':memory:')
+  db.exec(sql)
+  db.close()
+  return { sql }
+}
+
+describe('types pass', () => {
+  it('BIGSERIAL → INTEGER (rowid alias on PRIMARY KEY)', () => {
+    const { sql } = compilesAndRuns('CREATE TABLE t (id BIGSERIAL PRIMARY KEY)')
+    expect(sql).toMatch(/INTEGER PRIMARY KEY/i)
+    expect(sql).not.toMatch(/bigserial/i)
+  })
+
+  it('jsonb → TEXT', () => {
+    const { sql } = compilesAndRuns(
+      'CREATE TABLE t (id text PRIMARY KEY, p jsonb NOT NULL)'
+    )
+    expect(sql).toMatch(/p TEXT NOT NULL/i)
+    expect(sql).not.toMatch(/jsonb/i)
+  })
+
+  it('text[] → TEXT (arrays as JSON text)', () => {
+    const { sql } = compilesAndRuns(
+      'CREATE TABLE t (id text PRIMARY KEY, tags text[] NOT NULL)'
+    )
+    expect(sql).toMatch(/tags TEXT NOT NULL/i)
+    expect(sql).not.toMatch(/\[/)
+  })
+
+  it('timestamp with time zone → TEXT', () => {
+    const { sql } = compilesAndRuns(
+      'CREATE TABLE t (id text PRIMARY KEY, ts timestamp with time zone NOT NULL)'
+    )
+    expect(sql).toMatch(/ts TEXT NOT NULL/i)
+    expect(sql).not.toMatch(/timestamp/i)
+    expect(sql).not.toMatch(/with time zone/i)
+  })
+
+  it('varchar(N) → TEXT (drop length typmod)', () => {
+    const { sql } = compilesAndRuns(
+      'CREATE TABLE t (id text PRIMARY KEY, val varchar(64))'
+    )
+    expect(sql).toMatch(/val TEXT/i)
+    expect(sql).not.toMatch(/varchar/i)
+    expect(sql).not.toMatch(/\(64\)/)
+  })
+
+  it('boolean → INTEGER', () => {
+    const { sql } = compilesAndRuns(
+      'CREATE TABLE t (id text PRIMARY KEY, enabled boolean NOT NULL DEFAULT false)'
+    )
+    expect(sql).toMatch(/enabled INTEGER NOT NULL/i)
+    expect(sql).not.toMatch(/boolean/i)
+  })
+
+  it('uuid → TEXT', () => {
+    const { sql } = compilesAndRuns('CREATE TABLE t (id uuid PRIMARY KEY)')
+    expect(sql).toMatch(/id TEXT PRIMARY KEY/i)
+    expect(sql).not.toMatch(/uuid/i)
+  })
+
+  it('bytea → BLOB', () => {
+    const { sql } = compilesAndRuns('CREATE TABLE t (id text PRIMARY KEY, body bytea)')
+    expect(sql).toMatch(/body BLOB/i)
+    expect(sql).not.toMatch(/bytea/i)
+  })
+
+  it('numeric(10,2) → NUMERIC (drops precision)', () => {
+    const { sql } = compilesAndRuns(
+      'CREATE TABLE t (id text PRIMARY KEY, amount numeric(10,2))'
+    )
+    expect(sql).toMatch(/amount NUMERIC/i)
+    expect(sql).not.toMatch(/\(10/)
+  })
+
+  it('composite: all common chat-app types in one table', () => {
+    const { sql } = compilesAndRuns(
+      'CREATE TABLE event (' +
+        'id BIGSERIAL PRIMARY KEY, ' +
+        'user_id uuid NOT NULL, ' +
+        '"createdAt" timestamp with time zone DEFAULT NOW() NOT NULL, ' +
+        "payload jsonb NOT NULL DEFAULT '{}', " +
+        "tags text[] NOT NULL DEFAULT '{}', " +
+        'amount numeric(10,2), ' +
+        'enabled boolean NOT NULL DEFAULT true' +
+        ')'
+    )
+    expect(sql).toMatch(/INTEGER PRIMARY KEY/i)
+    expect(sql).toMatch(/user_id TEXT NOT NULL/i)
+    expect(sql).toMatch(/"createdAt" TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL/i)
+    expect(sql).toMatch(/payload TEXT NOT NULL/i)
+    expect(sql).toMatch(/tags TEXT NOT NULL/i)
+    expect(sql).toMatch(/amount NUMERIC/i)
+    expect(sql).toMatch(/enabled INTEGER NOT NULL/i)
+  })
+})

package/src/pg-sqlite-compiler/types.ts

@@ -0,0 +1,63 @@
+/**
+ * Schema metadata available to passes for type-aware translation.
+ *
+ * Implementations are caller-provided. The default NOOP_SCHEMA returns nothing
+ * and passes degrade gracefully (regex/AST-shape fallbacks).
+ *
+ * For orez, the eventual implementation will read from `ctx.storage.sql`
+ * PRAGMA + the `_orez_pg_metadata` catalog table.
+ */
+export interface SchemaInfo {
+  /** PG type name (e.g. "jsonb", "text[]") for a column, or undefined. */
+  getColumnType(schema: string, table: string, column: string): string | undefined
+
+  /** ENUM definition lookup by PG type name. */
+  getEnum(typeName: string): EnumInfo | undefined
+
+  /** Validate an ENUM literal value. */
+  isEnumValue(typeOid: number, label: string): boolean
+
+  /** Column list for a table (for SELECT * expansion, RETURNING * etc.). */
+  getTableColumns(schema: string, table: string): readonly string[] | undefined
+}
+
+export interface EnumInfo {
+  typeOid: number
+  values: readonly string[]
+}
+
+export interface CompileWarning {
+  kind: string
+  message: string
+  /** PG AST node tag where the warning was raised. */
+  near?: string
+}
+
+export interface CompileResult {
+  sql: string
+  warnings: CompileWarning[]
+}
+
+export interface CompileOptions {
+  schema?: SchemaInfo
+  pgVersion?: number
+  /** Override pass list (mainly for testing individual passes). */
+  passes?: Pass[]
+}
+
+/** Context passed to every pass. */
+export interface PassContext {
+  schema: SchemaInfo
+  warnings: CompileWarning[]
+  /**
+   * Optional pass list — if set, runPasses uses these instead of the default
+   * pipeline. Otherwise the full default pipeline runs.
+   */
+  passes?: Pass[]
+}
+
+/** A pass is a function that mutates a RawStmt in place. */
+export interface Pass {
+  name: string
+  run(rawStmt: any, ctx: PassContext): void
+}

package/src/replication/change-tracker.ts

@@ -20,6 +20,16 @@ export interface ChangeTrackingDb {
   query<T>(sql: string, params?: unknown[]): Promise<{ rows: T[] }>
 }
 
+// PGlite returns JSONB columns as parsed objects; the DO backend returns them
+// as JSON strings (it stores `row_data TEXT`). normalize once at the consumer
+// boundary so callers always get an object.
+function jsonRecord(value: unknown): Record<string, unknown> | null {
+  if (value === null || value === undefined) return null
+  if (typeof value === 'object') return value as Record<string, unknown>
+  if (typeof value !== 'string' || value === '') return null
+  return JSON.parse(value) as Record<string, unknown>
+}
+
 export async function installChangeTracking(db: ChangeTrackingDb): Promise<void> {
   // use _orez schema for internal tables - survives pg_restore of public schema
   await db.exec(`CREATE SCHEMA IF NOT EXISTS _orez`)
@@ -241,7 +251,12 @@ export async function getChangesSince(
     'SELECT watermark, table_name, op, row_data, old_data FROM _orez._zero_changes WHERE watermark > $1 ORDER BY watermark LIMIT $2',
     [watermark, limit]
   )
-  return result.rows
+  return result.rows.map((row) => ({
+    ...row,
+    watermark: Number(row.watermark),
+    row_data: jsonRecord(row.row_data),
+    old_data: jsonRecord(row.old_data),
+  }))
 }
 
 export async function purgeConsumedChanges(

package/src/replication/handler.test.ts

@@ -174,6 +174,19 @@ describe('handleStartReplication', () => {
     return types
   }
 
+  function countCopyDataFrames(buf: Uint8Array): number {
+    const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength)
+    let pos = 0
+    let count = 0
+    while (pos < buf.length) {
+      if (buf[pos] !== 0x64) return count
+      const len = dv.getInt32(pos + 1)
+      pos += 1 + len
+      count++
+    }
+    return count
+  }
+
   it('sends CopyBothResponse first', async () => {
     const { written, writer } = createWriter()
 
@@ -238,6 +251,28 @@ describe('handleStartReplication', () => {
     expect(insIdx).toBeLessThan(comIdx)
   })
 
+  it('writes one CopyData frame per socket chunk', async () => {
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db,
+      testMutex
+    )
+
+    await new Promise((r) => setTimeout(r, 100))
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('chunked', 123)`)
+    signalReplicationChange()
+    await new Promise((r) => setTimeout(r, 700))
+
+    const copyDataWrites = written.filter((msg) => msg[0] === 0x64)
+    expect(copyDataWrites.length).toBeGreaterThanOrEqual(4)
+    for (const msg of copyDataWrites) {
+      expect(countCopyDataFrames(msg)).toBe(1)
+    }
+  })
+
   it('streams UPDATE and DELETE operations', async () => {
     const { written, writer } = createWriter()
 

package/src/replication/handler.ts

@@ -117,6 +117,10 @@ let _replicationWakeup: (() => void) | null = null
  *  called by the proxy after executing writes on the postgres instance. */
 export function signalReplicationChange() {
   _replicationWakeup?.()
+  const globalWakeup = (globalThis as any).__orez_signal_replication
+  if (typeof globalWakeup === 'function' && globalWakeup !== _replicationWakeup) {
+    globalWakeup()
+  }
 }
 
 // cached setup results so reconnects skip the expensive mutex-holding setup phase.
@@ -1146,8 +1150,9 @@ async function streamChanges(
   const endLsn = nextLsn()
   messages.push(encodeWrappedChange(endLsn, endLsn, ts, encodeCommit(0, lsn, endLsn, ts)))
 
-  // write messages individually — works for both TCP sockets and in-process
-  // pipes (browser pipe handler parses one message per write() call)
+  // The MessagePort-backed socket delivers each write as one readable chunk.
+  // zero-cache parses one replication payload per chunk, so each CopyData frame
+  // must be written separately.
   let totalSize = 0
   for (const msg of messages) totalSize += msg.length
   log.debug.repl(

package/src/replication/pgoutput-encoder.test.ts

@@ -40,6 +40,24 @@ function rText(buf: Uint8Array, off: number): [string, number] {
   const str = new TextDecoder().decode(buf.subarray(off + 4, off + 4 + len))
   return [str, off + 4 + len]
 }
+function rTupleTextValues(buf: Uint8Array, off: number): [Array<string | null>, number] {
+  const values: Array<string | null> = []
+  const count = r16(buf, off)
+  let pos = off + 2
+  for (let i = 0; i < count; i++) {
+    const kind = buf[pos++]
+    if (kind === 0x6e) {
+      values.push(null)
+      continue
+    }
+    expect(kind).toBe(0x74)
+    const len = r32(buf, pos)
+    pos += 4
+    values.push(new TextDecoder().decode(buf.subarray(pos, pos + len)))
+    pos += len
+  }
+  return [values, pos]
+}
 
 describe('pgoutput-encoder', () => {
   describe('encodeBegin', () => {
@@ -276,6 +294,26 @@ describe('pgoutput-encoder', () => {
       }
       expect(buf[pos]).toBe(0x4e) // 'N' new tuple marker
     })
+
+    it('encodes sqlite integer booleans as postgres bool text', () => {
+      const boolCols: ColumnInfo[] = [
+        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
+        { name: 'completed', typeOid: 16, typeMod: -1 },
+      ]
+      const buf = encodeUpdate(
+        16384,
+        { id: '1', completed: 1 },
+        { id: '1', completed: 0 },
+        boolCols
+      )
+
+      expect(buf[5]).toBe(0x4f) // 'O' old tuple
+      const [oldValues, afterOld] = rTupleTextValues(buf, 6)
+      expect(oldValues).toEqual(['1', 'f'])
+      expect(buf[afterOld]).toBe(0x4e) // 'N' new tuple
+      const [newValues] = rTupleTextValues(buf, afterOld + 1)
+      expect(newValues).toEqual(['1', 't'])
+    })
   })
 
   describe('encodeDelete', () => {
@@ -364,6 +402,12 @@ describe('pgoutput-encoder', () => {
       const b = getTableOid('oid_test_y')
       expect(a).not.toBe(b)
     })
+
+    it('matches DoBackend catalog oids for flattened schema tables', () => {
+      expect(getTableOid('public.todo')).toBe(4392680)
+      expect(getTableOid('todo_0.clients')).toBe(9663976)
+      expect(getTableOid('todo_0.mutations')).toBe(8519194)
+    })
   })
 
   // roundtrip tests: encode with orez → parse with zero-cache's parser
@@ -384,8 +428,16 @@ describe('pgoutput-encoder', () => {
       PgoutputParser = (await import(parserPath)).PgoutputParser
     })
 
-    function makeParser() {
-      return new PgoutputParser(typeParsers)
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    function makeParser(parserOverrides: any = typeParsers) {
+      return new PgoutputParser(parserOverrides)
+    }
+
+    function makeBoolAwareParser() {
+      return makeParser({
+        getTypeParser: (oid: number) =>
+          oid === 16 ? (value: string) => value === 't' : String,
+      })
     }
 
     it('BEGIN roundtrip', () => {
@@ -490,6 +542,23 @@ describe('pgoutput-encoder', () => {
       expect(parsed.key).toBeNull()
     })
 
+    it('UPDATE roundtrip decodes sqlite boolean integers through zero-cache parser', () => {
+      const oid = getTableOid('rt.bool_update')
+      const cols: ColumnInfo[] = [
+        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
+        { name: 'completed', typeOid: 16, typeMod: -1 },
+      ]
+      const parser = makeBoolAwareParser()
+      parser.parse(encodeRelation(oid, 'public', 'bool_update', 0x64, cols))
+
+      const parsed = parser.parse(
+        encodeUpdate(oid, { id: '1', completed: 1 }, { id: '1', completed: 0 }, cols)
+      )
+      expect(parsed.tag).toBe('update')
+      expect(parsed.old.completed).toBe(false)
+      expect(parsed.new.completed).toBe(true)
+    })
+
     it('DELETE roundtrip', () => {
       const oid = getTableOid('rt.del_test')
       const cols: ColumnInfo[] = [