orez 0.0.48 → 0.0.49

package/src/pglite-manager.ts

@@ -16,7 +16,7 @@ export interface PGliteInstances {
 }
 
 // create a single pglite instance with given dataDir suffix
-export async function createInstance(
+async function createInstance(
   config: ZeroLiteConfig,
   name: string,
   withExtensions: boolean
@@ -133,15 +133,8 @@ export async function runMigrations(db: PGlite, config: ZeroLiteConfig): Promise
       continue
     }
 
-    const filePath = join(migrationsDir, file)
-    if (!existsSync(filePath)) {
-      // .ts-only custom migrations are handled by the app's own migration runner
-      log.debug.orez(`skipping migration (no .sql file): ${name}`)
-      continue
-    }
-
     log.debug.orez(`applying migration: ${name}`)
-    const sql = readFileSync(filePath, 'utf-8')
+    const sql = readFileSync(join(migrationsDir, file), 'utf-8')
 
     // split by drizzle's statement-breakpoint marker
     const statements = sql

package/src/replication/change-tracker.ts

@@ -140,58 +140,6 @@ async function installTriggersOnAllTables(db: PGlite): Promise<void> {
   log.debug.pglite(`installed change tracking triggers on ${count} tables`)
 }
 
-/**
- * re-install change tracking triggers on any public tables that don't have them.
- * catches tables created between startup and replication start.
- */
-export async function ensureChangeTrackingOnAllTables(db: PGlite): Promise<void> {
-  const pubName = process.env.ZERO_APP_PUBLICATIONS
-  let tables: { tablename: string }[]
-
-  if (pubName) {
-    const result = await db.query<{ tablename: string }>(
-      `SELECT tablename FROM pg_publication_tables
-       WHERE pubname = $1
-         AND schemaname = 'public'
-         AND tablename NOT LIKE '_zero_%'`,
-      [pubName]
-    )
-    tables = result.rows
-  } else {
-    const result = await db.query<{ tablename: string }>(
-      `SELECT tablename FROM pg_tables
-       WHERE schemaname = 'public'
-         AND tablename NOT IN ('migrations', '_zero_changes')
-         AND tablename NOT LIKE '_zero_%'`
-    )
-    tables = result.rows
-  }
-
-  // find tables missing the change trigger
-  const triggered = await db.query<{ event_object_table: string }>(
-    `SELECT DISTINCT event_object_table FROM information_schema.triggers
-     WHERE trigger_name = '_zero_change_trigger'
-       AND event_object_schema = 'public'`
-  )
-  const hasTracker = new Set(triggered.rows.map((r) => r.event_object_table))
-
-  let count = 0
-  for (const { tablename } of tables) {
-    if (hasTracker.has(tablename)) continue
-    const quoted = quoteIdent(tablename)
-    await db.exec(`
-      CREATE TRIGGER _zero_change_trigger
-        AFTER INSERT OR UPDATE OR DELETE ON public.${quoted}
-        FOR EACH ROW EXECUTE FUNCTION public._zero_track_change();
-    `)
-    count++
-  }
-
-  if (count > 0) {
-    log.debug.pglite(`installed change tracking on ${count} new tables`)
-  }
-}
-
 /**
  * install change tracking triggers on tables in shard schemas.
  * zero-cache creates shard schemas (e.g. chat_0) with clients/mutations
@@ -210,29 +158,10 @@ export async function installTriggersOnShardTables(db: PGlite): Promise<void> {
 
   if (result.rows.length === 0) return
 
-  // only track `clients` — that's the table zero-cache expects in the
-  // replication stream (needed for .server promise resolution).  other shard
-  // tables like `replicas` are zero-cache internal state and streaming them
-  // back causes "Unknown table" crashes in zero-cache's change-processor.
   let count = 0
   for (const { nspname } of result.rows) {
-    // remove stale triggers from non-clients tables (from previous versions)
-    const stale = await db.query<{ event_object_table: string }>(
-      `SELECT DISTINCT event_object_table FROM information_schema.triggers
-       WHERE trigger_name = '_zero_change_trigger'
-         AND event_object_schema = $1
-         AND event_object_table != 'clients'`,
-      [nspname]
-    )
-    for (const { event_object_table } of stale.rows) {
-      const qs = quoteIdent(nspname)
-      const qt = quoteIdent(event_object_table)
-      await db.exec(`DROP TRIGGER IF EXISTS _zero_change_trigger ON ${qs}.${qt}`)
-      log.debug.pglite(`removed stale shard trigger from ${nspname}.${event_object_table}`)
-    }
-
     const tables = await db.query<{ tablename: string }>(
-      `SELECT tablename FROM pg_tables WHERE schemaname = $1 AND tablename = 'clients'`,
+      `SELECT tablename FROM pg_tables WHERE schemaname = $1`,
       [nspname]
     )
 
@@ -266,17 +195,6 @@ export async function getChangesSince(
   return result.rows
 }
 
-export async function purgeConsumedChanges(
-  db: PGlite,
-  watermark: number
-): Promise<number> {
-  const result = await db.query<{ count: string }>(
-    'WITH deleted AS (DELETE FROM public._zero_changes WHERE watermark <= $1 RETURNING 1) SELECT count(*)::text AS count FROM deleted',
-    [watermark]
-  )
-  return Number(result.rows[0]?.count || 0)
-}
-
 export async function getCurrentWatermark(db: PGlite): Promise<number> {
   const result = await db.query<{ last_value: string; is_called: boolean }>(
     'SELECT last_value, is_called FROM public._zero_watermark'

package/src/replication/handler.ts

@@ -10,7 +10,6 @@ import { log } from '../log.js'
 import {
   getChangesSince,
   getCurrentWatermark,
-  purgeConsumedChanges,
   installTriggersOnShardTables,
   type ChangeRecord,
 } from './change-tracker.js'
@@ -32,9 +31,6 @@ import {
 import type { Mutex } from '../mutex.js'
 import type { PGlite } from '@electric-sql/pglite'
 
-// track concurrent replication handlers to detect reconnect-purge race
-let activeHandlerCount = 0
-
 export interface ReplicationWriter {
   write(data: Uint8Array): void
 }
@@ -265,11 +261,6 @@ export async function handleStartReplication(
   db: PGlite,
   mutex: Mutex
 ): Promise<void> {
-  activeHandlerCount++
-  const handlerId = activeHandlerCount
-  console.info(
-    `[orez-repl#${handlerId}] START_REPLICATION (active handlers: ${activeHandlerCount})`
-  )
   log.debug.proxy('replication: entering streaming mode')
 
   // send CopyBothResponse to enter streaming mode
@@ -351,7 +342,7 @@ export async function handleStartReplication(
     for (const schema of relevantSchemas) {
       if (schema === 'public') continue
       const shardTables = await db.query<{ tablename: string }>(
-        `SELECT tablename FROM pg_tables WHERE schemaname = $1 AND tablename = 'clients'`,
+        `SELECT tablename FROM pg_tables WHERE schemaname = $1`,
         [schema]
       )
       for (const { tablename } of shardTables.rows) {
@@ -461,23 +452,13 @@ export async function handleStartReplication(
     mutex.release()
   }
 
-  console.info(
-    `[orez-repl#${handlerId}] setup complete, starting poll (lastWatermark=${lastWatermark})`
-  )
-
   // track which tables we've sent RELATION messages for
   const sentRelations = new Set<string>()
   let txCounter = 1
 
   // polling + notification loop
-  // adaptive: poll fast when catching up, slow when idle
-  const pollIntervalIdle = 500
-  const pollIntervalCatchUp = 20
-  const batchSize = 2000
-  const purgeEveryN = 10
+  const pollInterval = 500
   let running = true
-  let pollsSincePurge = 0
-  let lastIdleLog = 0
 
   const poll = async () => {
     while (running) {
@@ -486,35 +467,12 @@ export async function handleStartReplication(
         await mutex.acquire()
         let changes: Awaited<ReturnType<typeof getChangesSince>>
         try {
-          changes = await getChangesSince(db, lastWatermark, batchSize)
+          changes = await getChangesSince(db, lastWatermark, 100)
         } finally {
           mutex.release()
         }
 
         if (changes.length > 0) {
-          // filter out shard tables that zero-cache doesn't expect.
-          // only `clients` is needed (for .server promise resolution).
-          // other shard tables (replicas, mutations) crash zero-cache
-          // with "Unknown table" in change-processor.
-          const batchEnd = changes[changes.length - 1].watermark
-          changes = changes.filter((c) => {
-            const dot = c.table_name.indexOf('.')
-            if (dot === -1) return true
-            const schema = c.table_name.substring(0, dot)
-            if (schema === 'public') return true
-            const table = c.table_name.substring(dot + 1)
-            return table === 'clients'
-          })
-
-          if (changes.length === 0) {
-            lastWatermark = batchEnd
-            continue
-          }
-
-          const tables = [...new Set(changes.map((c) => c.table_name))].join(',')
-          console.info(
-            `[orez-repl#${handlerId}] found ${changes.length} changes [${tables}] (wm ${lastWatermark}→${changes[changes.length - 1].watermark}, type=${typeof changes[0].watermark})`
-          )
           await streamChanges(
             changes,
             writer,
@@ -524,42 +482,14 @@ export async function handleStartReplication(
             excludedColumns,
             columnTypeOids
           )
-          lastWatermark = batchEnd
-
-          // purge consumed changes periodically to free wasm memory
-          pollsSincePurge++
-          if (pollsSincePurge >= purgeEveryN) {
-            pollsSincePurge = 0
-            await mutex.acquire()
-            try {
-              const purged = await purgeConsumedChanges(db, lastWatermark)
-              if (purged > 0) {
-                console.info(
-                  `[orez-repl#${handlerId}] purged ${purged} changes (wm<=${lastWatermark})`
-                )
-              }
-            } finally {
-              mutex.release()
-            }
-          }
-        } else {
-          // throttled idle logging (every 10s)
-          const now = Date.now()
-          if (now - lastIdleLog > 10000) {
-            lastIdleLog = now
-            console.info(
-              `[orez-repl#${handlerId}] idle (lastWatermark=${lastWatermark}, type=${typeof lastWatermark})`
-            )
-          }
+          lastWatermark = changes[changes.length - 1].watermark
         }
 
         // send keepalive
         const ts = nowMicros()
         writer.write(encodeKeepalive(currentLsn, ts, false))
 
-        // if we got a full batch, there's likely more - poll fast
-        const delay = changes.length >= batchSize ? pollIntervalCatchUp : pollIntervalIdle
-        await new Promise((resolve) => setTimeout(resolve, delay))
+        await new Promise((resolve) => setTimeout(resolve, pollInterval))
       } catch (err: unknown) {
         const msg = err instanceof Error ? err.message : String(err)
         log.debug.proxy(`replication poll error: ${msg}`)
@@ -574,10 +504,7 @@ export async function handleStartReplication(
 
   log.debug.proxy('replication: starting poll loop')
   await poll()
-  activeHandlerCount--
-  console.info(
-    `[orez-repl#${handlerId}] poll loop exited (remaining handlers: ${activeHandlerCount})`
-  )
+  log.debug.proxy('replication: poll loop exited')
 }
 
 async function streamChanges(

package/src/replication/pgoutput-encoder.test.ts

@@ -364,223 +364,6 @@ describe('pgoutput-encoder', () => {
     })
   })
 
-  // roundtrip tests: encode with orez → parse with zero-cache's parser
-  // this validates the fundamental contract between orez and zero-cache
-  describe('roundtrip: orez encoder → zero-cache parser', () => {
-    // absolute path bypasses package.json exports restriction
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
-    const { PgoutputParser } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.js')
-
-    // mock type parsers: unknown OIDs default to String (identity for text)
-    const typeParsers = { getTypeParser: () => String }
-
-    function makeParser() {
-      return new PgoutputParser(typeParsers)
-    }
-
-    it('BEGIN roundtrip', () => {
-      const lsn = 0x1000200n
-      const ts = BigInt(Date.now()) * 1000n
-      const parser = makeParser()
-      const parsed = parser.parse(encodeBegin(lsn, ts, 42))
-
-      expect(parsed.tag).toBe('begin')
-      expect(parsed.commitLsn).toBe('00000000/01000200')
-      expect(parsed.xid).toBe(42)
-      expect(parsed.commitTime).toBe(ts)
-    })
-
-    it('COMMIT roundtrip', () => {
-      const lsn = 0x1000200n
-      const endLsn = 0x1000300n
-      const ts = BigInt(Date.now()) * 1000n
-      const parser = makeParser()
-      const parsed = parser.parse(encodeCommit(0, lsn, endLsn, ts))
-
-      expect(parsed.tag).toBe('commit')
-      expect(parsed.commitLsn).toBe('00000000/01000200')
-      expect(parsed.commitEndLsn).toBe('00000000/01000300')
-      expect(parsed.commitTime).toBe(ts)
-    })
-
-    it('RELATION roundtrip', () => {
-      const oid = getTableOid('rt.rel_test')
-      const cols: ColumnInfo[] = [
-        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
-        { name: 'name', typeOid: 25, typeMod: -1 },
-      ]
-      const parser = makeParser()
-      const parsed = parser.parse(encodeRelation(oid, 'public', 'rel_test', 0x64, cols))
-
-      expect(parsed.tag).toBe('relation')
-      expect(parsed.schema).toBe('public')
-      expect(parsed.name).toBe('rel_test')
-      expect(parsed.columns).toHaveLength(2)
-      expect(parsed.keyColumns).toEqual(['id'])
-    })
-
-    it('INSERT roundtrip', () => {
-      const oid = getTableOid('rt.ins_test')
-      const cols: ColumnInfo[] = [
-        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
-        { name: 'val', typeOid: 25, typeMod: -1 },
-      ]
-      const parser = makeParser()
-      parser.parse(encodeRelation(oid, 'public', 'ins_test', 0x64, cols))
-
-      const parsed = parser.parse(encodeInsert(oid, { id: 'abc', val: 'hello' }, cols))
-      expect(parsed.tag).toBe('insert')
-      expect(parsed.new.id).toBe('abc')
-      expect(parsed.new.val).toBe('hello')
-    })
-
-    it('INSERT with null', () => {
-      const oid = getTableOid('rt.null_test')
-      const cols: ColumnInfo[] = [
-        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
-        { name: 'opt', typeOid: 25, typeMod: -1 },
-      ]
-      const parser = makeParser()
-      parser.parse(encodeRelation(oid, 'public', 'null_test', 0x64, cols))
-
-      const parsed = parser.parse(encodeInsert(oid, { id: 'x', opt: null }, cols))
-      expect(parsed.new.opt).toBeNull()
-    })
-
-    it('UPDATE with old row roundtrip', () => {
-      const oid = getTableOid('rt.upd_test')
-      const cols: ColumnInfo[] = [
-        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
-        { name: 'val', typeOid: 25, typeMod: -1 },
-      ]
-      const parser = makeParser()
-      parser.parse(encodeRelation(oid, 'public', 'upd_test', 0x64, cols))
-
-      const parsed = parser.parse(
-        encodeUpdate(oid, { id: '1', val: 'new' }, { id: '1', val: 'old' }, cols)
-      )
-      expect(parsed.tag).toBe('update')
-      expect(parsed.new.val).toBe('new')
-      expect(parsed.old.val).toBe('old')
-    })
-
-    it('UPDATE without old row', () => {
-      const oid = getTableOid('rt.upd_no_old')
-      const cols: ColumnInfo[] = [
-        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
-        { name: 'val', typeOid: 25, typeMod: -1 },
-      ]
-      const parser = makeParser()
-      parser.parse(encodeRelation(oid, 'public', 'upd_no_old', 0x64, cols))
-
-      const parsed = parser.parse(encodeUpdate(oid, { id: '1', val: 'v' }, null, cols))
-      expect(parsed.tag).toBe('update')
-      expect(parsed.new.val).toBe('v')
-      expect(parsed.old).toBeNull()
-      expect(parsed.key).toBeNull()
-    })
-
-    it('DELETE roundtrip', () => {
-      const oid = getTableOid('rt.del_test')
-      const cols: ColumnInfo[] = [
-        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
-        { name: 'val', typeOid: 25, typeMod: -1 },
-      ]
-      const parser = makeParser()
-      parser.parse(encodeRelation(oid, 'public', 'del_test', 0x64, cols))
-
-      const parsed = parser.parse(encodeDelete(oid, { id: 'gone', val: 'x' }, cols))
-      expect(parsed.tag).toBe('delete')
-      expect(parsed.key.id).toBe('gone')
-    })
-
-    it('full transaction: BEGIN → RELATION → INSERT → COMMIT', () => {
-      const parser = makeParser()
-      const lsn = 0x2000000n
-      const endLsn = 0x2000100n
-      const ts = BigInt(Date.now()) * 1000n
-
-      const begin = parser.parse(encodeBegin(lsn, ts, 1))
-      expect(begin.commitLsn).toBe('00000000/02000000')
-
-      const oid = getTableOid('rt.full_tx')
-      const cols: ColumnInfo[] = [
-        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
-        { name: 'data', typeOid: 25, typeMod: -1 },
-      ]
-      parser.parse(encodeRelation(oid, 'public', 'full_tx', 0x64, cols))
-
-      const ins = parser.parse(encodeInsert(oid, { id: '1', data: 'test' }, cols))
-      expect(ins.new.id).toBe('1')
-
-      const commit = parser.parse(encodeCommit(0, lsn, endLsn, ts))
-      expect(commit.commitLsn).toBe(begin.commitLsn)
-    })
-
-    it('XLogData + CopyData wrapper roundtrip with parser', () => {
-      const lsn = 0x3000000n
-      const ts = BigInt(Date.now()) * 1000n
-      const pgoutput = encodeBegin(lsn, ts, 1)
-      const xlog = wrapXLogData(lsn, lsn, ts, pgoutput)
-      const frame = wrapCopyData(xlog)
-
-      // unwrap CopyData
-      const copyLen = r32(frame, 1)
-      const inner = frame.subarray(5, 1 + copyLen)
-
-      // parse like stream.js
-      expect(inner[0]).toBe(0x77)
-      const streamLsn = new DataView(inner.buffer, inner.byteOffset).getBigUint64(1)
-      expect(streamLsn).toBe(lsn)
-
-      // parse pgoutput
-      const parser = makeParser()
-      const parsed = parser.parse(inner.subarray(25))
-      expect(parsed.tag).toBe('begin')
-      expect(parsed.commitLsn).toBe('00000000/03000000')
-    })
-
-    it('shard schema encoding', () => {
-      const oid = getTableOid('rt.chat_0.clients')
-      const cols: ColumnInfo[] = [
-        { name: 'id', typeOid: 25, typeMod: -1, isKey: true },
-        { name: 'lastMutationID', typeOid: 20, typeMod: -1 },
-      ]
-      const parser = makeParser()
-      const rel = parser.parse(encodeRelation(oid, 'chat_0', 'clients', 0x64, cols))
-      expect(rel.schema).toBe('chat_0')
-      expect(rel.name).toBe('clients')
-    })
-
-    it('LSN ordering: slot < streaming changes', () => {
-      // validates that streaming changes will be seen as "new" by zero-cache
-      let testLsn = 0x1000000n
-      const next = () => { testLsn += 0x100n; return testLsn }
-
-      const slotLsn = next()    // CREATE_REPLICATION_SLOT
-      const beginLsn = next()   // first streaming BEGIN
-      const commitLsn = next()  // first streaming COMMIT
-
-      expect(beginLsn).toBeGreaterThan(slotLsn)
-      expect(commitLsn).toBeGreaterThan(beginLsn)
-
-      // verify lexi version ordering is preserved
-      // eslint-disable-next-line @typescript-eslint/no-require-imports
-      const { versionToLexi } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/types/lexi-version.js')
-      // eslint-disable-next-line @typescript-eslint/no-require-imports
-      const { toBigInt: lsnToBigInt } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/services/change-source/pg/lsn.js')
-
-      const slotHex = `00000000/${slotLsn.toString(16).padStart(8, '0')}`.toUpperCase()
-      const beginHex = `00000000/${beginLsn.toString(16).padStart(8, '0')}`.toUpperCase()
-
-      const slotVersion = versionToLexi(lsnToBigInt(slotHex))
-      const beginVersion = versionToLexi(lsnToBigInt(beginHex))
-
-      // lexi versions must maintain ordering
-      expect(beginVersion > slotVersion).toBe(true)
-    })
-  })
-
   describe('double-wrap: CopyData(XLogData(message))', () => {
     // this is the exact framing zero-cache expects for every replication message
     it('produces parseable nested structure', () => {