orez 0.0.48 → 0.0.50

package/src/pglite-manager.ts

@@ -16,7 +16,7 @@ export interface PGliteInstances {
 }
 
 // create a single pglite instance with given dataDir suffix
-export async function createInstance(
+async function createInstance(
   config: ZeroLiteConfig,
   name: string,
   withExtensions: boolean
@@ -133,15 +133,8 @@ export async function runMigrations(db: PGlite, config: ZeroLiteConfig): Promise
       continue
     }
 
-    const filePath = join(migrationsDir, file)
-    if (!existsSync(filePath)) {
-      // .ts-only custom migrations are handled by the app's own migration runner
-      log.debug.orez(`skipping migration (no .sql file): ${name}`)
-      continue
-    }
-
     log.debug.orez(`applying migration: ${name}`)
-    const sql = readFileSync(filePath, 'utf-8')
+    const sql = readFileSync(join(migrationsDir, file), 'utf-8')
 
     // split by drizzle's statement-breakpoint marker
     const statements = sql

package/src/replication/change-tracker.test.ts

@@ -3,6 +3,8 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 
 import {
   installChangeTracking,
+  installTriggersOnShardTables,
+  purgeConsumedChanges,
   getChangesSince,
   getCurrentWatermark,
 } from './change-tracker'
@@ -184,3 +186,115 @@ describe('change-tracker', () => {
     expect(special[0].row_data).toMatchObject({ val: 'works' })
   })
 })
+
+describe('shard table tracking', () => {
+  let db: PGlite
+
+  beforeEach(async () => {
+    db = new PGlite()
+    await db.waitReady
+    await installChangeTracking(db)
+  })
+
+  afterEach(async () => {
+    await db.close()
+  })
+
+  it('only tracks clients table in shard schemas, not replicas/mutations', async () => {
+    // zero-cache creates shard schemas like chat_0 with clients, replicas, mutations.
+    // only clients needs tracking — replicas/mutations changes crash zero-cache
+    // with "Unknown table chat_0.replicas" because they aren't in zero's schema.
+    await db.exec(`
+      CREATE SCHEMA chat_0;
+      CREATE TABLE chat_0.clients (
+        "clientGroupID" TEXT NOT NULL,
+        "clientID" TEXT NOT NULL,
+        "lastMutationID" BIGINT,
+        "userID" TEXT,
+        PRIMARY KEY ("clientGroupID", "clientID")
+      );
+      CREATE TABLE chat_0.replicas (
+        id TEXT PRIMARY KEY,
+        version TEXT,
+        cookie TEXT
+      );
+      CREATE TABLE chat_0.mutations (
+        id TEXT PRIMARY KEY,
+        "clientID" TEXT,
+        name TEXT,
+        args JSONB
+      );
+    `)
+
+    await installTriggersOnShardTables(db)
+
+    // insert into all three tables
+    await db.exec(
+      `INSERT INTO chat_0.clients ("clientGroupID", "clientID", "lastMutationID") VALUES ('cg1', 'c1', 1)`
+    )
+    await db.exec(`INSERT INTO chat_0.replicas (id, version) VALUES ('r1', 'v1')`)
+    await db.exec(
+      `INSERT INTO chat_0.mutations (id, "clientID", name) VALUES ('m1', 'c1', 'sendMessage')`
+    )
+
+    const changes = await getChangesSince(db, 0)
+    const tables = changes.map((c) => c.table_name)
+
+    // only clients should be tracked
+    expect(tables).toContain('chat_0.clients')
+    expect(tables).not.toContain('chat_0.replicas')
+    expect(tables).not.toContain('chat_0.mutations')
+  })
+
+  it('purges consumed changes to prevent OOM', async () => {
+    // _zero_changes accumulates forever in 0.0.37. with wasm pglite,
+    // this eventually causes OOM. we need a purge mechanism.
+    await db.exec(`
+      CREATE TABLE public.items (id SERIAL PRIMARY KEY, val TEXT)
+    `)
+    await installChangeTracking(db)
+
+    // insert some data
+    for (let i = 0; i < 10; i++) {
+      await db.exec(`INSERT INTO public.items (val) VALUES ('item${i}')`)
+    }
+
+    const changes = await getChangesSince(db, 0)
+    expect(changes).toHaveLength(10)
+    const lastWatermark = changes[changes.length - 1].watermark
+
+    // purge consumed changes up to the watermark we've processed
+    await purgeConsumedChanges(db, lastWatermark)
+
+    // after purge, no changes before that watermark should remain
+    const remaining = await getChangesSince(db, 0)
+    expect(remaining).toHaveLength(0)
+  })
+
+  it('tracks tables created after initial installChangeTracking', async () => {
+    // simulate zero-cache creating shard schema AFTER replication starts.
+    // in production, zero-cache creates chat_0 schema + clients table
+    // after the replication connection is already established.
+    // the change tracker must pick up these new tables.
+    await db.exec(`
+      CREATE SCHEMA chat_0;
+      CREATE TABLE chat_0.clients (
+        "clientGroupID" TEXT NOT NULL,
+        "clientID" TEXT NOT NULL,
+        "lastMutationID" BIGINT,
+        PRIMARY KEY ("clientGroupID", "clientID")
+      );
+    `)
+
+    // re-running installTriggersOnShardTables should pick up new tables
+    await installTriggersOnShardTables(db)
+
+    await db.exec(
+      `INSERT INTO chat_0.clients ("clientGroupID", "clientID", "lastMutationID") VALUES ('cg1', 'c1', 1)`
+    )
+
+    const changes = await getChangesSince(db, 0)
+    expect(changes).toHaveLength(1)
+    expect(changes[0].table_name).toBe('chat_0.clients')
+  })
+})

package/src/replication/change-tracker.ts

@@ -140,58 +140,6 @@ async function installTriggersOnAllTables(db: PGlite): Promise<void> {
   log.debug.pglite(`installed change tracking triggers on ${count} tables`)
 }
 
-/**
- * re-install change tracking triggers on any public tables that don't have them.
- * catches tables created between startup and replication start.
- */
-export async function ensureChangeTrackingOnAllTables(db: PGlite): Promise<void> {
-  const pubName = process.env.ZERO_APP_PUBLICATIONS
-  let tables: { tablename: string }[]
-
-  if (pubName) {
-    const result = await db.query<{ tablename: string }>(
-      `SELECT tablename FROM pg_publication_tables
-       WHERE pubname = $1
-         AND schemaname = 'public'
-         AND tablename NOT LIKE '_zero_%'`,
-      [pubName]
-    )
-    tables = result.rows
-  } else {
-    const result = await db.query<{ tablename: string }>(
-      `SELECT tablename FROM pg_tables
-       WHERE schemaname = 'public'
-         AND tablename NOT IN ('migrations', '_zero_changes')
-         AND tablename NOT LIKE '_zero_%'`
-    )
-    tables = result.rows
-  }
-
-  // find tables missing the change trigger
-  const triggered = await db.query<{ event_object_table: string }>(
-    `SELECT DISTINCT event_object_table FROM information_schema.triggers
-     WHERE trigger_name = '_zero_change_trigger'
-       AND event_object_schema = 'public'`
-  )
-  const hasTracker = new Set(triggered.rows.map((r) => r.event_object_table))
-
-  let count = 0
-  for (const { tablename } of tables) {
-    if (hasTracker.has(tablename)) continue
-    const quoted = quoteIdent(tablename)
-    await db.exec(`
-      CREATE TRIGGER _zero_change_trigger
-        AFTER INSERT OR UPDATE OR DELETE ON public.${quoted}
-        FOR EACH ROW EXECUTE FUNCTION public._zero_track_change();
-    `)
-    count++
-  }
-
-  if (count > 0) {
-    log.debug.pglite(`installed change tracking on ${count} new tables`)
-  }
-}
-
 /**
  * install change tracking triggers on tables in shard schemas.
  * zero-cache creates shard schemas (e.g. chat_0) with clients/mutations
@@ -211,7 +159,7 @@ export async function installTriggersOnShardTables(db: PGlite): Promise<void> {
   if (result.rows.length === 0) return
 
   // only track `clients` — that's the table zero-cache expects in the
-  // replication stream (needed for .server promise resolution).  other shard
+  // replication stream (needed for .server promise resolution). other shard
   // tables like `replicas` are zero-cache internal state and streaming them
   // back causes "Unknown table" crashes in zero-cache's change-processor.
   let count = 0
@@ -228,7 +176,9 @@ export async function installTriggersOnShardTables(db: PGlite): Promise<void> {
       const qs = quoteIdent(nspname)
       const qt = quoteIdent(event_object_table)
       await db.exec(`DROP TRIGGER IF EXISTS _zero_change_trigger ON ${qs}.${qt}`)
-      log.debug.pglite(`removed stale shard trigger from ${nspname}.${event_object_table}`)
+      log.debug.pglite(
+        `removed stale shard trigger from ${nspname}.${event_object_table}`
+      )
     }
 
     const tables = await db.query<{ tablename: string }>(

package/src/replication/handler.ts

@@ -32,9 +32,6 @@ import {
 import type { Mutex } from '../mutex.js'
 import type { PGlite } from '@electric-sql/pglite'
 
-// track concurrent replication handlers to detect reconnect-purge race
-let activeHandlerCount = 0
-
 export interface ReplicationWriter {
   write(data: Uint8Array): void
 }
@@ -265,11 +262,6 @@ export async function handleStartReplication(
   db: PGlite,
   mutex: Mutex
 ): Promise<void> {
-  activeHandlerCount++
-  const handlerId = activeHandlerCount
-  console.info(
-    `[orez-repl#${handlerId}] START_REPLICATION (active handlers: ${activeHandlerCount})`
-  )
   log.debug.proxy('replication: entering streaming mode')
 
   // send CopyBothResponse to enter streaming mode
@@ -461,10 +453,6 @@ export async function handleStartReplication(
     mutex.release()
   }
 
-  console.info(
-    `[orez-repl#${handlerId}] setup complete, starting poll (lastWatermark=${lastWatermark})`
-  )
-
   // track which tables we've sent RELATION messages for
   const sentRelations = new Set<string>()
   let txCounter = 1
@@ -475,13 +463,26 @@ export async function handleStartReplication(
   const pollIntervalCatchUp = 20
   const batchSize = 2000
   const purgeEveryN = 10
+  const shardRescanEveryN = 20
   let running = true
   let pollsSincePurge = 0
-  let lastIdleLog = 0
+  let pollsSinceShardRescan = 0
 
   const poll = async () => {
     while (running) {
       try {
+        // periodically re-scan for new shard schemas (e.g. chat_0 created by zero-cache)
+        pollsSinceShardRescan++
+        if (pollsSinceShardRescan >= shardRescanEveryN) {
+          pollsSinceShardRescan = 0
+          await mutex.acquire()
+          try {
+            await installTriggersOnShardTables(db)
+          } finally {
+            mutex.release()
+          }
+        }
+
         // acquire mutex to avoid conflicting with proxy connections
         await mutex.acquire()
         let changes: Awaited<ReturnType<typeof getChangesSince>>
@@ -511,10 +512,6 @@ export async function handleStartReplication(
             continue
           }
 
-          const tables = [...new Set(changes.map((c) => c.table_name))].join(',')
-          console.info(
-            `[orez-repl#${handlerId}] found ${changes.length} changes [${tables}] (wm ${lastWatermark}→${changes[changes.length - 1].watermark}, type=${typeof changes[0].watermark})`
-          )
           await streamChanges(
             changes,
             writer,
@@ -534,23 +531,12 @@ export async function handleStartReplication(
             try {
               const purged = await purgeConsumedChanges(db, lastWatermark)
               if (purged > 0) {
-                console.info(
-                  `[orez-repl#${handlerId}] purged ${purged} changes (wm<=${lastWatermark})`
-                )
+                log.debug.proxy(`purged ${purged} consumed changes`)
               }
             } finally {
               mutex.release()
             }
           }
-        } else {
-          // throttled idle logging (every 10s)
-          const now = Date.now()
-          if (now - lastIdleLog > 10000) {
-            lastIdleLog = now
-            console.info(
-              `[orez-repl#${handlerId}] idle (lastWatermark=${lastWatermark}, type=${typeof lastWatermark})`
-            )
-          }
         }
 
         // send keepalive
@@ -574,10 +560,7 @@ export async function handleStartReplication(
 
   log.debug.proxy('replication: starting poll loop')
   await poll()
-  activeHandlerCount--
-  console.info(
-    `[orez-repl#${handlerId}] poll loop exited (remaining handlers: ${activeHandlerCount})`
-  )
+  log.debug.proxy('replication: poll loop exited')
 }
 
 async function streamChanges(

package/src/replication/pgoutput-encoder.test.ts

@@ -369,7 +369,9 @@ describe('pgoutput-encoder', () => {
   describe('roundtrip: orez encoder → zero-cache parser', () => {
     // absolute path bypasses package.json exports restriction
     // eslint-disable-next-line @typescript-eslint/no-require-imports
-    const { PgoutputParser } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.js')
+    const {
+      PgoutputParser,
+    } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.js')
 
     // mock type parsers: unknown OIDs default to String (identity for text)
     const typeParsers = { getTypeParser: () => String }
@@ -555,20 +557,27 @@ describe('pgoutput-encoder', () => {
     it('LSN ordering: slot < streaming changes', () => {
       // validates that streaming changes will be seen as "new" by zero-cache
       let testLsn = 0x1000000n
-      const next = () => { testLsn += 0x100n; return testLsn }
+      const next = () => {
+        testLsn += 0x100n
+        return testLsn
+      }
 
-      const slotLsn = next()    // CREATE_REPLICATION_SLOT
-      const beginLsn = next()   // first streaming BEGIN
-      const commitLsn = next()  // first streaming COMMIT
+      const slotLsn = next() // CREATE_REPLICATION_SLOT
+      const beginLsn = next() // first streaming BEGIN
+      const commitLsn = next() // first streaming COMMIT
 
       expect(beginLsn).toBeGreaterThan(slotLsn)
       expect(commitLsn).toBeGreaterThan(beginLsn)
 
       // verify lexi version ordering is preserved
       // eslint-disable-next-line @typescript-eslint/no-require-imports
-      const { versionToLexi } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/types/lexi-version.js')
+      const {
+        versionToLexi,
+      } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/types/lexi-version.js')
       // eslint-disable-next-line @typescript-eslint/no-require-imports
-      const { toBigInt: lsnToBigInt } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/services/change-source/pg/lsn.js')
+      const {
+        toBigInt: lsnToBigInt,
+      } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/services/change-source/pg/lsn.js')
 
       const slotHex = `00000000/${slotLsn.toString(16).padStart(8, '0')}`.toUpperCase()
       const beginHex = `00000000/${beginLsn.toString(16).padStart(8, '0')}`.toUpperCase()