orez 0.0.48 → 0.0.50

package/src/replication/zero-compat.test.ts

@@ -701,22 +701,17 @@ describe('zero-cache pgoutput compatibility', { timeout: 30000 }, () => {
     const s = await stream()
     const q = s.messages
 
-    // insert sequentially with waits to avoid batching
     await db.exec(`INSERT INTO public.foo (id) VALUES ('t1')`)
     await db.exec(`INSERT INTO public.foo (id) VALUES ('t2')`)
     await db.exec(`INSERT INTO public.foo (id) VALUES ('t3')`)
 
-    // orez may batch changes into fewer transactions, so just verify
-    // all inserts arrive and every begin has a matching commit
     const all: ZcMessage[] = []
     const deadline = Date.now() + 8000
     while (Date.now() < deadline) {
       const m = await q.dequeue(2000).catch(() => null)
       if (!m) break
       if (m.tag !== 'keepalive') all.push(m)
-      const inserts = all.filter((x) => x.tag === 'insert')
-      const commits = all.filter((x) => x.tag === 'commit')
-      if (inserts.length >= 3 && commits.length >= 1) break
+      if (all.filter((x) => x.tag === 'commit').length >= 3) break
     }
 
     const begins = all.filter((m) => m.tag === 'begin')
@@ -945,230 +940,108 @@ describe('zero-cache pgoutput compatibility', { timeout: 30000 }, () => {
 
     s.close()
   })
-})
-
-/**
- * postgres.js replication stream test.
- *
- * uses the same postgres library and code path that zero-cache uses
- * in its stream.js subscribe function. validates that orez's CopyData
- * frames are correctly parsed by postgres.js's wire protocol handler
- * and that zero-cache's PgoutputParser can consume the payloads.
- */
-describe('postgres.js replication stream (zero-cache code path)', { timeout: 30000 }, () => {
-  let db: PGlite
-  let server: Server
-  let port: number
 
-  beforeEach(async () => {
-    db = new PGlite()
-    await db.waitReady
+  it('shard replicas/mutations changes NOT streamed (only clients)', async () => {
+    // zero-cache creates shard schemas (chat_0) with clients, replicas, mutations.
+    // if we stream replicas/mutations changes, zero-cache crashes with
+    // "Unknown table chat_0.replicas". only clients changes should be streamed.
     await db.exec(`
-      CREATE TABLE public.items (
+      CREATE SCHEMA chat_0;
+      CREATE TABLE chat_0.clients (
+        "clientGroupID" TEXT NOT NULL,
+        "clientID" TEXT NOT NULL,
+        "lastMutationID" BIGINT,
+        PRIMARY KEY ("clientGroupID", "clientID")
+      );
+      CREATE TABLE chat_0.replicas (
         id TEXT PRIMARY KEY,
-        val INTEGER,
-        note TEXT
-      )
+        version TEXT
+      );
+      CREATE TABLE chat_0.mutations (
+        id TEXT PRIMARY KEY,
+        "clientID" TEXT,
+        name TEXT
+      );
     `)
-    await db.exec(`CREATE PUBLICATION zero_data FOR ALL TABLES`)
     await installChangeTracking(db)
 
-    const config = { ...getConfig(), pgPort: 0 }
-    server = await startPgProxy(db, config)
-    port = (server.address() as AddressInfo).port
-  })
-
-  afterEach(async () => {
-    server?.close()
-    await db?.close()
-  })
-
-  it('postgres.js receives CopyData and parseStreamMessage decodes it', { timeout: 30000 }, async () => {
-    // import postgres (same lib zero-cache uses)
-    const pg = (await import('postgres')).default
-
-    // create a regular connection for queries
-    const regular = pg({
-      host: '127.0.0.1',
-      port,
-      user: 'user',
-      password: 'password',
-      database: 'postgres',
-      max: 1,
-    })
-
-    // create replication connection (same as zero-cache's subscribe)
-    const session = pg({
-      host: '127.0.0.1',
-      port,
-      user: 'user',
-      password: 'password',
-      database: 'postgres',
-      max: 1,
-      fetch_types: false,
-      idle_timeout: null,
-      max_lifetime: null,
-      connection: { replication: 'database' },
-    })
-
-    try {
-      // create slot (same as zero-cache)
-      await session.unsafe(
-        `CREATE_REPLICATION_SLOT "pgjs_test" TEMPORARY LOGICAL pgoutput NOEXPORT_SNAPSHOT`
-      ).simple()
-
-      // start replication stream (same pattern as stream.js)
-      const stream = session.unsafe(
-        `START_REPLICATION SLOT "pgjs_test" LOGICAL 0/0 (proto_version '1', publication_names 'zero_data', messages 'true')`
-      ).execute()
-
-      const [readable, _writable] = await Promise.all([
-        stream.readable(),
-        stream.writable(),
-      ])
-
-      // import zero-cache's actual parser
-      // eslint-disable-next-line @typescript-eslint/no-require-imports
-      const { PgoutputParser } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.js')
-      const typeParsers = { getTypeParser: () => String }
-      const parser = new PgoutputParser(typeParsers)
-
-      // parseStreamMessage from zero-cache's stream.js
-      function parseStreamMessage(buffer: Buffer): [bigint, any] | null {
-        if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) return null
-        const lsn = buffer.readBigUInt64BE(1)
-        if (buffer[0] === 0x77) {
-          return [lsn, parser.parse(buffer.subarray(25))]
-        }
-        if (buffer.readInt8(17)) {
-          return [lsn, { tag: 'keepalive' }]
-        }
-        return null // keepalive with shouldRespond=false
-      }
+    const s = await stream()
+    const q = s.messages
 
-      // collect parsed messages
-      const messages: any[] = []
-      const collectDone = new Promise<void>((resolve) => {
-        readable.on('data', (chunk: Buffer) => {
-          const result = parseStreamMessage(chunk)
-          if (result) {
-            const [_lsn, msg] = result
-            messages.push(msg)
-          }
-        })
-        setTimeout(resolve, 3000)
-      })
+    // insert into all three shard tables + a public table
+    await db.exec(
+      `INSERT INTO chat_0.clients ("clientGroupID", "clientID", "lastMutationID") VALUES ('cg1', 'c1', 1)`
+    )
+    await db.exec(`INSERT INTO chat_0.replicas (id, version) VALUES ('r1', 'v1')`)
+    await db.exec(
+      `INSERT INTO chat_0.mutations (id, "clientID", name) VALUES ('m1', 'c1', 'send')`
+    )
+    await db.exec(`INSERT INTO public.foo (id) VALUES ('normal')`)
 
-      await new Promise((r) => setTimeout(r, 500))
-      await regular.unsafe(`INSERT INTO public.items (id, val, note) VALUES ('pgjs', 42, 'postgres.js test')`)
-
-      await collectDone
-      readable.destroy()
-
-      // filter out keepalives
-      const data = messages.filter((m: any) => m.tag !== 'keepalive')
-
-      // should have: begin, relation, insert, commit
-      const tags = data.map((m: any) => m.tag)
-      expect(tags).toContain('begin')
-      expect(tags).toContain('relation')
-      expect(tags).toContain('insert')
-      expect(tags).toContain('commit')
-
-      // validate BEGIN has commitLsn as string (not BigInt)
-      const begin = data.find((m: any) => m.tag === 'begin')
-      expect(typeof begin.commitLsn).toBe('string')
-      expect(begin.commitLsn).toMatch(/^[0-9A-F]+\/[0-9A-F]+$/)
-
-      // validate RELATION has correct structure
-      const rel = data.find((m: any) => m.tag === 'relation')
-      expect(rel.schema).toBe('public')
-      expect(rel.name).toBe('items')
-      expect(rel.columns.length).toBe(3)
-
-      // validate INSERT has parsed values
-      const ins = data.find((m: any) => m.tag === 'insert')
-      expect(ins.relation.name).toBe('items')
-      expect(ins.new.id).toBe('pgjs')
-      expect(ins.new.val).toBe('42')
-      expect(ins.new.note).toBe('postgres.js test')
-
-      // validate COMMIT has commitLsn and commitEndLsn
-      const commit = data.find((m: any) => m.tag === 'commit')
-      expect(typeof commit.commitLsn).toBe('string')
-      expect(typeof commit.commitEndLsn).toBe('string')
-
-      // validate LSN ordering: commit.commitEndLsn > begin.commitLsn
-      // eslint-disable-next-line @typescript-eslint/no-require-imports
-      const { toBigInt: lsnToBigInt } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/services/change-source/pg/lsn.js')
-      expect(lsnToBigInt(commit.commitEndLsn)).toBeGreaterThan(lsnToBigInt(begin.commitLsn))
-
-      // validate lexi version conversion works (storer uses this)
-      // eslint-disable-next-line @typescript-eslint/no-require-imports
-      const { versionToLexi } = require('/Users/n8/orez/node_modules/@rocicorp/zero/out/zero-cache/src/types/lexi-version.js')
-      const beginVersion = versionToLexi(lsnToBigInt(begin.commitLsn))
-      const commitVersion = versionToLexi(lsnToBigInt(commit.commitEndLsn))
-      expect(typeof beginVersion).toBe('string')
-      expect(commitVersion > beginVersion).toBe(true)
-
-    } finally {
-      await regular.end()
-      // session.end() can hang because the replication handler keeps polling.
-      // just force-close the underlying connection by destroying the socket.
-      await session.end({ timeout: 2 }).catch(() => {})
+    // collect all inserts for a few seconds
+    const inserts: ZcInsert[] = []
+    const deadline = Date.now() + 4000
+    while (Date.now() < deadline) {
+      const m = await q.dequeue(1500).catch(() => null)
+      if (!m) break
+      if (m.tag === 'insert') inserts.push(m as ZcInsert)
     }
-  })
-
-  it('postgres.js handles concurrent regular + replication connections', { timeout: 30000 }, async () => {
-    const pg = (await import('postgres')).default
 
-    const regular = pg({
-      host: '127.0.0.1', port,
-      user: 'user', password: 'password', database: 'postgres',
-      max: 1,
-    })
+    // should see clients + foo inserts, but NOT replicas or mutations
+    const streamedTables = inserts.map((i) => `${i.relation.schema}.${i.relation.name}`)
+    expect(streamedTables).toContain('public.foo')
+    expect(streamedTables).toContain('chat_0.clients')
+    expect(streamedTables).not.toContain('chat_0.replicas')
+    expect(streamedTables).not.toContain('chat_0.mutations')
 
-    const session = pg({
-      host: '127.0.0.1', port,
-      user: 'user', password: 'password', database: 'postgres',
-      max: 1,
-      fetch_types: false,
-      idle_timeout: null,
-      max_lifetime: null,
-      connection: { replication: 'database' },
-    })
+    s.close()
+  })
 
-    try {
-      await session.unsafe(
-        `CREATE_REPLICATION_SLOT "conc_test" TEMPORARY LOGICAL pgoutput NOEXPORT_SNAPSHOT`
-      ).simple()
+  it('shard clients table created AFTER replication starts still gets tracked', async () => {
+    // zero-cache creates shard schemas after the replication connection is live.
+    // the poll loop must detect new shard tables and install triggers dynamically.
+    const s = await stream()
+    const q = s.messages
 
-      const stream = session.unsafe(
-        `START_REPLICATION SLOT "conc_test" LOGICAL 0/0 (proto_version '1', publication_names 'zero_data', messages 'true')`
-      ).execute()
+    // give replication time to start polling
+    await new Promise((r) => setTimeout(r, 300))
 
-      const [readable] = await Promise.all([stream.readable(), stream.writable()])
+    // now create shard schema (simulating zero-cache's DDL during initial sync)
+    await db.exec(`
+      CREATE SCHEMA chat_0;
+      CREATE TABLE chat_0.clients (
+        "clientGroupID" TEXT NOT NULL,
+        "clientID" TEXT NOT NULL,
+        "lastMutationID" BIGINT,
+        PRIMARY KEY ("clientGroupID", "clientID")
+      );
+    `)
 
-      const received: Buffer[] = []
-      readable.on('data', (chunk: Buffer) => received.push(chunk))
+    // wait for poll loop to detect new table (rescan every ~10s)
+    await new Promise((r) => setTimeout(r, 12000))
 
-      await new Promise((r) => setTimeout(r, 300))
+    // insert data that should be captured
+    await db.exec(
+      `INSERT INTO chat_0.clients ("clientGroupID", "clientID", "lastMutationID") VALUES ('cg1', 'c1', 42)`
+    )
 
-      // do 5 inserts via regular connection while replication is active
-      for (let i = 0; i < 5; i++) {
-        await regular.unsafe(`INSERT INTO public.items (id, val) VALUES ('c${i}', ${i})`)
+    // the insert should appear in the replication stream
+    let found = false
+    const deadline = Date.now() + 5000
+    while (Date.now() < deadline) {
+      const m = await q.dequeue(2000).catch(() => null)
+      if (!m) break
+      if (m.tag === 'insert') {
+        const ins = m as ZcInsert
+        if (ins.relation.schema === 'chat_0' && ins.relation.name === 'clients') {
+          found = true
+          break
+        }
       }
+    }
 
-      await new Promise((r) => setTimeout(r, 2000))
-      readable.destroy()
-
-      // should have received XLogData frames (0x77) from replication
-      const xlogFrames = received.filter((b) => b[0] === 0x77)
-      expect(xlogFrames.length).toBeGreaterThan(0)
+    expect(found).toBe(true)
 
-    } finally {
-      await regular.end()
-      await session.end({ timeout: 2 }).catch(() => {})
-    }
+    s.close()
   })
 })

package/src/shim/hooks.mjs

@@ -34,7 +34,7 @@ const OrigDatabase = mod.Database;
 const SqliteError = mod.SqliteError;
 function Database(...args) {
   const db = new OrigDatabase(...args);
-  try { db.pragma('busy_timeout = 30000'); db.pragma('synchronous = normal'); } catch(e) {}
+  try { db.pragma('journal_mode = delete'); db.pragma('busy_timeout = 30000'); db.pragma('synchronous = normal'); } catch(e) {}
   return db;
 }
 Database.prototype = OrigDatabase.prototype;

package/src/vite-plugin.ts

@@ -7,21 +7,16 @@ import type { Plugin } from 'vite'
 export interface OrezPluginOptions extends Partial<ZeroLiteConfig> {
   s3?: boolean
   s3Port?: number
-  admin?: boolean
-  adminPort?: number
-  adminLogs?: boolean
 }
 
 export default function orez(options?: OrezPluginOptions): Plugin {
   let stop: (() => Promise<void>) | null = null
   let s3Server: Server | null = null
-  let adminServer: Server | null = null
 
   return {
     name: 'orez',
 
     async configureServer(server) {
-      const startTime = Date.now()
       const result = await startZeroLite(options)
       stop = result.stop
 
@@ -33,30 +28,7 @@ export default function orez(options?: OrezPluginOptions): Plugin {
         })
       }
 
-      if (options?.admin && result.logStore) {
-        const { findPort } = await import('./port.js')
-        const { log } = await import('./log.js')
-        const adminPort = options.adminPort || result.config.zeroPort + 2
-        const resolvedPort = await findPort(adminPort)
-        const { startAdminServer } = await import('./admin/server.js')
-        adminServer = await startAdminServer({
-          port: resolvedPort,
-          logStore: result.logStore,
-          config: result.config,
-          zeroEnv: result.zeroEnv,
-          actions: result.actions,
-          startTime,
-          httpLog: result.httpLogStore || undefined,
-        })
-        log.orez(`admin: http://127.0.0.1:${resolvedPort}`)
-        if (result.config.adminLogs) {
-          const { resolve } = await import('node:path')
-          log.orez(`logs: ${resolve(result.config.dataDir, 'logs', 'orez.log')}`)
-        }
-      }
-
       server.httpServer?.on('close', async () => {
-        adminServer?.close()
         s3Server?.close()
         if (stop) {
           await stop()

package/src/wasm-sqlite.test.ts

@@ -19,8 +19,7 @@ import { resolve } from 'node:path'
 
 // import bedrock-sqlite directly (our wasm build)
 // @ts-expect-error - CJS module
-import bedrockSqlite from 'bedrock-sqlite'
-const { Database } = bedrockSqlite
+import { Database } from 'bedrock-sqlite'
 import { describe, test, expect, beforeEach, afterEach } from 'vitest'
 
 // helper: temp db file