orez 0.0.1

package/src/replication/change-tracker.test.ts

@@ -0,0 +1,179 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { PGlite } from '@electric-sql/pglite'
+import { installChangeTracking, getChangesSince, getCurrentWatermark } from './change-tracker'
+
+describe('change-tracker', () => {
+  let db: PGlite
+
+  beforeEach(async () => {
+    db = new PGlite()
+    await db.waitReady
+    await db.exec(`
+      CREATE TABLE public.items (
+        id SERIAL PRIMARY KEY,
+        name TEXT NOT NULL,
+        value INTEGER
+      )
+    `)
+    await installChangeTracking(db)
+  })
+
+  afterEach(async () => {
+    await db.close()
+  })
+
+  it('captures INSERT', async () => {
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1)`)
+
+    const changes = await getChangesSince(db, 0)
+    expect(changes).toHaveLength(1)
+    expect(changes[0].op).toBe('INSERT')
+    expect(changes[0].table_name).toBe('items')
+    expect(changes[0].row_data).toMatchObject({ name: 'a', value: 1 })
+    expect(changes[0].old_data).toBeNull()
+  })
+
+  it('captures UPDATE with old + new data', async () => {
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1)`)
+    await db.exec(`UPDATE public.items SET value = 99 WHERE name = 'a'`)
+
+    const changes = await getChangesSince(db, 0)
+    expect(changes).toHaveLength(2)
+    expect(changes[1].op).toBe('UPDATE')
+    expect(changes[1].row_data).toMatchObject({ value: 99 })
+    expect(changes[1].old_data).toMatchObject({ value: 1 })
+  })
+
+  it('captures DELETE with old data', async () => {
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1)`)
+    await db.exec(`DELETE FROM public.items WHERE name = 'a'`)
+
+    const changes = await getChangesSince(db, 0)
+    expect(changes).toHaveLength(2)
+    expect(changes[1].op).toBe('DELETE')
+    expect(changes[1].old_data).toMatchObject({ name: 'a', value: 1 })
+    expect(changes[1].row_data).toBeNull()
+  })
+
+  it('watermarks increase monotonically', async () => {
+    for (let i = 0; i < 5; i++) {
+      await db.exec(`INSERT INTO public.items (name, value) VALUES ('item${i}', ${i})`)
+    }
+
+    const changes = await getChangesSince(db, 0)
+    expect(changes).toHaveLength(5)
+    for (let i = 1; i < changes.length; i++) {
+      expect(changes[i].watermark).toBeGreaterThan(changes[i - 1].watermark)
+    }
+  })
+
+  it('getChangesSince filters by watermark', async () => {
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1)`)
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('b', 2)`)
+
+    const all = await getChangesSince(db, 0)
+    const afterFirst = await getChangesSince(db, all[0].watermark)
+
+    expect(afterFirst).toHaveLength(1)
+    expect(afterFirst[0].row_data).toMatchObject({ name: 'b' })
+  })
+
+  it('respects limit', async () => {
+    for (let i = 0; i < 10; i++) {
+      await db.exec(`INSERT INTO public.items (name, value) VALUES ('x', ${i})`)
+    }
+
+    const limited = await getChangesSince(db, 0, 3)
+    expect(limited).toHaveLength(3)
+  })
+
+  it('getCurrentWatermark returns 0 before any inserts', async () => {
+    const wm = await getCurrentWatermark(db)
+    expect(wm).toBe(0)
+  })
+
+  it('getCurrentWatermark advances', async () => {
+    // first insert consumes the initial sequence value
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('x', 1)`)
+    const before = await getCurrentWatermark(db)
+    expect(before).toBeGreaterThan(0)
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('y', 2)`)
+    const after = await getCurrentWatermark(db)
+    expect(after).toBeGreaterThan(before)
+  })
+
+  it('tracks multiple tables', async () => {
+    await db.exec(`CREATE TABLE public.other (id SERIAL PRIMARY KEY, label TEXT)`)
+    await installChangeTracking(db) // reinstall picks up new table
+
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1)`)
+    await db.exec(`INSERT INTO public.other (label) VALUES ('b')`)
+
+    const changes = await getChangesSince(db, 0)
+    const tables = new Set(changes.map((c) => c.table_name))
+    expect(tables).toContain('items')
+    expect(tables).toContain('other')
+  })
+
+  it('handles rapid inserts (50 rows)', async () => {
+    for (let i = 0; i < 50; i++) {
+      await db.exec(`INSERT INTO public.items (name, value) VALUES ('r${i}', ${i})`)
+    }
+
+    const changes = await getChangesSince(db, 0)
+    expect(changes).toHaveLength(50)
+
+    for (let i = 1; i < changes.length; i++) {
+      expect(changes[i].watermark).toBeGreaterThan(changes[i - 1].watermark)
+    }
+  })
+
+  it('does not track internal _zero_ tables', async () => {
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('x', 1)`)
+
+    const changes = await getChangesSince(db, 0)
+    // only the items insert, not the _zero_changes insert that the trigger itself caused
+    const internalChanges = changes.filter((c) => c.table_name.startsWith('_zero_'))
+    expect(internalChanges).toHaveLength(0)
+  })
+
+  it('handles NULL column values', async () => {
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('nulltest', NULL)`)
+
+    const changes = await getChangesSince(db, 0)
+    expect(changes[0].row_data).toMatchObject({ name: 'nulltest', value: null })
+  })
+
+  it('handles multi-row update', async () => {
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1), ('b', 2), ('c', 3)`)
+    await db.exec(`UPDATE public.items SET value = value * 10`)
+
+    const changes = await getChangesSince(db, 0)
+    const updates = changes.filter((c) => c.op === 'UPDATE')
+    expect(updates).toHaveLength(3)
+  })
+
+  it('preserves change ordering across mixed operations', async () => {
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1)`)
+    await db.exec(`UPDATE public.items SET value = 2 WHERE name = 'a'`)
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('b', 3)`)
+    await db.exec(`DELETE FROM public.items WHERE name = 'a'`)
+
+    const changes = await getChangesSince(db, 0)
+    const ops = changes.map((c) => c.op)
+    expect(ops).toEqual(['INSERT', 'UPDATE', 'INSERT', 'DELETE'])
+  })
+
+  it('tracks tables with special characters in names', async () => {
+    await db.exec(`CREATE TABLE public."my""table" (id SERIAL PRIMARY KEY, val TEXT)`)
+    await installChangeTracking(db)
+
+    await db.exec(`INSERT INTO public."my""table" (val) VALUES ('works')`)
+
+    const changes = await getChangesSince(db, 0)
+    const special = changes.filter((c) => c.table_name === 'my"table')
+    expect(special).toHaveLength(1)
+    expect(special[0].op).toBe('INSERT')
+    expect(special[0].row_data).toMatchObject({ val: 'works' })
+  })
+})

package/src/replication/change-tracker.ts

@@ -0,0 +1,115 @@
+import type { PGlite } from '@electric-sql/pglite'
+
+export interface ChangeRecord {
+  id: number
+  watermark: number
+  table_name: string
+  op: 'INSERT' | 'UPDATE' | 'DELETE'
+  row_data: Record<string, unknown> | null
+  old_data: Record<string, unknown> | null
+  changed_at: string
+}
+
+export async function installChangeTracking(db: PGlite): Promise<void> {
+  // create changes table and watermark sequence
+  await db.exec(`
+    CREATE SEQUENCE IF NOT EXISTS public._zero_watermark;
+
+    CREATE TABLE IF NOT EXISTS public._zero_changes (
+      id BIGSERIAL PRIMARY KEY,
+      watermark BIGINT NOT NULL DEFAULT nextval('public._zero_watermark'),
+      table_name TEXT NOT NULL,
+      op TEXT NOT NULL,
+      row_data JSONB,
+      old_data JSONB,
+      changed_at TIMESTAMPTZ DEFAULT NOW()
+    );
+
+    CREATE INDEX IF NOT EXISTS _zero_changes_watermark_idx ON public._zero_changes (watermark);
+
+    CREATE TABLE IF NOT EXISTS public._zero_replication_slots (
+      slot_name TEXT PRIMARY KEY,
+      restart_lsn TEXT NOT NULL DEFAULT '0/1000000',
+      confirmed_flush_lsn TEXT NOT NULL DEFAULT '0/1000000',
+      wal_status TEXT NOT NULL DEFAULT 'reserved',
+      plugin TEXT NOT NULL DEFAULT 'pgoutput',
+      slot_type TEXT NOT NULL DEFAULT 'logical',
+      active BOOLEAN NOT NULL DEFAULT false,
+      active_pid INTEGER DEFAULT NULL,
+      created_at TIMESTAMPTZ DEFAULT NOW()
+    );
+  `)
+
+  // create trigger function
+  await db.exec(`
+    CREATE OR REPLACE FUNCTION public._zero_track_change() RETURNS TRIGGER AS $$
+    BEGIN
+      IF TG_OP = 'DELETE' THEN
+        INSERT INTO public._zero_changes (table_name, op, old_data)
+        VALUES (TG_TABLE_NAME, 'DELETE', row_to_json(OLD)::jsonb);
+        RETURN OLD;
+      ELSIF TG_OP = 'UPDATE' THEN
+        INSERT INTO public._zero_changes (table_name, op, row_data, old_data)
+        VALUES (TG_TABLE_NAME, 'UPDATE', row_to_json(NEW)::jsonb, row_to_json(OLD)::jsonb);
+        RETURN NEW;
+      ELSIF TG_OP = 'INSERT' THEN
+        INSERT INTO public._zero_changes (table_name, op, row_data)
+        VALUES (TG_TABLE_NAME, 'INSERT', row_to_json(NEW)::jsonb);
+        RETURN NEW;
+      END IF;
+      RETURN NULL;
+    END;
+    $$ LANGUAGE plpgsql;
+  `)
+
+  // install triggers on all public tables
+  await installTriggersOnAllTables(db)
+}
+
+function quoteIdent(name: string): string {
+  return '"' + name.replace(/"/g, '""') + '"'
+}
+
+async function installTriggersOnAllTables(db: PGlite): Promise<void> {
+  const tables = await db.query<{ tablename: string }>(
+    `SELECT tablename FROM pg_tables
+     WHERE schemaname = 'public'
+       AND tablename NOT IN ('migrations', '_zero_changes')
+       AND tablename NOT LIKE '_zero_%'`
+  )
+
+  let count = 0
+  for (const { tablename } of tables.rows) {
+    const quoted = quoteIdent(tablename)
+    await db.exec(`
+      DROP TRIGGER IF EXISTS _zero_change_trigger ON public.${quoted};
+      CREATE TRIGGER _zero_change_trigger
+        AFTER INSERT OR UPDATE OR DELETE ON public.${quoted}
+        FOR EACH ROW EXECUTE FUNCTION public._zero_track_change();
+    `)
+    count++
+  }
+
+  console.info(`[orez] installed change tracking triggers on ${count} tables`)
+}
+
+export async function getChangesSince(
+  db: PGlite,
+  watermark: number,
+  limit = 1000
+): Promise<ChangeRecord[]> {
+  const result = await db.query<ChangeRecord>(
+    'SELECT * FROM public._zero_changes WHERE watermark > $1 ORDER BY watermark LIMIT $2',
+    [watermark, limit]
+  )
+  return result.rows
+}
+
+export async function getCurrentWatermark(db: PGlite): Promise<number> {
+  const result = await db.query<{ last_value: string; is_called: boolean }>(
+    'SELECT last_value, is_called FROM public._zero_watermark'
+  )
+  const { last_value, is_called } = result.rows[0]
+  if (!is_called) return 0
+  return Number(last_value)
+}

package/src/replication/handler.test.ts

@@ -0,0 +1,331 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { PGlite } from '@electric-sql/pglite'
+import {
+  handleReplicationQuery,
+  handleStartReplication,
+  type ReplicationWriter,
+} from './handler'
+import { installChangeTracking } from './change-tracker'
+
+// parse wire protocol RowDescription+DataRow response into columns/values
+function parseResponse(buf: Uint8Array): { columns: string[]; values: string[] } | null {
+  if (buf[0] !== 0x54) return null // RowDescription
+
+  const dv = new DataView(buf.buffer, buf.byteOffset)
+  let pos = 7
+  const numFields = dv.getInt16(5)
+  const columns: string[] = []
+  for (let i = 0; i < numFields; i++) {
+    let end = pos
+    while (buf[end] !== 0) end++
+    columns.push(new TextDecoder().decode(buf.subarray(pos, end)))
+    pos = end + 1 + 4 + 2 + 4 + 2 + 4 + 2
+  }
+
+  if (buf[pos] !== 0x44) return { columns, values: [] }
+  pos += 7
+  const values: string[] = []
+  for (let i = 0; i < numFields; i++) {
+    const len = dv.getInt32(pos)
+    pos += 4
+    values.push(new TextDecoder().decode(buf.subarray(pos, pos + len)))
+    pos += len
+  }
+
+  return { columns, values }
+}
+
+describe('handleReplicationQuery', () => {
+  let db: PGlite
+
+  beforeEach(async () => {
+    db = new PGlite()
+    await db.waitReady
+    await installChangeTracking(db)
+  })
+
+  afterEach(async () => {
+    await db.close()
+  })
+
+  it('IDENTIFY_SYSTEM returns system info', async () => {
+    const res = await handleReplicationQuery('IDENTIFY_SYSTEM', db)
+    expect(res).not.toBeNull()
+
+    const parsed = parseResponse(res!)
+    expect(parsed!.columns).toEqual(['systemid', 'timeline', 'xlogpos', 'dbname'])
+    expect(parsed!.values[0]).toBe('1234567890')
+    expect(parsed!.values[1]).toBe('1')
+    expect(parsed!.values[3]).toBe('postgres')
+    // xlogpos should be a valid LSN format
+    expect(parsed!.values[2]).toMatch(/^[0-9A-F]+\/[0-9A-F]+$/)
+  })
+
+  it('CREATE_REPLICATION_SLOT persists and returns slot info', async () => {
+    const res = await handleReplicationQuery(
+      'CREATE_REPLICATION_SLOT "test_slot" TEMPORARY LOGICAL pgoutput NOEXPORT_SNAPSHOT',
+      db
+    )
+
+    const parsed = parseResponse(res!)
+    expect(parsed!.values[0]).toBe('test_slot')
+    expect(parsed!.values[3]).toBe('pgoutput')
+
+    const slots = await db.query<{ slot_name: string }>(
+      `SELECT slot_name FROM public._zero_replication_slots WHERE slot_name = 'test_slot'`
+    )
+    expect(slots.rows).toHaveLength(1)
+  })
+
+  it('DROP_REPLICATION_SLOT removes slot', async () => {
+    await handleReplicationQuery('CREATE_REPLICATION_SLOT "drop_me" TEMPORARY LOGICAL pgoutput', db)
+    await handleReplicationQuery('DROP_REPLICATION_SLOT "drop_me"', db)
+
+    const slots = await db.query<{ count: string }>(
+      `SELECT count(*) as count FROM public._zero_replication_slots WHERE slot_name = 'drop_me'`
+    )
+    expect(Number(slots.rows[0].count)).toBe(0)
+  })
+
+  it('wal_level query returns logical', async () => {
+    const res = await handleReplicationQuery(
+      "SELECT current_setting('wal_level'), version()",
+      db
+    )
+    expect(res).not.toBeNull()
+    const parsed = parseResponse(res!)
+    expect(parsed!.values[0]).toBe('logical')
+  })
+
+  it('ALTER ROLE returns success', async () => {
+    const res = await handleReplicationQuery('ALTER ROLE user REPLICATION', db)
+    expect(res).not.toBeNull()
+    // should contain CommandComplete
+    expect(res![0]).toBe(0x43) // 'C'
+  })
+
+  it('returns null for unknown queries', async () => {
+    expect(await handleReplicationQuery('SELECT 1', db)).toBeNull()
+  })
+})
+
+describe('handleStartReplication', () => {
+  let db: PGlite
+  let replicationPromise: Promise<void>
+
+  beforeEach(async () => {
+    db = new PGlite()
+    await db.waitReady
+    await db.exec(`
+      CREATE TABLE public.items (
+        id SERIAL PRIMARY KEY,
+        name TEXT NOT NULL,
+        value INTEGER
+      )
+    `)
+    await installChangeTracking(db)
+  })
+
+  afterEach(async () => {
+    // closing db causes poll loop to exit with 'closed' error
+    await db.close()
+    // wait for the replication promise to settle
+    await replicationPromise?.catch(() => {})
+  })
+
+  function createWriter() {
+    const written: Uint8Array[] = []
+    const writer: ReplicationWriter = {
+      write(data: Uint8Array) {
+        written.push(new Uint8Array(data))
+      },
+    }
+    return { written, writer }
+  }
+
+  // extract the pgoutput message type from a CopyData(XLogData(msg)) frame
+  function payloadType(msg: Uint8Array): number | null {
+    if (msg[0] !== 0x64) return null // CopyData
+    if (msg[5] !== 0x77) return null // XLogData
+    return msg[30] // actual message type byte
+  }
+
+  it('sends CopyBothResponse first', async () => {
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db
+    )
+
+    await new Promise((r) => setTimeout(r, 200))
+
+    expect(written.length).toBeGreaterThan(0)
+    expect(written[0][0]).toBe(0x57) // 'W' CopyBothResponse
+  })
+
+  it('sends keepalives', async () => {
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db
+    )
+
+    await new Promise((r) => setTimeout(r, 700))
+
+    const keepalives = written.filter(
+      (msg) => msg[0] === 0x64 && msg[5] === 0x6b
+    )
+    expect(keepalives.length).toBeGreaterThan(0)
+  })
+
+  it('streams INSERT as BEGIN+RELATION+INSERT+COMMIT', async () => {
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db
+    )
+
+    await new Promise((r) => setTimeout(r, 100))
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('streamed', 123)`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    const types = written.map(payloadType).filter((t): t is number => t !== null)
+
+    expect(types).toContain(0x42) // BEGIN
+    expect(types).toContain(0x52) // RELATION
+    expect(types).toContain(0x49) // INSERT
+    expect(types).toContain(0x43) // COMMIT
+
+    // order: BEGIN before RELATION before INSERT before COMMIT
+    const beginIdx = types.indexOf(0x42)
+    const relIdx = types.indexOf(0x52)
+    const insIdx = types.indexOf(0x49)
+    const comIdx = types.indexOf(0x43)
+    expect(beginIdx).toBeLessThan(relIdx)
+    expect(relIdx).toBeLessThan(insIdx)
+    expect(insIdx).toBeLessThan(comIdx)
+  })
+
+  it('streams UPDATE and DELETE operations', async () => {
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db
+    )
+
+    await new Promise((r) => setTimeout(r, 100))
+
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('mut', 1)`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    await db.exec(`UPDATE public.items SET value = 2 WHERE name = 'mut'`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    await db.exec(`DELETE FROM public.items WHERE name = 'mut'`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    const types = written.map(payloadType).filter((t): t is number => t !== null)
+    expect(types).toContain(0x49) // INSERT
+    expect(types).toContain(0x55) // UPDATE
+    expect(types).toContain(0x44) // DELETE
+  }, 10_000)
+
+  it('only sends RELATION once per table', async () => {
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db
+    )
+
+    await new Promise((r) => setTimeout(r, 100))
+
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1)`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('b', 2)`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    const types = written.map(payloadType).filter((t): t is number => t !== null)
+    const relationCount = types.filter((t) => t === 0x52).length
+    expect(relationCount).toBe(1)
+  }, 10_000)
+
+  it('sends RELATION for each distinct table', async () => {
+    await db.exec(`CREATE TABLE public.other (id SERIAL PRIMARY KEY, label TEXT)`)
+    await installChangeTracking(db)
+
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db
+    )
+
+    await new Promise((r) => setTimeout(r, 100))
+
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('a', 1)`)
+    await db.exec(`INSERT INTO public.other (label) VALUES ('b')`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    const types = written.map(payloadType).filter((t): t is number => t !== null)
+    const relationCount = types.filter((t) => t === 0x52).length
+    expect(relationCount).toBe(2)
+  })
+
+  it('handles rapid sequential inserts', async () => {
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db
+    )
+
+    await new Promise((r) => setTimeout(r, 100))
+
+    for (let i = 0; i < 20; i++) {
+      await db.exec(`INSERT INTO public.items (name, value) VALUES ('r${i}', ${i})`)
+    }
+
+    // wait multiple poll cycles
+    await new Promise((r) => setTimeout(r, 1500))
+
+    const inserts = written.map(payloadType).filter((t) => t === 0x49)
+    expect(inserts.length).toBe(20)
+  }, 10_000)
+
+  it('each transaction has matching BEGIN and COMMIT', async () => {
+    const { written, writer } = createWriter()
+
+    replicationPromise = handleStartReplication(
+      'START_REPLICATION SLOT "s" LOGICAL 0/0',
+      writer,
+      db
+    )
+
+    await new Promise((r) => setTimeout(r, 100))
+
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('tx1', 1)`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    await db.exec(`INSERT INTO public.items (name, value) VALUES ('tx2', 2)`)
+    await new Promise((r) => setTimeout(r, 700))
+
+    const types = written.map(payloadType).filter((t): t is number => t !== null)
+    const begins = types.filter((t) => t === 0x42).length
+    const commits = types.filter((t) => t === 0x43).length
+    expect(begins).toBe(commits)
+    expect(begins).toBeGreaterThanOrEqual(1)
+  }, 10_000)
+})