orez 0.2.27 → 0.2.29

package/src/replication/change-tracker.ts

@@ -1,279 +0,0 @@
-import { log } from '../log.js'
-
-export interface ChangeRecord {
-  watermark: number
-  table_name: string
-  op: 'INSERT' | 'UPDATE' | 'DELETE'
-  row_data: Record<string, unknown> | null
-  old_data: Record<string, unknown> | null
-}
-
-/**
- * minimal db interface change tracking needs. any sql executor that provides
- * these two methods (e.g., a PGlite instance or a proxy wrapper) is accepted.
- * using a structural type here keeps this module free of heavy imports so it
- * can be loaded as the standalone `orez/change-tracking` entrypoint without
- * pulling in pglite-manager or other server-only modules.
- */
-export interface ChangeTrackingDb {
-  exec(sql: string): Promise<Array<{ affectedRows?: number }>>
-  query<T>(sql: string, params?: unknown[]): Promise<{ rows: T[] }>
-}
-
-// PGlite returns JSONB columns as parsed objects; the DO backend returns them
-// as JSON strings (it stores `row_data TEXT`). normalize once at the consumer
-// boundary so callers always get an object.
-function jsonRecord(value: unknown): Record<string, unknown> | null {
-  if (value === null || value === undefined) return null
-  if (typeof value === 'object') return value as Record<string, unknown>
-  if (typeof value !== 'string' || value === '') return null
-  return JSON.parse(value) as Record<string, unknown>
-}
-
-export async function installChangeTracking(db: ChangeTrackingDb): Promise<void> {
-  // use _orez schema for internal tables - survives pg_restore of public schema
-  await db.exec(`CREATE SCHEMA IF NOT EXISTS _orez`)
-
-  // create changes table and watermark sequence
-  // watermark is the primary key - monotonically increasing, no separate id needed
-  await db.exec(`
-    CREATE SEQUENCE IF NOT EXISTS _orez._zero_watermark;
-
-    CREATE TABLE IF NOT EXISTS _orez._zero_changes (
-      watermark BIGINT NOT NULL DEFAULT nextval('_orez._zero_watermark') PRIMARY KEY,
-      table_name TEXT NOT NULL,
-      op TEXT NOT NULL CHECK (op IN ('INSERT', 'UPDATE', 'DELETE')),
-      row_data JSONB,
-      old_data JSONB
-    );
-
-    CREATE TABLE IF NOT EXISTS _orez._zero_replication_slots (
-      slot_name TEXT PRIMARY KEY,
-      restart_lsn TEXT NOT NULL DEFAULT '0/1000000',
-      confirmed_flush_lsn TEXT NOT NULL DEFAULT '0/1000000',
-      wal_status TEXT NOT NULL DEFAULT 'reserved',
-      plugin TEXT NOT NULL DEFAULT 'pgoutput',
-      slot_type TEXT NOT NULL DEFAULT 'logical',
-      active BOOLEAN NOT NULL DEFAULT false,
-      active_pid INTEGER DEFAULT NULL,
-      created_at TIMESTAMPTZ DEFAULT NOW()
-    );
-  `)
-
-  // create trigger functions (writes to _orez schema)
-  // uses to_jsonb() directly instead of row_to_json()::jsonb to avoid double conversion.
-  // per-row trigger for single-row operations
-  await db.exec(`
-    CREATE OR REPLACE FUNCTION public._zero_track_change() RETURNS TRIGGER AS $$
-    BEGIN
-      IF TG_OP = 'DELETE' THEN
-        INSERT INTO _orez._zero_changes (table_name, op, old_data)
-        VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'DELETE', to_jsonb(OLD));
-        RETURN OLD;
-      ELSIF TG_OP = 'UPDATE' THEN
-        -- skip no-op updates where no columns actually changed
-        IF to_jsonb(NEW) = to_jsonb(OLD) THEN
-          RETURN NEW;
-        END IF;
-        INSERT INTO _orez._zero_changes (table_name, op, row_data, old_data)
-        VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'UPDATE', to_jsonb(NEW), to_jsonb(OLD));
-        RETURN NEW;
-      ELSE
-        INSERT INTO _orez._zero_changes (table_name, op, row_data)
-        VALUES (TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, 'INSERT', to_jsonb(NEW));
-        RETURN NEW;
-      END IF;
-    END;
-    $$ LANGUAGE plpgsql;
-  `)
-
-  // install triggers on all public tables
-  await installTriggersOnAllTables(db)
-}
-
-function quoteIdent(name: string): string {
-  return '"' + name.replace(/"/g, '""') + '"'
-}
-
-async function installTriggersOnAllTables(db: ChangeTrackingDb): Promise<void> {
-  // If a publication is configured, respect it strictly. This avoids accidentally
-  // streaming private tables when publication membership is temporarily empty.
-  const pubName = process.env.ZERO_APP_PUBLICATIONS?.trim()
-  let tables: { tablename: string }[]
-  if (pubName) {
-    const result = await db.query<{ tablename: string }>(
-      `SELECT tablename FROM pg_publication_tables
-       WHERE pubname = $1
-         AND schemaname = 'public'
-         AND tablename NOT LIKE '_zero_%'`,
-      [pubName]
-    )
-    tables = result.rows
-    if (tables.length > 0) {
-      log.debug.pglite(`using publication "${pubName}" (${tables.length} tables)`)
-    } else {
-      log.pglite(`publication "${pubName}" is empty; installing no public table triggers`)
-    }
-  } else {
-    const all = await db.query<{ tablename: string }>(
-      `SELECT tablename FROM pg_tables
-       WHERE schemaname = 'public'
-         AND tablename NOT IN ('migrations')
-         AND tablename NOT LIKE '_zero_%'`
-    )
-    tables = all.rows
-    log.debug.pglite(`using all public tables (${tables.length})`)
-  }
-
-  // drop stale triggers from tables NOT in the publication
-  // (these may exist from a prior install before the publication was created)
-  const publishedSet = new Set(tables.map((t) => t.tablename))
-  const allTriggered = await db.query<{ event_object_table: string }>(
-    `SELECT DISTINCT event_object_table FROM information_schema.triggers
-     WHERE trigger_name = '_zero_change_trigger'
-       AND event_object_schema = 'public'`
-  )
-  for (const { event_object_table } of allTriggered.rows) {
-    if (!publishedSet.has(event_object_table)) {
-      const quoted = quoteIdent(event_object_table)
-      await db.exec(`DROP TRIGGER IF EXISTS _zero_change_trigger ON public.${quoted}`)
-      log.debug.pglite(
-        `removed stale trigger from non-published table: ${event_object_table}`
-      )
-    }
-  }
-
-  let count = 0
-  for (const { tablename } of tables) {
-    const quoted = quoteIdent(tablename)
-    await db.exec(`
-      DROP TRIGGER IF EXISTS _zero_change_trigger ON public.${quoted};
-      CREATE TRIGGER _zero_change_trigger
-        AFTER INSERT OR UPDATE OR DELETE ON public.${quoted}
-        FOR EACH ROW EXECUTE FUNCTION public._zero_track_change();
-    `)
-    count++
-  }
-
-  if (count > 0) {
-    log.debug.pglite(`installed change tracking triggers on ${count} tables`)
-  } else {
-    log.debug.pglite(`no tables to install change tracking triggers on`)
-  }
-}
-
-/**
- * install change tracking triggers on tables in shard schemas.
- * zero-cache creates shard schemas (e.g. chat_0) with clients/mutations
- * tables that track mutation confirmations. these must be replicated
- * for .server promises to resolve.
- *
- * caches already-tracked shard tables to avoid redundant DDL while still
- * handling zero-cache creating a schema before the internal tables exist.
- */
-const trackedShardTables = new Set<string>()
-const TRACKED_SHARD_TABLES = new Set(['clients', 'mutations'])
-
-/** reset shard schema cache (for tests) */
-export function resetShardSchemaCache(): void {
-  trackedShardTables.clear()
-}
-
-export async function installTriggersOnShardTables(db: ChangeTrackingDb): Promise<void> {
-  const result = await db.query<{ nspname: string }>(
-    `SELECT nspname FROM pg_namespace
-     WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'public')
-       AND nspname NOT LIKE 'pg_%'
-       AND nspname NOT LIKE 'zero_%'
-       AND nspname NOT LIKE '_zero_%'
-       AND nspname NOT LIKE '%/%'`
-  )
-
-  if (result.rows.length === 0) return
-
-  // only track the shard tables zero-cache expects in the replication stream.
-  // `clients` advances LMID for successful .server promises; `mutations`
-  // carries per-mutation results for application errors. other shard tables
-  // like `replicas` are zero-cache internal state and streaming them back
-  // causes "Unknown table" crashes in zero-cache's change-processor.
-  let count = 0
-  for (const { nspname } of result.rows) {
-    // remove stale triggers from non-replicated shard tables.
-    const stale = await db.query<{ event_object_table: string }>(
-      `SELECT DISTINCT event_object_table FROM information_schema.triggers
-       WHERE trigger_name = '_zero_change_trigger'
-         AND event_object_schema = $1
-         AND event_object_table != ALL($2)`,
-      [nspname, [...TRACKED_SHARD_TABLES]]
-    )
-    for (const { event_object_table } of stale.rows) {
-      const qs = quoteIdent(nspname)
-      const qt = quoteIdent(event_object_table)
-      await db.exec(`DROP TRIGGER IF EXISTS _zero_change_trigger ON ${qs}.${qt}`)
-      log.debug.pglite(
-        `removed stale shard trigger from ${nspname}.${event_object_table}`
-      )
-    }
-
-    const tables = await db.query<{ tablename: string }>(
-      `SELECT tablename FROM pg_tables WHERE schemaname = $1 AND tablename = ANY($2)`,
-      [nspname, [...TRACKED_SHARD_TABLES]]
-    )
-
-    for (const { tablename } of tables.rows) {
-      const key = `${nspname}.${tablename}`
-      if (trackedShardTables.has(key)) continue
-
-      const quotedSchema = quoteIdent(nspname)
-      const quotedTable = quoteIdent(tablename)
-      await db.exec(`
-        DROP TRIGGER IF EXISTS _zero_change_trigger ON ${quotedSchema}.${quotedTable};
-        CREATE TRIGGER _zero_change_trigger
-          AFTER INSERT OR UPDATE OR DELETE ON ${quotedSchema}.${quotedTable}
-          FOR EACH ROW EXECUTE FUNCTION public._zero_track_change();
-      `)
-      trackedShardTables.add(key)
-      count++
-    }
-  }
-
-  if (count > 0) {
-    log.debug.pglite(`installed change tracking on ${count} shard tables`)
-  }
-}
-
-export async function getChangesSince(
-  db: ChangeTrackingDb,
-  watermark: number,
-  limit = 50000
-): Promise<ChangeRecord[]> {
-  const result = await db.query<ChangeRecord>(
-    'SELECT watermark, table_name, op, row_data, old_data FROM _orez._zero_changes WHERE watermark > $1 ORDER BY watermark LIMIT $2',
-    [watermark, limit]
-  )
-  return result.rows.map((row) => ({
-    ...row,
-    watermark: Number(row.watermark),
-    row_data: jsonRecord(row.row_data),
-    old_data: jsonRecord(row.old_data),
-  }))
-}
-
-export async function purgeConsumedChanges(
-  db: ChangeTrackingDb,
-  watermark: number
-): Promise<number> {
-  const result = await db.exec(
-    `DELETE FROM _orez._zero_changes WHERE watermark <= ${Number(watermark)}`
-  )
-  return result[0]?.affectedRows ?? 0
-}
-
-export async function getCurrentWatermark(db: ChangeTrackingDb): Promise<number> {
-  const result = await db.query<{ last_value: string; is_called: boolean }>(
-    'SELECT last_value, is_called FROM _orez._zero_watermark'
-  )
-  const { last_value, is_called } = result.rows[0]
-  if (!is_called) return 0
-  return Number(last_value)
-}