@reventlessdev/reventless-postgres 3.0.0-alpha.3 → 3.0.0-alpha.4

package/src/EventLogChangeFeed.res.mjs

@@ -0,0 +1,134 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as PgDriver$ReventlessPostgres from "./PgDriver.res.mjs";
+import * as DcbEventLogStorage_Postgres$ReventlessPostgres from "./DcbEventLogStorage_Postgres.res.mjs";
+
+let selectColumns = "\n  lpad(transaction_id::text, 20, '0') || ':' || lpad(global_seq::text, 20, '0') AS cursor,\n  aggregate_id,\n  seq_nr,\n  payload,\n  msg_id\n";
+
+function coerceInt(v) {
+  switch (typeof v) {
+    case "string" :
+      return Stdlib_Option.getOr(Stdlib_Int.fromString(v, undefined), 0);
+    case "number" :
+      return v | 0;
+    default:
+      return 0;
+  }
+}
+
+function rowToEvent(row) {
+  let str = key => {
+    let match = row[key];
+    if (typeof match === "string") {
+      return match;
+    } else {
+      return "";
+    }
+  };
+  let match = row["msg_id"];
+  let tmp;
+  tmp = typeof match === "string" ? match : undefined;
+  return {
+    cursor: str("cursor"),
+    aggregateId: str("aggregate_id"),
+    seqNr: Stdlib_Option.getOr(Stdlib_Option.map(row["seq_nr"], coerceInt), 0),
+    payload: Stdlib_Option.getOr(row["payload"], null),
+    msgId: tmp
+  };
+}
+
+async function readBatch(pool, logName, after, limitOpt) {
+  let limit = limitOpt !== undefined ? limitOpt : 500;
+  let params = [logName];
+  let where = "log_name = $1 AND transaction_id < pg_snapshot_xmin(pg_current_snapshot())";
+  let match = Stdlib_Option.flatMap(after, DcbEventLogStorage_Postgres$ReventlessPostgres.decodeCursor);
+  if (match !== undefined) {
+    params.push(match[0]);
+    params.push(match[1]);
+    where = where + " AND (transaction_id > $2::xid8 OR (transaction_id = $2::xid8 AND global_seq > $3::bigint))";
+  }
+  let sql = `SELECT ` + selectColumns + ` FROM event_log WHERE ` + where + ` ORDER BY transaction_id ASC, global_seq ASC LIMIT ` + limit.toString();
+  let rows = await PgDriver$ReventlessPostgres.query(pool, sql, params);
+  let events = rows.map(rowToEvent);
+  let last = events[events.length - 1 | 0];
+  let cursor = last !== undefined ? last.cursor : undefined;
+  return {
+    events: events,
+    cursor: cursor
+  };
+}
+
+async function loadCheckpoint(pool, subscriber) {
+  let row = await PgDriver$ReventlessPostgres.queryOne(pool, "SELECT lpad(last_tx::text, 20, '0') || ':' || lpad(last_global_seq::text, 20, '0') AS cursor\n       FROM event_log_subscription WHERE subscriber = $1 AND last_global_seq > 0", [subscriber]);
+  if (row === undefined) {
+    return;
+  }
+  let match = row["cursor"];
+  if (typeof match === "string") {
+    return match;
+  }
+}
+
+async function saveCheckpoint(pool, subscriber, cursor) {
+  let match = DcbEventLogStorage_Postgres$ReventlessPostgres.decodeCursor(cursor);
+  if (match !== undefined) {
+    await PgDriver$ReventlessPostgres.query(pool, "INSERT INTO event_log_subscription(subscriber, last_tx, last_global_seq)\n       VALUES ($1, $2::xid8, $3::bigint)\n       ON CONFLICT (subscriber)\n       DO UPDATE SET last_tx = EXCLUDED.last_tx, last_global_seq = EXCLUDED.last_global_seq", [
+      subscriber,
+      match[0],
+      match[1]
+    ]);
+    return;
+  }
+}
+
+function listen(pool, logName, onWake) {
+  return PgDriver$ReventlessPostgres.listen(pool, "evlog_" + logName, param => onWake());
+}
+
+function unlisten(client, logName) {
+  return PgDriver$ReventlessPostgres.unlisten(client, "evlog_" + logName);
+}
+
+async function drain(pool, logName, subscriber, limitOpt, handle) {
+  let limit = limitOpt !== undefined ? limitOpt : 500;
+  let processed = 0;
+  let cursor = await loadCheckpoint(pool, subscriber);
+  let $$continue = true;
+  while ($$continue) {
+    let match = await readBatch(pool, logName, cursor, limit);
+    let events = match.events;
+    if (events.length === 0) {
+      $$continue = false;
+    } else {
+      let newCursor = match.cursor;
+      await handle(events);
+      if (newCursor !== undefined) {
+        await saveCheckpoint(pool, subscriber, newCursor);
+        cursor = newCursor;
+      }
+      processed = processed + events.length | 0;
+      if (events.length < limit) {
+        $$continue = false;
+      }
+    }
+  };
+  return processed;
+}
+
+let Dcb;
+
+export {
+  Dcb,
+  selectColumns,
+  coerceInt,
+  rowToEvent,
+  readBatch,
+  loadCheckpoint,
+  saveCheckpoint,
+  listen,
+  unlisten,
+  drain,
+}
+/* PgDriver-ReventlessPostgres Not a pure module */

package/src/PgSchema.res

@@ -36,6 +36,19 @@ let tableStatements = [
      schema_hash   text   NOT NULL,
      PRIMARY KEY (log_name, aggregate_id)
    )",
+  // --- Classic change feed (B2.5) ---
+  // Stamp every classic event with its committing xid8 so the feed can apply the
+  // same xmin read barrier the DCB feed uses — global_seq (an IDENTITY assigned at
+  // INSERT, before commit) is unsafe as a feed cursor on its own (a late-committing
+  // lower global_seq would be skipped). Idempotent; the volatile default auto-stamps
+  // every future insert with no change to EventLogStorage_Postgres.append.
+  "ALTER TABLE event_log ADD COLUMN IF NOT EXISTS transaction_id xid8 NOT NULL DEFAULT pg_current_xact_id()",
+  "CREATE INDEX IF NOT EXISTS event_log_tx_gseq ON event_log (log_name, transaction_id, global_seq)",
+  "CREATE TABLE IF NOT EXISTS event_log_subscription (
+     subscriber      text   PRIMARY KEY,
+     last_tx         xid8   NOT NULL DEFAULT '0'::xid8,
+     last_global_seq bigint NOT NULL DEFAULT 0
+   )",
   // --- DCB event log (Phase B) ---
   "CREATE TABLE IF NOT EXISTS dcb_event (
      log_name        text        NOT NULL,
@@ -236,7 +249,28 @@ END;
 $fn$;
 "
 
-let functionStatements = [dcbConditionWhereFn, dcbAppendFn]
+// Classic change-feed wakeup (B2.5). A statement-level AFTER INSERT trigger fires
+// one pg_notify per distinct log_name touched by the statement, so the classic feed
+// gets the same near-real-time wakeup the DCB append's inline pg_notify provides —
+// without touching EventLogStorage_Postgres.append. No-op if nobody is LISTENing; a
+// low-frequency fallback tick in the consumer covers missed notifications.
+let eventLogNotifyFn = "
+CREATE OR REPLACE FUNCTION event_log_notify() RETURNS trigger
+LANGUAGE plpgsql AS $fn$
+BEGIN
+  PERFORM pg_notify('evlog_' || log_name, '')
+    FROM (SELECT DISTINCT log_name FROM new_rows) s;
+  RETURN NULL;
+END;
+$fn$;
+
+CREATE OR REPLACE TRIGGER event_log_notify_trg
+  AFTER INSERT ON event_log
+  REFERENCING NEW TABLE AS new_rows
+  FOR EACH STATEMENT EXECUTE FUNCTION event_log_notify();
+"
+
+let functionStatements = [dcbConditionWhereFn, dcbAppendFn, eventLogNotifyFn]
 
 let ensureSchema = async (pool: PgDriver.pool): unit => {
   for i in 0 to tableStatements->Array.length - 1 {
@@ -251,6 +285,6 @@ let ensureSchema = async (pool: PgDriver.pool): unit => {
 // Mirrors the reventless-local contract that reset wipes data, not structure.
 let truncateAll = async (pool: PgDriver.pool): unit => {
   await pool->PgDriver.exec(
-    "TRUNCATE event_log, snapshot, dcb_event, dcb_scope, dcb_subscription RESTART IDENTITY",
+    "TRUNCATE event_log, snapshot, dcb_event, dcb_scope, dcb_subscription, event_log_subscription RESTART IDENTITY",
   )
 }

package/src/PgSchema.res.mjs

@@ -5,6 +5,9 @@ import * as PgDriver$ReventlessPostgres from "./PgDriver.res.mjs";
 let tableStatements = [
   "CREATE TABLE IF NOT EXISTS event_log (\n     log_name      text   NOT NULL,\n     aggregate_id  text   NOT NULL,\n     seq_nr        bigint NOT NULL,\n     payload       jsonb  NOT NULL,\n     msg_id        text,\n     global_seq    bigint GENERATED ALWAYS AS IDENTITY,\n     PRIMARY KEY (log_name, aggregate_id, seq_nr)\n   )",
   "CREATE TABLE IF NOT EXISTS snapshot (\n     log_name      text   NOT NULL,\n     aggregate_id  text   NOT NULL,\n     seq_nr        bigint NOT NULL,\n     state         jsonb  NOT NULL,\n     schema_hash   text   NOT NULL,\n     PRIMARY KEY (log_name, aggregate_id)\n   )",
+  "ALTER TABLE event_log ADD COLUMN IF NOT EXISTS transaction_id xid8 NOT NULL DEFAULT pg_current_xact_id()",
+  "CREATE INDEX IF NOT EXISTS event_log_tx_gseq ON event_log (log_name, transaction_id, global_seq)",
+  "CREATE TABLE IF NOT EXISTS event_log_subscription (\n     subscriber      text   PRIMARY KEY,\n     last_tx         xid8   NOT NULL DEFAULT '0'::xid8,\n     last_global_seq bigint NOT NULL DEFAULT 0\n   )",
   "CREATE TABLE IF NOT EXISTS dcb_event (\n     log_name        text        NOT NULL,\n     position        bigint      GENERATED ALWAYS AS IDENTITY,\n     transaction_id  xid8        NOT NULL DEFAULT pg_current_xact_id(),\n     event_type      text        NOT NULL,\n     tags            text[]      NOT NULL DEFAULT '{}',\n     data            jsonb       NOT NULL,\n     meta            jsonb       NOT NULL,\n     recorded_at     timestamptz NOT NULL DEFAULT now(),\n     PRIMARY KEY (log_name, position)\n   )",
   "CREATE INDEX IF NOT EXISTS dcb_event_tags_gin ON dcb_event USING gin (tags)",
   "CREATE INDEX IF NOT EXISTS dcb_event_type_pos ON dcb_event (log_name, event_type, position)",
@@ -17,9 +20,12 @@ let dcbConditionWhereFn = "\nCREATE OR REPLACE FUNCTION dcb_after_frag(p_tx xid8
 
 let dcbAppendFn = "\nCREATE OR REPLACE FUNCTION dcb_append(\n  p_log_name      text,\n  p_events        jsonb,\n  p_condition     jsonb DEFAULT NULL,\n  p_lock_strategy text  DEFAULT 'advisory'\n) RETURNS text\nLANGUAGE plpgsql AS $fn$\nDECLARE\n  v_lock_keys bigint[];\n  v_after_raw text;\n  v_after_tx  xid8;\n  v_after_pos bigint;\n  v_where     text;\n  v_conflict  boolean;\n  v_last_pos  bigint;\n  v_key       bigint;\nBEGIN\n  -- 1. Lock keys: every tag in written events UNION every tag in the condition.\n  SELECT array_agg(DISTINCT k ORDER BY k) INTO v_lock_keys\n  FROM (\n    SELECT hashtextextended(p_log_name || '=' || tag, 0) AS k\n      FROM jsonb_array_elements(p_events) ev,\n           jsonb_array_elements_text(ev->'tags') tag\n    UNION\n    SELECT hashtextextended(p_log_name || '=' || tag, 0) AS k\n      FROM jsonb_array_elements(COALESCE(p_condition->'query', '[]'::jsonb)) qi,\n           jsonb_array_elements_text(COALESCE(qi->'tags', '[]'::jsonb)) tag\n  ) s;\n\n  -- 2. Acquire in sorted order.\n  IF v_lock_keys IS NOT NULL THEN\n    IF p_lock_strategy = 'rows' THEN\n      INSERT INTO dcb_scope(log_name, scope_hash)\n        SELECT p_log_name, k FROM unnest(v_lock_keys) k\n        ON CONFLICT DO NOTHING;\n      PERFORM 1 FROM dcb_scope\n        WHERE log_name = p_log_name AND scope_hash = ANY(v_lock_keys)\n        ORDER BY scope_hash\n        FOR UPDATE;\n    ELSE\n      FOREACH v_key IN ARRAY v_lock_keys LOOP\n        PERFORM pg_advisory_xact_lock(v_key);\n      END LOOP;\n    END IF;\n  END IF;\n\n  -- 3. Exact condition check under the held locks (READ COMMITTED is race-free\n  --    here: a conflicting concurrent writer is either committed+visible or\n  --    blocked on our lock).\n  IF p_condition IS NOT NULL THEN\n    v_after_raw := NULLIF(p_condition->>'after', '');\n    IF v_after_raw IS NOT NULL THEN\n      -- Cursor is '<xid8>:<position>', each zero-padded to 20 digits. Strip the\n      -- padding before casting (bigint tolerates leading zeros; be explicit for\n      -- xid8, guarding the all-zeros case).\n      v_after_tx  := COALESCE(NULLIF(ltrim(split_part(v_after_raw, ':', 1), '0'), ''), '0')::xid8;\n      v_after_pos := split_part(v_after_raw, ':', 2)::bigint;\n    END IF;\n    v_where := dcb_condition_where(p_log_name, p_condition->'query', v_after_tx, v_after_pos);\n    EXECUTE 'SELECT EXISTS (SELECT 1 FROM dcb_event WHERE ' || v_where || ')'\n      INTO v_conflict;\n    IF v_conflict THEN\n      RAISE EXCEPTION 'dcb_conflict' USING ERRCODE = 'P0001';\n    END IF;\n  END IF;\n\n  -- 4. Insert every event in array order; positions come from the IDENTITY seq.\n  WITH ins AS (\n    INSERT INTO dcb_event(log_name, event_type, tags, data, meta)\n    SELECT p_log_name,\n           ev->>'event_type',\n           ARRAY(SELECT jsonb_array_elements_text(ev->'tags')),\n           ev->'data',\n           ev->'meta'\n      FROM jsonb_array_elements(p_events) WITH ORDINALITY AS t(ev, ord)\n     ORDER BY ord\n    RETURNING position\n  )\n  SELECT max(position) INTO v_last_pos FROM ins;\n\n  -- 5. Wake the change feed (Phase D). No-op if nobody is LISTENing.\n  PERFORM pg_notify('dcb_' || p_log_name, v_last_pos::text);\n\n  RETURN lpad(pg_current_xact_id()::text, 20, '0') || ':' || lpad(v_last_pos::text, 20, '0');\nEND;\n$fn$;\n";
 
+let eventLogNotifyFn = "\nCREATE OR REPLACE FUNCTION event_log_notify() RETURNS trigger\nLANGUAGE plpgsql AS $fn$\nBEGIN\n  PERFORM pg_notify('evlog_' || log_name, '')\n    FROM (SELECT DISTINCT log_name FROM new_rows) s;\n  RETURN NULL;\nEND;\n$fn$;\n\nCREATE OR REPLACE TRIGGER event_log_notify_trg\n  AFTER INSERT ON event_log\n  REFERENCING NEW TABLE AS new_rows\n  FOR EACH STATEMENT EXECUTE FUNCTION event_log_notify();\n";
+
 let functionStatements = [
   dcbConditionWhereFn,
-  dcbAppendFn
+  dcbAppendFn,
+  eventLogNotifyFn
 ];
 
 async function ensureSchema(pool) {
@@ -32,13 +38,14 @@ async function ensureSchema(pool) {
 }
 
 async function truncateAll(pool) {
-  return await PgDriver$ReventlessPostgres.exec(pool, "TRUNCATE event_log, snapshot, dcb_event, dcb_scope, dcb_subscription RESTART IDENTITY");
+  return await PgDriver$ReventlessPostgres.exec(pool, "TRUNCATE event_log, snapshot, dcb_event, dcb_scope, dcb_subscription, event_log_subscription RESTART IDENTITY");
 }
 
 export {
   tableStatements,
   dcbConditionWhereFn,
   dcbAppendFn,
+  eventLogNotifyFn,
   functionStatements,
   ensureSchema,
   truncateAll,

package/tests/PostgresIntegrationTest.res

@@ -233,4 +233,72 @@ switch processEnv->Dict.get("PG_URL") {
       expect(again)->toBe(0)
     })
   })
+
+  describe("classic change feed (event_log, B2.5)", () => {
+    testPromise("drains classic events in commit order, with fields, then checkpoints", async () => {
+      let (_n, ops, _s) = EventLogStorage_Postgres.makeStorage(~pool, ~name="cf-orders", ~opts)
+      let _ = await ops.append(0, "agg-1", [evJson("m0", 1), evJson("m1", 2)])
+      let _ = await ops.append(0, "agg-2", [evJson("m2", 3)])
+
+      let collected = ref([])
+      let processed = await EventLogChangeFeed.drain(
+        pool,
+        ~logName="cf-orders",
+        ~subscriber="cf-sub",
+        ~handle=async evs => collected := collected.contents->Array.concat(evs),
+      )
+      expect(processed)->toBe(3)
+      expect(collected.contents->Array.length)->toBe(3)
+
+      // Fields round-trip from the stored row.
+      let first = collected.contents->Array.getUnsafe(0)
+      expect(first.aggregateId)->toBe("agg-1")
+      expect(first.seqNr)->toBe(0)
+      expect(first.msgId)->toEqual(Some("m0"))
+
+      // Cursors are strictly increasing in drain order.
+      let cursors = collected.contents->Array.map(e => e.cursor)
+      let increasing = ref(true)
+      for i in 1 to cursors->Array.length - 1 {
+        if cursors->Array.getUnsafe(i) <= cursors->Array.getUnsafe(i - 1) {
+          increasing := false
+        }
+      }
+      expect(increasing.contents)->toBe(true)
+
+      // Second drain from the saved checkpoint sees nothing new.
+      let again = await EventLogChangeFeed.drain(
+        pool,
+        ~logName="cf-orders",
+        ~subscriber="cf-sub",
+        ~handle=async _ => (),
+      )
+      expect(again)->toBe(0)
+    })
+
+    testPromise("concurrent appends across aggregates: every event drained exactly once", async () => {
+      let (_n, ops, _s) = EventLogStorage_Postgres.makeStorage(~pool, ~name="cf-concurrent", ~opts)
+      let n = 20
+      // One event per aggregate, all dispatched before awaiting — genuinely
+      // concurrent on the pool, so global_seq is assigned in interleaved/undefined
+      // commit order. The xmin fence must still drain every event exactly once.
+      let promises =
+        Array.make(~length=n, 0)->Array.mapWithIndex((_, i) =>
+          ops.append(0, "agg-" ++ Int.toString(i), [evJson("m" ++ Int.toString(i), i)])
+        )
+      let _ = await Promise.all(promises)
+
+      let collected = ref([])
+      let _ = await EventLogChangeFeed.drain(
+        pool,
+        ~logName="cf-concurrent",
+        ~subscriber="cf-conc-sub",
+        ~handle=async evs => collected := collected.contents->Array.concat(evs),
+      )
+      // No skips: all N events present. No dupes: N distinct cursors.
+      expect(collected.contents->Array.length)->toBe(n)
+      let distinctCursors = collected.contents->Array.map(e => e.cursor)->Set.fromArray
+      expect(distinctCursors->Set.size)->toBe(n)
+    })
+  })
 }

package/tests/PostgresIntegrationTest.res.mjs

@@ -6,6 +6,7 @@ import * as Message$ReventlessCore from "@reventlessdev/reventless-core/src/Mess
 import * as PgDriver$ReventlessPostgres from "../src/PgDriver.res.mjs";
 import * as PgSchema$ReventlessPostgres from "../src/PgSchema.res.mjs";
 import * as PgChangeFeed$ReventlessPostgres from "../src/PgChangeFeed.res.mjs";
+import * as EventLogChangeFeed$ReventlessPostgres from "../src/EventLogChangeFeed.res.mjs";
 import * as EventLogStorage_Postgres$ReventlessPostgres from "../src/EventLogStorage_Postgres.res.mjs";
 import * as DcbEventLogStorage_Postgres$ReventlessPostgres from "../src/DcbEventLogStorage_Postgres.res.mjs";
 
@@ -313,6 +314,54 @@ if (url !== undefined) {
       globalThis.expect(again).toBe(0);
     });
   });
+  globalThis.describe("classic change feed (event_log, B2.5)", () => {
+    globalThis.test("drains classic events in commit order, with fields, then checkpoints", async () => {
+      let match = EventLogStorage_Postgres$ReventlessPostgres.makeStorage(pool, "cf-orders", opts, undefined);
+      let ops = match[1];
+      await ops.append(0, "agg-1", [
+        evJson("m0", 1),
+        evJson("m1", 2)
+      ]);
+      await ops.append(0, "agg-2", [evJson("m2", 3)]);
+      let collected = {
+        contents: []
+      };
+      let processed = await EventLogChangeFeed$ReventlessPostgres.drain(pool, "cf-orders", "cf-sub", undefined, async evs => {
+        collected.contents = collected.contents.concat(evs);
+      });
+      globalThis.expect(processed).toBe(3);
+      globalThis.expect(collected.contents.length).toBe(3);
+      let first = collected.contents[0];
+      globalThis.expect(first.aggregateId).toBe("agg-1");
+      globalThis.expect(first.seqNr).toBe(0);
+      globalThis.expect(first.msgId).toEqual("m0");
+      let cursors = collected.contents.map(e => e.cursor);
+      let increasing = true;
+      for (let i = 1, i_finish = cursors.length; i < i_finish; ++i) {
+        if (cursors[i] <= cursors[i - 1 | 0]) {
+          increasing = false;
+        }
+      }
+      globalThis.expect(increasing).toBe(true);
+      let again = await EventLogChangeFeed$ReventlessPostgres.drain(pool, "cf-orders", "cf-sub", undefined, async param => {});
+      globalThis.expect(again).toBe(0);
+    });
+    globalThis.test("concurrent appends across aggregates: every event drained exactly once", async () => {
+      let match = EventLogStorage_Postgres$ReventlessPostgres.makeStorage(pool, "cf-concurrent", opts, undefined);
+      let ops = match[1];
+      let promises = Stdlib_Array.make(20, 0).map((param, i) => ops.append(0, "agg-" + i.toString(), [evJson("m" + i.toString(), i)]));
+      await Promise.all(promises);
+      let collected = {
+        contents: []
+      };
+      await EventLogChangeFeed$ReventlessPostgres.drain(pool, "cf-concurrent", "cf-conc-sub", undefined, async evs => {
+        collected.contents = collected.contents.concat(evs);
+      });
+      globalThis.expect(collected.contents.length).toBe(20);
+      let distinctCursors = new Set(collected.contents.map(e => e.cursor));
+      globalThis.expect(distinctCursors.size).toBe(20);
+    });
+  });
 } else {
   globalThis.test("Postgres integration (skipped — set PG_URL to run)", () => {
     globalThis.expect(true).toBe(true);