@reventlessdev/reventless-postgres 3.0.0-alpha.1 → 3.0.0-alpha.11

package/lib/ocaml/EventLogStorage_Postgres.res

@@ -1,176 +1,18 @@
-// Postgres-backed classic (OCC) event log storage. Direct port of
-// EventLogStorage_Sqlite; the only structural difference is that every operation
-// is async (pg has no synchronous client) and `payload jsonb` returns already
-// parsed, so there is no per-row JSON.parse.
+// Deploy/local wrapper over the runtime-pure EventLogStorage_Postgres_Ops.
 //
-// Schema (PgSchema.res): event_log(log_name, aggregate_id, seq_nr, payload,
-// msg_id, global_seq IDENTITY, PRIMARY KEY(log_name, aggregate_id, seq_nr)).
-//
-// Optimistic concurrency: the append is a single guarded statement —
-//   INSERT ... SELECT ... WHERE $seq = (SELECT COALESCE(MAX(seq_nr),-1)+1 ...)
-// which is atomic and gap-proof. A wrong seq (gap or already-consumed) fails the
-// WHERE → 0 rows → Conflict; two writers racing the same seq collide on the PK →
-// one commits, the loser's unique_violation → Conflict.
+// The runtime ops (and helpers like `countAll`) live in
+// EventLogStorage_Postgres_Ops, which imports NO `@pulumi/pulumi` so the deployed
+// Lambda import graph stays clean (the aggregate entry point pulls the ops in
+// unconditionally at cold start — a deploy-time dep here would fail resolution on
+// real Lambda). This module `include`s that pure module and adds only the
+// deploy-time `EventLog_Adapter` shape (`operations` wrapped in `Pulumi.Output`),
+// keeping the historic `makeStorage` 3-tuple `(name, ops, adapter)` API unchanged.
+// See docs/plans/deployed-lambda-esm-self-containment.md (Rung-3 finding).
 
 open ReventlessCore
+include EventLogStorage_Postgres_Ops
 
-// Injected by reventless-local to feed the projection-checkpoint low-watermark
-// (ProjectionPending, aggregate axis). Standalone/deploy use passes nothing.
-type onAppended = array<(string, int)> => unit
-let noTracking: onAppended = _ => ()
-
-let decodePayload = (row: dict<JSON.t>): JSON.t =>
-  row->Dict.get("payload")->Option.getOr(JSON.Encode.null)
-
-let coerceInt = (v: JSON.t): int =>
-  switch v {
-  | JSON.Number(n) => Float.toInt(n)
-  | JSON.String(s) => s->Int.fromString->Option.getOr(0)
-  | _ => 0
-  }
-
-// Total events across all aggregates. Seeds the event-tap counter at startup so
-// timeline numbering continues across restarts.
-let countAll = async (pool: PgDriver.pool): int =>
-  switch await pool->PgDriver.queryOne("SELECT COUNT(*)::bigint AS c FROM event_log", []) {
-  | Some(row) => row->Dict.get("c")->Option.map(coerceInt)->Option.getOr(0)
-  | None => 0
-  }
-
-let isUniqueViolation = (exn: exn): bool => {
-  let msg = exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("")
-  // pg surfaces the SQLSTATE on `.code`; the message also carries the constraint.
-  msg->String.includes("duplicate key") ||
-  msg->String.includes("unique") ||
-  msg->String.includes("23505")
-}
-
-let makeStorage = (~pool: PgDriver.pool, ~name: string, ~opts as _, ~onAppended=noTracking) => {
-  let appendTracked = async (~track, seqNr: int, id: string, jsons: array<JSON.t>) => {
-    // Empty append is a legitimate no-op (at-least-once retries, idempotent
-    // commands), NOT a conflict — return before the guarded insert whose 0-row
-    // result would otherwise be read as a conflict.
-    if jsons->Array.length == 0 {
-      Ok()
-    } else {
-      try {
-        let rows = await pool->PgDriver.query(
-          "INSERT INTO event_log(log_name, aggregate_id, seq_nr, payload, msg_id)
-           SELECT $1, $2, $3::bigint + (ord - 1), value, value->>'msgId'
-             FROM jsonb_array_elements($4::jsonb) WITH ORDINALITY AS t(value, ord)
-            WHERE $3::bigint = (SELECT COALESCE(MAX(seq_nr), -1) + 1
-                                  FROM event_log
-                                 WHERE log_name = $1 AND aggregate_id = $2)
-           RETURNING global_seq, msg_id",
-          [
-            JSON.Encode.string(name),
-            JSON.Encode.string(id),
-            JSON.Encode.int(seqNr),
-            JSON.Encode.string(JSON.stringify(JSON.Array(jsons))),
-          ],
-        )
-        if rows->Array.length == 0 {
-          // WHERE guard rejected the seq — a genuine OCC conflict (gap or stale).
-          Error(EventLog.Conflict)
-        } else {
-          if track {
-            onAppended(
-              rows->Array.filterMap(row =>
-                switch (row->Dict.get("msg_id"), row->Dict.get("global_seq")) {
-                | (Some(JSON.String(msgId)), Some(gseq)) => Some((msgId, coerceInt(gseq)))
-                | _ => None
-                }
-              ),
-            )
-          }
-          Ok()
-        }
-      } catch {
-      | exn =>
-        isUniqueViolation(exn)
-          ? Error(EventLog.Conflict)
-          : Error(
-              EventLog.StorageFailure(
-                exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("storage error"),
-              ),
-            )
-      }
-    }
-  }
-
-  let append: EventLog.append<string, JSON.t> = (seqNr, id, jsons) =>
-    appendTracked(~track=true, seqNr, id, jsons)
-
-  let replayArray = async (id: string, ~fromSeq=0): array<JSON.t> =>
-    (await pool->PgDriver.query(
-      "SELECT payload FROM event_log
-        WHERE log_name = $1 AND aggregate_id = $2 AND seq_nr >= $3::bigint
-        ORDER BY seq_nr ASC",
-      [JSON.Encode.string(name), JSON.Encode.string(id), JSON.Encode.int(fromSeq)],
-    ))->Array.map(decodePayload)
-
-  let replay: EventLog.replay<string, JSON.t> = id => replayArray(id)
-
-  let replayStream = (id, ~fromSeq=?) =>
-    Effect.promise(() => replayArray(id, ~fromSeq?))
-    ->Stream.fromEffect
-    ->Stream.flatMap(arr => Stream.fromIterable(arr))
-
-  let latestSnapshot: EventLog.latestSnapshot<string> = async id =>
-    switch await pool->PgDriver.queryOne(
-      "SELECT seq_nr, state, schema_hash FROM snapshot WHERE log_name = $1 AND aggregate_id = $2",
-      [JSON.Encode.string(name), JSON.Encode.string(id)],
-    ) {
-    | Some(row) =>
-      switch (row->Dict.get("seq_nr"), row->Dict.get("state"), row->Dict.get("schema_hash")) {
-      | (Some(seq), Some(state), Some(JSON.String(schemaHash))) =>
-        Ok(Some({EventLog.seqNr: coerceInt(seq), state, schemaHash}))
-      | _ => Error("snapshot row has unexpected shape")
-      }
-    | None => Ok(None)
-    }
-
-  let writeSnapshot: EventLog.writeSnapshot<string> = async (id, snap) =>
-    try {
-      let _ = await pool->PgDriver.query(
-        "INSERT INTO snapshot(log_name, aggregate_id, seq_nr, state, schema_hash)
-         VALUES ($1, $2, $3::bigint, $4::jsonb, $5)
-         ON CONFLICT (log_name, aggregate_id)
-         DO UPDATE SET seq_nr = EXCLUDED.seq_nr, state = EXCLUDED.state, schema_hash = EXCLUDED.schema_hash",
-        [
-          JSON.Encode.string(name),
-          JSON.Encode.string(id),
-          JSON.Encode.int(snap.EventLog.seqNr),
-          JSON.Encode.string(JSON.stringify(snap.state)),
-          JSON.Encode.string(snap.schemaHash),
-        ],
-      )
-      Ok()
-    } catch {
-    | exn =>
-      Error(exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("snapshot write error"))
-    }
-
-  let appendStream: EventLog.appendStream<string, JSON.t> = (startingSeqNr, id, stream) =>
-    stream
-    ->Stream.runCollect
-    ->Effect.flatMap(jsons =>
-      Effect.promise(() => appendTracked(~track=false, startingSeqNr, id, jsons))->Effect.flatMap(result =>
-        switch result {
-        | Ok() => Effect.succeed()
-        | Error(EventLog.Conflict) => Effect.fail("conflict")
-        | Error(StorageFailure(msg)) => Effect.fail(msg)
-        }
-      )
-    )
-
-  let ops: EventLog_Adapter.operations = {
-    append,
-    replay,
-    replayStream,
-    appendStream,
-    latestSnapshot,
-    writeSnapshot,
-  }
+let makeStorage = (~pool: PgDriver.pool, ~name: string, ~opts, ~onAppended=noTracking) => {
+  let (name, ops) = makeOps(~pool, ~name, ~opts, ~onAppended)
   (name, ops, {EventLog_Adapter.resources: [], operations: Pulumi.Output.make(ops)})
 }

package/lib/ocaml/EventLogStorage_Postgres_Ops.res

@@ -0,0 +1,180 @@
+// Runtime-pure classic (OCC) event-log operations for the Postgres backend.
+//
+// This module holds everything the DEPLOYED Lambda runtime needs and imports NO
+// `@pulumi/pulumi` — deploy-time-only. The deployed aggregate entry point pulls
+// these ops in unconditionally at cold start, so any deploy-time dependency here
+// leaks into the Lambda import graph and fails resolution (`@pulumi/pulumi` exists
+// in no Lambda — not the layer, not `/var/runtime`). The thin
+// `EventLogStorage_Postgres` wrapper adds the deploy-time `Pulumi.Output`-shaped
+// adapter on top for local/deploy consumers; the runtime imports `makeOps` here.
+// See docs/plans/deployed-lambda-esm-self-containment.md (Rung-3 finding).
+
+open ReventlessCore
+
+// Injected by reventless-local to feed the projection-checkpoint low-watermark
+// (ProjectionPending, aggregate axis). Standalone/deploy use passes nothing.
+type onAppended = array<(string, int)> => unit
+let noTracking: onAppended = _ => ()
+
+let decodePayload = (row: dict<JSON.t>): JSON.t =>
+  row->Dict.get("payload")->Option.getOr(JSON.Encode.null)
+
+let coerceInt = (v: JSON.t): int =>
+  switch v {
+  | JSON.Number(n) => Float.toInt(n)
+  | JSON.String(s) => s->Int.fromString->Option.getOr(0)
+  | _ => 0
+  }
+
+// Total events across all aggregates. Seeds the event-tap counter at startup so
+// timeline numbering continues across restarts.
+let countAll = async (pool: PgDriver.pool): int =>
+  switch await pool->PgDriver.queryOne("SELECT COUNT(*)::bigint AS c FROM event_log", []) {
+  | Some(row) => row->Dict.get("c")->Option.map(coerceInt)->Option.getOr(0)
+  | None => 0
+  }
+
+let isUniqueViolation = (exn: exn): bool => {
+  let msg = exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("")
+  // pg surfaces the SQLSTATE on `.code`; the message also carries the constraint.
+  msg->String.includes("duplicate key") ||
+  msg->String.includes("unique") ||
+  msg->String.includes("23505")
+}
+
+// Builds the classic runtime op set bound to `pool`. Returns `(name, ops)`; the
+// deploy-time `EventLog_Adapter` wrapper lives in `EventLogStorage_Postgres`.
+let makeOps = (
+  ~pool: PgDriver.pool,
+  ~name: string,
+  ~opts as _,
+  ~onAppended=noTracking,
+): (string, EventLog_Adapter.operations) => {
+  let appendTracked = async (~track, seqNr: int, id: string, jsons: array<JSON.t>) => {
+    // Empty append is a legitimate no-op (at-least-once retries, idempotent
+    // commands), NOT a conflict — return before the guarded insert whose 0-row
+    // result would otherwise be read as a conflict.
+    if jsons->Array.length == 0 {
+      Ok()
+    } else {
+      try {
+        let rows = await pool->PgDriver.query(
+          "INSERT INTO event_log(log_name, aggregate_id, seq_nr, payload, msg_id)
+           SELECT $1, $2, $3::bigint + (ord - 1), value, value->>'msgId'
+             FROM jsonb_array_elements($4::jsonb) WITH ORDINALITY AS t(value, ord)
+            WHERE $3::bigint = (SELECT COALESCE(MAX(seq_nr), -1) + 1
+                                  FROM event_log
+                                 WHERE log_name = $1 AND aggregate_id = $2)
+           RETURNING global_seq, msg_id",
+          [
+            JSON.Encode.string(name),
+            JSON.Encode.string(id),
+            JSON.Encode.int(seqNr),
+            JSON.Encode.string(JSON.stringify(JSON.Array(jsons))),
+          ],
+        )
+        if rows->Array.length == 0 {
+          // WHERE guard rejected the seq — a genuine OCC conflict (gap or stale).
+          Error(EventLog.Conflict)
+        } else {
+          if track {
+            onAppended(
+              rows->Array.filterMap(row =>
+                switch (row->Dict.get("msg_id"), row->Dict.get("global_seq")) {
+                | (Some(JSON.String(msgId)), Some(gseq)) => Some((msgId, coerceInt(gseq)))
+                | _ => None
+                }
+              ),
+            )
+          }
+          Ok()
+        }
+      } catch {
+      | exn =>
+        isUniqueViolation(exn)
+          ? Error(EventLog.Conflict)
+          : Error(
+              EventLog.StorageFailure(
+                exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("storage error"),
+              ),
+            )
+      }
+    }
+  }
+
+  let append: EventLog.append<string, JSON.t> = (seqNr, id, jsons) =>
+    appendTracked(~track=true, seqNr, id, jsons)
+
+  let replayArray = async (id: string, ~fromSeq=0): array<JSON.t> =>
+    (await pool->PgDriver.query(
+      "SELECT payload FROM event_log
+        WHERE log_name = $1 AND aggregate_id = $2 AND seq_nr >= $3::bigint
+        ORDER BY seq_nr ASC",
+      [JSON.Encode.string(name), JSON.Encode.string(id), JSON.Encode.int(fromSeq)],
+    ))->Array.map(decodePayload)
+
+  let replay: EventLog.replay<string, JSON.t> = id => replayArray(id)
+
+  let replayStream = (id, ~fromSeq=?) =>
+    Effect.promise(() => replayArray(id, ~fromSeq?))
+    ->Stream.fromEffect
+    ->Stream.flatMap(arr => Stream.fromIterable(arr))
+
+  let latestSnapshot: EventLog.latestSnapshot<string> = async id =>
+    switch await pool->PgDriver.queryOne(
+      "SELECT seq_nr, state, schema_hash FROM snapshot WHERE log_name = $1 AND aggregate_id = $2",
+      [JSON.Encode.string(name), JSON.Encode.string(id)],
+    ) {
+    | Some(row) =>
+      switch (row->Dict.get("seq_nr"), row->Dict.get("state"), row->Dict.get("schema_hash")) {
+      | (Some(seq), Some(state), Some(JSON.String(schemaHash))) =>
+        Ok(Some({EventLog.seqNr: coerceInt(seq), state, schemaHash}))
+      | _ => Error("snapshot row has unexpected shape")
+      }
+    | None => Ok(None)
+    }
+
+  let writeSnapshot: EventLog.writeSnapshot<string> = async (id, snap) =>
+    try {
+      let _ = await pool->PgDriver.query(
+        "INSERT INTO snapshot(log_name, aggregate_id, seq_nr, state, schema_hash)
+         VALUES ($1, $2, $3::bigint, $4::jsonb, $5)
+         ON CONFLICT (log_name, aggregate_id)
+         DO UPDATE SET seq_nr = EXCLUDED.seq_nr, state = EXCLUDED.state, schema_hash = EXCLUDED.schema_hash",
+        [
+          JSON.Encode.string(name),
+          JSON.Encode.string(id),
+          JSON.Encode.int(snap.EventLog.seqNr),
+          JSON.Encode.string(JSON.stringify(snap.state)),
+          JSON.Encode.string(snap.schemaHash),
+        ],
+      )
+      Ok()
+    } catch {
+    | exn =>
+      Error(exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("snapshot write error"))
+    }
+
+  let appendStream: EventLog.appendStream<string, JSON.t> = (startingSeqNr, id, stream) =>
+    stream
+    ->Stream.runCollect
+    ->Effect.flatMap(jsons =>
+      Effect.promise(() => appendTracked(~track=false, startingSeqNr, id, jsons))->Effect.flatMap(result =>
+        switch result {
+        | Ok() => Effect.succeed()
+        | Error(EventLog.Conflict) => Effect.fail("conflict")
+        | Error(StorageFailure(msg)) => Effect.fail(msg)
+        }
+      )
+    )
+
+  let ops: EventLog_Adapter.operations = {
+    append,
+    replay,
+    replayStream,
+    appendStream,
+    latestSnapshot,
+    writeSnapshot,
+  }
+  (name, ops)
+}

package/lib/ocaml/PgSchema.res

@@ -36,6 +36,19 @@ let tableStatements = [
      schema_hash   text   NOT NULL,
      PRIMARY KEY (log_name, aggregate_id)
    )",
+  // --- Classic change feed (B2.5) ---
+  // Stamp every classic event with its committing xid8 so the feed can apply the
+  // same xmin read barrier the DCB feed uses — global_seq (an IDENTITY assigned at
+  // INSERT, before commit) is unsafe as a feed cursor on its own (a late-committing
+  // lower global_seq would be skipped). Idempotent; the volatile default auto-stamps
+  // every future insert with no change to EventLogStorage_Postgres.append.
+  "ALTER TABLE event_log ADD COLUMN IF NOT EXISTS transaction_id xid8 NOT NULL DEFAULT pg_current_xact_id()",
+  "CREATE INDEX IF NOT EXISTS event_log_tx_gseq ON event_log (log_name, transaction_id, global_seq)",
+  "CREATE TABLE IF NOT EXISTS event_log_subscription (
+     subscriber      text   PRIMARY KEY,
+     last_tx         xid8   NOT NULL DEFAULT '0'::xid8,
+     last_global_seq bigint NOT NULL DEFAULT 0
+   )",
   // --- DCB event log (Phase B) ---
   "CREATE TABLE IF NOT EXISTS dcb_event (
      log_name        text        NOT NULL,
@@ -236,7 +249,28 @@ END;
 $fn$;
 "
 
-let functionStatements = [dcbConditionWhereFn, dcbAppendFn]
+// Classic change-feed wakeup (B2.5). A statement-level AFTER INSERT trigger fires
+// one pg_notify per distinct log_name touched by the statement, so the classic feed
+// gets the same near-real-time wakeup the DCB append's inline pg_notify provides —
+// without touching EventLogStorage_Postgres.append. No-op if nobody is LISTENing; a
+// low-frequency fallback tick in the consumer covers missed notifications.
+let eventLogNotifyFn = "
+CREATE OR REPLACE FUNCTION event_log_notify() RETURNS trigger
+LANGUAGE plpgsql AS $fn$
+BEGIN
+  PERFORM pg_notify('evlog_' || log_name, '')
+    FROM (SELECT DISTINCT log_name FROM new_rows) s;
+  RETURN NULL;
+END;
+$fn$;
+
+CREATE OR REPLACE TRIGGER event_log_notify_trg
+  AFTER INSERT ON event_log
+  REFERENCING NEW TABLE AS new_rows
+  FOR EACH STATEMENT EXECUTE FUNCTION event_log_notify();
+"
+
+let functionStatements = [dcbConditionWhereFn, dcbAppendFn, eventLogNotifyFn]
 
 let ensureSchema = async (pool: PgDriver.pool): unit => {
   for i in 0 to tableStatements->Array.length - 1 {
@@ -251,6 +285,6 @@ let ensureSchema = async (pool: PgDriver.pool): unit => {
 // Mirrors the reventless-local contract that reset wipes data, not structure.
 let truncateAll = async (pool: PgDriver.pool): unit => {
   await pool->PgDriver.exec(
-    "TRUNCATE event_log, snapshot, dcb_event, dcb_scope, dcb_subscription RESTART IDENTITY",
+    "TRUNCATE event_log, snapshot, dcb_event, dcb_scope, dcb_subscription, event_log_subscription RESTART IDENTITY",
   )
 }

package/lib/ocaml/PostgresIntegrationTest.res

@@ -233,4 +233,72 @@ switch processEnv->Dict.get("PG_URL") {
       expect(again)->toBe(0)
     })
   })
+
+  describe("classic change feed (event_log, B2.5)", () => {
+    testPromise("drains classic events in commit order, with fields, then checkpoints", async () => {
+      let (_n, ops, _s) = EventLogStorage_Postgres.makeStorage(~pool, ~name="cf-orders", ~opts)
+      let _ = await ops.append(0, "agg-1", [evJson("m0", 1), evJson("m1", 2)])
+      let _ = await ops.append(0, "agg-2", [evJson("m2", 3)])
+
+      let collected = ref([])
+      let processed = await EventLogChangeFeed.drain(
+        pool,
+        ~logName="cf-orders",
+        ~subscriber="cf-sub",
+        ~handle=async evs => collected := collected.contents->Array.concat(evs),
+      )
+      expect(processed)->toBe(3)
+      expect(collected.contents->Array.length)->toBe(3)
+
+      // Fields round-trip from the stored row.
+      let first = collected.contents->Array.getUnsafe(0)
+      expect(first.aggregateId)->toBe("agg-1")
+      expect(first.seqNr)->toBe(0)
+      expect(first.msgId)->toEqual(Some("m0"))
+
+      // Cursors are strictly increasing in drain order.
+      let cursors = collected.contents->Array.map(e => e.cursor)
+      let increasing = ref(true)
+      for i in 1 to cursors->Array.length - 1 {
+        if cursors->Array.getUnsafe(i) <= cursors->Array.getUnsafe(i - 1) {
+          increasing := false
+        }
+      }
+      expect(increasing.contents)->toBe(true)
+
+      // Second drain from the saved checkpoint sees nothing new.
+      let again = await EventLogChangeFeed.drain(
+        pool,
+        ~logName="cf-orders",
+        ~subscriber="cf-sub",
+        ~handle=async _ => (),
+      )
+      expect(again)->toBe(0)
+    })
+
+    testPromise("concurrent appends across aggregates: every event drained exactly once", async () => {
+      let (_n, ops, _s) = EventLogStorage_Postgres.makeStorage(~pool, ~name="cf-concurrent", ~opts)
+      let n = 20
+      // One event per aggregate, all dispatched before awaiting — genuinely
+      // concurrent on the pool, so global_seq is assigned in interleaved/undefined
+      // commit order. The xmin fence must still drain every event exactly once.
+      let promises =
+        Array.make(~length=n, 0)->Array.mapWithIndex((_, i) =>
+          ops.append(0, "agg-" ++ Int.toString(i), [evJson("m" ++ Int.toString(i), i)])
+        )
+      let _ = await Promise.all(promises)
+
+      let collected = ref([])
+      let _ = await EventLogChangeFeed.drain(
+        pool,
+        ~logName="cf-concurrent",
+        ~subscriber="cf-conc-sub",
+        ~handle=async evs => collected := collected.contents->Array.concat(evs),
+      )
+      // No skips: all N events present. No dupes: N distinct cursors.
+      expect(collected.contents->Array.length)->toBe(n)
+      let distinctCursors = collected.contents->Array.map(e => e.cursor)->Set.fromArray
+      expect(distinctCursors->Set.size)->toBe(n)
+    })
+  })
 }