@reventlessdev/reventless-postgres 3.0.0-alpha.1 → 3.0.0-alpha.100

package/lib/ocaml/DcbEventLogStorage_Postgres_Ops.res

@@ -0,0 +1,350 @@
+// Runtime-pure DCB event-log operations for the Postgres backend.
+//
+// Holds everything the DEPLOYED Lambda runtime needs and imports NO
+// `@pulumi/pulumi` (deploy-time only). The deployed DCB command entry point pulls
+// these ops in unconditionally at cold start, so a deploy-time dependency here
+// would leak into the Lambda import graph and fail resolution on real Lambda. The
+// thin `DcbEventLogStorage_Postgres` wrapper `include`s this and adds the
+// deploy-time `Pulumi.Output`-shaped adapter for local/deploy consumers.
+// See docs/plans/deployed-lambda-esm-self-containment.md (Rung-3 finding).
+
+open ReventlessCore
+open Reventless
+
+// Lock strategy for concurrent appends (B4). Advisory (default) is safe under
+// PgBouncer transaction mode; RowLocks avoids advisory-lock connection pinning
+// on RDS Proxy at the cost of a companion `dcb_scope` table.
+type lockStrategy = [#AdvisoryLocks | #RowLocks]
+let strategyParam = s =>
+  switch s {
+  | #AdvisoryLocks => "advisory"
+  | #RowLocks => "rows"
+  }
+
+// Injected by reventless-local to feed the projection-checkpoint low-watermark
+// (ProjectionPending, Dcb axis). Standalone/deploy use passes nothing.
+type onAppended = array<(string, int)> => unit
+let noTracking: onAppended = _ => ()
+
+// SELECT list shared by read and readStream. `data`/`meta` are jsonb (returned
+// already parsed); `tags` is text[] (a JS array); the cursor and recorded_at are
+// computed in SQL so the row maps straight to a rawSequencedEvent.
+let selectColumns = "
+  lpad(transaction_id::text, 20, '0') || ':' || lpad(position::text, 20, '0') AS cursor,
+  event_type,
+  tags,
+  data,
+  meta,
+  to_char(recorded_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"') AS recorded_at
+"
+
+// Decode a 'key=value' tag string (split on the first '=') back to a DcbTag.tag.
+let decodeTag = (s: string): DcbTag.tag =>
+  switch s->String.indexOf("=") {
+  | -1 => {key: s, value: ""}
+  | i => {
+      key: s->String.slice(~start=0, ~end=i),
+      value: s->String.slice(~start=i + 1, ~end=s->String.length),
+    }
+  }
+
+let decodeTags = (v: option<JSON.t>): array<DcbTag.tag> =>
+  switch v {
+  | Some(JSON.Array(arr)) =>
+    arr->Array.filterMap(x =>
+      switch x {
+      | JSON.String(s) => Some(decodeTag(s))
+      | _ => None
+      }
+    )
+  | _ => []
+  }
+
+let rowToEvent = (row: dict<JSON.t>): DcbEventLog_Adapter.rawSequencedEvent => {
+  let str = key =>
+    switch row->Dict.get(key) {
+    | Some(JSON.String(s)) => s
+    | _ => ""
+    }
+  let meta = switch row->Dict.get("meta") {
+  | Some(metaJson) => metaJson->S.parseJsonOrThrow(Reventless.Message.metaSchema)
+  | None => JsError.throwWithMessage("missing meta column in dcb_event row")
+  }
+  {
+    DcbEventLog_Adapter.position: str("cursor"),
+    eventType: str("event_type"),
+    data: row->Dict.get("data")->Option.getOr(JSON.Encode.null),
+    tags: decodeTags(row->Dict.get("tags")),
+    meta,
+    recordedAt: str("recorded_at"),
+  }
+}
+
+// A tiny positional-parameter builder. Postgres uses `$1, $2, …` (not `?`), so we
+// accumulate params and mint tokens as we build the WHERE. A token may be reused
+// (same `$N`) to reference one param twice — used for the `after` cursor columns.
+type paramBuilder = {mutable params: array<JSON.t>}
+let mkBuilder = () => {params: []}
+let param = (b: paramBuilder, v: JSON.t): string => {
+  b.params->Array.push(v)
+  "$" ++ Int.toString(b.params->Array.length)
+}
+
+// Build the read WHERE + params for a query. `applyFence` gates the xmin read
+// barrier (true for consumer reads, so cursors are stable). `after` is the
+// decoded (tx, pos) cursor pair as numeric strings.
+let buildReadWhere = (
+  b: paramBuilder,
+  ~name: string,
+  ~query: DcbTag.query,
+  ~after: option<(string, string)>,
+  ~applyFence: bool,
+): string => {
+  let parts = [`log_name = ${b->param(JSON.Encode.string(name))}`]
+  if applyFence {
+    parts->Array.push("transaction_id < pg_snapshot_xmin(pg_current_snapshot())")
+  }
+  switch after {
+  | Some((tx, pos)) =>
+    let txP = b->param(JSON.Encode.string(tx))
+    let posP = b->param(JSON.Encode.string(pos))
+    parts->Array.push(
+      `(transaction_id > ${txP}::xid8 OR (transaction_id = ${txP}::xid8 AND position > ${posP}::bigint))`,
+    )
+  | None => ()
+  }
+
+  if query->Array.length > 0 {
+    let clauseSqls = query->Array.map(clause => {
+      let clauseParts = []
+      switch clause.eventTypes {
+      | Some(types) if types->Array.length > 0 =>
+        let ph = types->Array.map(t => b->param(JSON.Encode.string(t)))->Array.join(",")
+        clauseParts->Array.push(`event_type IN (${ph})`)
+      | _ => ()
+      }
+      switch clause.tags {
+      | Some(tags) if tags->Array.length > 0 =>
+        let arr =
+          tags
+          ->Array.map(t => b->param(JSON.Encode.string(t.key ++ "=" ++ t.value)))
+          ->Array.join(",")
+        clauseParts->Array.push(`tags @> ARRAY[${arr}]::text[]`)
+      | _ => ()
+      }
+      clauseParts->Array.length == 0 ? "TRUE" : "(" ++ clauseParts->Array.join(" AND ") ++ ")"
+    })
+    parts->Array.push("(" ++ clauseSqls->Array.join(" OR ") ++ ")")
+  }
+
+  parts->Array.join(" AND ")
+}
+
+// Strip the zero-padding a cursor carries for string-sortability, leaving a
+// plain decimal both `::xid8` and `::bigint` parse unambiguously ("0" if empty).
+let stripZeros = (s: string): string => {
+  let t = s->String.replaceRegExp(%re("/^0+/"), "")
+  t == "" ? "0" : t
+}
+
+// Split a '<xid8>:<position>' cursor into its two numeric-string columns.
+let decodeCursor = (pos: DcbTag.sequencePosition): option<(string, string)> =>
+  switch pos->String.split(":") {
+  | [tx, p] => Some((stripZeros(tx), stripZeros(p)))
+  | _ => None
+  }
+
+// Rows-per-page for readStream keyset pagination.
+let pageSize = 500
+
+let isConflict = (exn: exn): bool => {
+  let msg = exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("")
+  msg->String.includes("dcb_conflict") ||
+  msg->String.includes("duplicate key") ||
+  msg->String.includes("23505")
+}
+
+let countAll = async (pool: PgDriver.pool): int =>
+  switch await pool->PgDriver.queryOne("SELECT COUNT(*)::bigint AS c FROM dcb_event", []) {
+  | Some(row) =>
+    switch row->Dict.get("c") {
+    | Some(JSON.String(s)) => s->Int.fromString->Option.getOr(0)
+    | Some(JSON.Number(n)) => Float.toInt(n)
+    | _ => 0
+    }
+  | None => 0
+  }
+
+// Builds the DCB runtime op set bound to `pool`. Returns `(name, ops)`; the
+// deploy-time `DcbEventLog_Adapter` wrapper lives in `DcbEventLogStorage_Postgres`.
+let makeOps = (
+  ~pool: PgDriver.pool,
+  ~name: string,
+  ~indexes as _,
+  ~partitionTag as _,
+  ~crossPartitionTagKeys as _=?,
+  ~opts as _,
+  ~lockStrategy: lockStrategy=#AdvisoryLocks,
+  ~onAppended=noTracking,
+): (string, DcbEventLog_Adapter.operations) => {
+  // Fenced head cursor: the highest (transaction_id, position) visible under the
+  // xmin barrier. Stable across reads, so it is a safe consistency marker.
+  let currentHead = async (): option<string> =>
+    switch await pool->PgDriver.queryOne(
+      "SELECT lpad(transaction_id::text, 20, '0') || ':' || lpad(position::text, 20, '0') AS head
+         FROM dcb_event
+        WHERE log_name = $1 AND transaction_id < pg_snapshot_xmin(pg_current_snapshot())
+        ORDER BY transaction_id DESC, position DESC
+        LIMIT 1",
+      [JSON.Encode.string(name)],
+    ) {
+    | Some(row) =>
+      switch row->Dict.get("head") {
+      | Some(JSON.String(s)) => Some(s)
+      | _ => None
+      }
+    | None => None
+    }
+
+  let runQuery = async (~query, ~after): array<DcbEventLog_Adapter.rawSequencedEvent> => {
+    let b = mkBuilder()
+    let where = buildReadWhere(b, ~name, ~query, ~after, ~applyFence=true)
+    let sql = `SELECT ${selectColumns} FROM dcb_event WHERE ${where} ORDER BY transaction_id ASC, position ASC`
+    (await pool->PgDriver.query(sql, b.params))->Array.map(rowToEvent)
+  }
+
+  let read = async (~query: DcbTag.query, ~after=?): DcbEventLog_Adapter.rawReadResult => {
+    let afterCols = after->Option.flatMap(decodeCursor)
+    let events = await runQuery(~query, ~after=afterCols)
+    switch await currentHead() {
+    | Some(head) => {DcbEventLog_Adapter.events, headPosition: head}
+    | None => ({DcbEventLog_Adapter.events: events}: DcbEventLog_Adapter.rawReadResult)
+    }
+  }
+
+  let append = async (
+    newEvents: array<DcbEventLog_Adapter.rawStoredEvent>,
+    ~condition=?,
+  ): result<DcbTag.sequencePosition, ReventlessInfra.DcbEventLog.appendError> => {
+    let eventsJson = JSON.Array(
+      newEvents->Array.map(ev =>
+        JSON.Object(
+          Dict.fromArray([
+            ("event_type", JSON.Encode.string(ev.eventType)),
+            (
+              "tags",
+              JSON.Array(ev.tags->Array.map(t => JSON.Encode.string(t.key ++ "=" ++ t.value))),
+            ),
+            ("data", ev.data),
+            ("meta", ev.meta->S.reverseConvertToJsonOrThrow(Reventless.Message.metaSchema)),
+          ]),
+        )
+      ),
+    )
+    let conditionParam = switch condition {
+    | Some(cond: DcbTag.appendCondition) =>
+      let queryJson = JSON.Array(
+        cond.query->Array.map(clause =>
+          JSON.Object(
+            Dict.fromArray([
+              (
+                "eventTypes",
+                switch clause.eventTypes {
+                | Some(types) => JSON.Array(types->Array.map(JSON.Encode.string))
+                | None => JSON.Encode.null
+                },
+              ),
+              (
+                "tags",
+                switch clause.tags {
+                | Some(tags) =>
+                  JSON.Array(tags->Array.map(t => JSON.Encode.string(t.key ++ "=" ++ t.value)))
+                | None => JSON.Encode.null
+                },
+              ),
+            ]),
+          )
+        ),
+      )
+      let fields = [("query", queryJson)]
+      switch cond.after {
+      | Some(a) => fields->Array.push(("after", JSON.Encode.string(a)))
+      | None => ()
+      }
+      JSON.Encode.string(JSON.stringify(JSON.Object(Dict.fromArray(fields))))
+    | None => JSON.Encode.null
+    }
+
+    try {
+      let rows = await pool->PgDriver.query(
+        "SELECT dcb_append($1, $2::jsonb, $3::jsonb, $4) AS pos",
+        [
+          JSON.Encode.string(name),
+          JSON.Encode.string(JSON.stringify(eventsJson)),
+          conditionParam,
+          JSON.Encode.string(strategyParam(lockStrategy)),
+        ],
+      )
+      switch rows->Array.get(0)->Option.flatMap(r => r->Dict.get("pos")) {
+      | Some(JSON.String(cursor)) =>
+        // Feed the projection checkpoint (Dcb axis): (msgId, numeric position).
+        onAppended(
+          newEvents->Array.filterMap(ev =>
+            switch decodeCursor(cursor) {
+            | Some((_, pos)) =>
+              pos->Int.fromString->Option.map(p => (ev.meta.Reventless.Message.msgId, p))
+            | None => None
+            }
+          ),
+        )
+        Ok(cursor)
+      | _ => Error(ReventlessInfra.DcbEventLog.StorageFailure("dcb_append returned no position"))
+      }
+    } catch {
+    | exn =>
+      isConflict(exn)
+        ? Error(ReventlessInfra.DcbEventLog.Conflict)
+        : Error(
+            ReventlessInfra.DcbEventLog.StorageFailure(
+              exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("storage error"),
+            ),
+          )
+    }
+  }
+
+  // ~strongConsistency is a genuine no-op here: every read applies the xmin
+  // barrier, so it is always safely consistent (at the cost of a few ms lag).
+  //
+  // Keyset pagination on (transaction_id, position): one ordered SQL source, no
+  // per-clause fan-out / k-way merge / dedup (the DynamoDB runtime's layers have
+  // no Postgres counterpart). Each page reads strictly after the previous page's
+  // last row; a short page ends the stream.
+  let readStream = (~query: DcbTag.query, ~after=?, ~strongConsistency as _=?) =>
+    Stream.paginateEffect(after->Option.flatMap(decodeCursor), cursor =>
+      Effect.promise(() => {
+        let b = mkBuilder()
+        let where = buildReadWhere(b, ~name, ~query, ~after=cursor, ~applyFence=true)
+        let sql = `SELECT ${selectColumns}, transaction_id::text AS tx_raw, position::text AS pos_raw FROM dcb_event WHERE ${where} ORDER BY transaction_id ASC, position ASC LIMIT ${Int.toString(pageSize)}`
+        pool->PgDriver.query(sql, b.params)
+      })->Effect.map(rows => {
+        let events = rows->Array.map(rowToEvent)
+        // Continue only on a full page; the next `after` is this page's last row.
+        let next = if rows->Array.length < pageSize {
+          None
+        } else {
+          switch rows->Array.get(rows->Array.length - 1) {
+          | Some(lastRow) =>
+            switch (lastRow->Dict.get("tx_raw"), lastRow->Dict.get("pos_raw")) {
+            | (Some(JSON.String(tx)), Some(JSON.String(p))) => Some(Some((tx, p)))
+            | _ => None
+            }
+          | None => None
+          }
+        }
+        (events, next)
+      })
+    )
+
+  let ops: DcbEventLog_Adapter.operations = {read, append, readStream}
+  (name, ops)
+}

package/lib/ocaml/EventLogChangeFeed.res

@@ -0,0 +1,187 @@
+// Classic `event_log` change feed (B2.5) — the aggregate-path analogue of
+// `PgChangeFeed` (which serves the DCB `dcb_event` log).
+//
+// Like the DCB feed this is NOT logical decoding: classic events are already
+// durably queryable in-table, so the feed is an xmin-fenced read over `event_log`
+// plus an `event_log_subscription` checkpoint. A statement-level AFTER INSERT
+// trigger (`event_log_notify`, see PgSchema) issues `pg_notify('evlog_<log>')`,
+// turning polling into near-real-time wakeup; a low-frequency fallback tick in the
+// consumer covers missed notifications.
+//
+// Cursor semantics mirror the DCB feed exactly: the opaque `<xid8>:<global_seq>`
+// string (zero-padded so string order == numeric order). Reads apply the xmin
+// barrier (`transaction_id < pg_snapshot_xmin(pg_current_snapshot())`), so a
+// returned cursor is stable forever — an event committed late (lower global_seq,
+// later commit) is never skipped by a checkpointing reader. `global_seq` alone,
+// being an IDENTITY assigned at INSERT, could NOT provide that guarantee.
+
+// Reuse the shared `<a>:<b>` cursor codec from the DCB storage — identical
+// format. Point at the runtime-pure `_Ops` module (no `@pulumi/pulumi`): this
+// feed is drained by the deployed change-feed relay Lambda, whose cold-start
+// graph must stay Pulumi-free.
+module Dcb = DcbEventLogStorage_Postgres_Ops
+
+// One classic event as the feed surfaces it. `payload` is the stored event JSON
+// verbatim; a bus/topic relay transforms it into its target shape (e.g. the AWS
+// EventCollector `{id, meta, event}` body) — that transform is not this package's
+// concern.
+type classicEvent = {
+  cursor: string,
+  aggregateId: string,
+  seqNr: int,
+  payload: JSON.t,
+  msgId: option<string>,
+}
+
+// One page of the feed. `events` are in (transaction_id, global_seq) order;
+// `cursor` is the checkpoint to persist and to pass as the next `after` (None when
+// the page was empty — nothing new).
+type batch = {
+  events: array<classicEvent>,
+  cursor: option<string>,
+}
+
+let selectColumns = "
+  lpad(transaction_id::text, 20, '0') || ':' || lpad(global_seq::text, 20, '0') AS cursor,
+  aggregate_id,
+  seq_nr,
+  payload,
+  msg_id
+"
+
+let coerceInt = (v: JSON.t): int =>
+  switch v {
+  | JSON.Number(n) => Float.toInt(n)
+  | JSON.String(s) => s->Int.fromString->Option.getOr(0)
+  | _ => 0
+  }
+
+let rowToEvent = (row: dict<JSON.t>): classicEvent => {
+  let str = key =>
+    switch row->Dict.get(key) {
+    | Some(JSON.String(s)) => s
+    | _ => ""
+    }
+  {
+    cursor: str("cursor"),
+    aggregateId: str("aggregate_id"),
+    seqNr: row->Dict.get("seq_nr")->Option.map(coerceInt)->Option.getOr(0),
+    payload: row->Dict.get("payload")->Option.getOr(JSON.Encode.null),
+    msgId: switch row->Dict.get("msg_id") {
+    | Some(JSON.String(s)) => Some(s)
+    | _ => None
+    },
+  }
+}
+
+// Read up to `limit` events after `after` (the xmin-fenced feed query — whole log).
+// Returns the events and the cursor of the last one.
+let readBatch = async (
+  pool: PgDriver.pool,
+  ~logName: string,
+  ~after: option<string>=?,
+  ~limit: int=500,
+): batch => {
+  let params = [JSON.Encode.string(logName)]
+  let where = ref("log_name = $1 AND transaction_id < pg_snapshot_xmin(pg_current_snapshot())")
+  switch after->Option.flatMap(Dcb.decodeCursor) {
+  | Some((tx, pos)) =>
+    // $2 = tx (referenced twice), $3 = global_seq.
+    params->Array.push(JSON.Encode.string(tx))
+    params->Array.push(JSON.Encode.string(pos))
+    where :=
+      where.contents ++
+      " AND (transaction_id > $2::xid8 OR (transaction_id = $2::xid8 AND global_seq > $3::bigint))"
+  | None => ()
+  }
+  let sql = `SELECT ${selectColumns} FROM event_log WHERE ${where.contents} ORDER BY transaction_id ASC, global_seq ASC LIMIT ${Int.toString(limit)}`
+  let rows = await pool->PgDriver.query(sql, params)
+  let events = rows->Array.map(rowToEvent)
+  let cursor = switch events->Array.get(events->Array.length - 1) {
+  | Some(last) => Some(last.cursor)
+  | None => None
+  }
+  {events, cursor}
+}
+
+// --- Checkpoint contract (event_log_subscription) ---
+
+// The persisted checkpoint for `subscriber`, encoded as a cursor, or None if the
+// subscriber has never committed (→ replay from the beginning).
+let loadCheckpoint = async (pool: PgDriver.pool, ~subscriber: string): option<string> =>
+  switch await pool->PgDriver.queryOne(
+    "SELECT lpad(last_tx::text, 20, '0') || ':' || lpad(last_global_seq::text, 20, '0') AS cursor
+       FROM event_log_subscription WHERE subscriber = $1 AND last_global_seq > 0",
+    [JSON.Encode.string(subscriber)],
+  ) {
+  | Some(row) =>
+    switch row->Dict.get("cursor") {
+    | Some(JSON.String(s)) => Some(s)
+    | _ => None
+    }
+  | None => None
+  }
+
+// Persist the checkpoint. Idempotent upsert; decodes the cursor to its two columns
+// so a later `loadCheckpoint` round-trips exactly.
+let saveCheckpoint = async (pool: PgDriver.pool, ~subscriber: string, ~cursor: string): unit =>
+  switch Dcb.decodeCursor(cursor) {
+  | Some((tx, pos)) =>
+    let _ = await pool->PgDriver.query(
+      "INSERT INTO event_log_subscription(subscriber, last_tx, last_global_seq)
+       VALUES ($1, $2::xid8, $3::bigint)
+       ON CONFLICT (subscriber)
+       DO UPDATE SET last_tx = EXCLUDED.last_tx, last_global_seq = EXCLUDED.last_global_seq",
+      [JSON.Encode.string(subscriber), JSON.Encode.string(tx), JSON.Encode.string(pos)],
+    )
+  | None => ()
+  }
+
+// --- Wakeup ---
+
+// LISTEN on `evlog_<logName>`; `onWake` fires on every notification. Returns the
+// dedicated client (caller owns it — release via unlisten). Pair with a
+// low-frequency fallback tick so a missed NOTIFY only adds latency, never a stall.
+let listen = (pool: PgDriver.pool, ~logName: string, ~onWake: unit => unit): promise<
+  PgDriver.client,
+> => pool->PgDriver.listen(~channel="evlog_" ++ logName, ~onNotify=_ => onWake())
+
+let unlisten = (client: PgDriver.client, ~logName: string): promise<unit> =>
+  client->PgDriver.unlisten(~channel="evlog_" ++ logName)
+
+// --- Reference consumer driver ---
+
+// Drain the feed for `subscriber` from its checkpoint: repeatedly readBatch →
+// `handle` the events → saveCheckpoint, until a short page. `handle` must be
+// idempotent (a crash between handle and checkpoint replays the last batch).
+// Returns the number of events processed.
+let drain = async (
+  pool: PgDriver.pool,
+  ~logName: string,
+  ~subscriber: string,
+  ~limit: int=500,
+  ~handle: array<classicEvent> => promise<unit>,
+): int => {
+  let processed = ref(0)
+  let cursor = ref(await loadCheckpoint(pool, ~subscriber))
+  let continue = ref(true)
+  while continue.contents {
+    let {events, cursor: newCursor} = await readBatch(pool, ~logName, ~after=?cursor.contents, ~limit)
+    if events->Array.length == 0 {
+      continue := false
+    } else {
+      await handle(events)
+      switch newCursor {
+      | Some(c) =>
+        await saveCheckpoint(pool, ~subscriber, ~cursor=c)
+        cursor := Some(c)
+      | None => ()
+      }
+      processed := processed.contents + events->Array.length
+      if events->Array.length < limit {
+        continue := false
+      }
+    }
+  }
+  processed.contents
+}