@reventlessdev/reventless-local 3.0.0-alpha.114 → 3.0.0-alpha.116

package/src/adapter/EventCollector/LocalEventCollectorChannel.res

@@ -85,3 +85,36 @@ module Make = (Bus: LocalBus.T) => {
     []
   }
 }
+
+// Projection-flavoured channel: identical wiring to `Make`, plus registers the
+// resolved handler in the Bus projection catch-up registry so the SQLite
+// backend's startup catch-up (ProjectionCheckpoint) can replay missed stored
+// events into it. Only projection collectors (the ReadModel builders) use this
+// — automation/translation collectors must never receive catch-up events (they
+// would re-run side effects and re-dispatch commands), so they stay on `Make`.
+module MakeProjection = (Bus: LocalBus.T) => {
+  module Base = Make(Bus)
+
+  type callbackEvent = Base.callbackEvent
+  type channelParts = Base.channelParts
+  type runtimeParts = Base.runtimeParts
+
+  let make = Base.make
+
+  let connect: ReventlessCore.EventCollector_Adapter.connect<
+    callbackEvent,
+    'context,
+    channelParts,
+    runtimeParts,
+  > = (~name, ~channelSpecs, ~runtime, ~opts) => {
+    let _reg =
+      runtime.parts.handlerDeferred
+      ->Deferred.await_
+      ->Effect.flatMap(handler =>
+        Effect.sync(() => Bus.registerProjectionCatchupHandler(name, handler))
+      )
+      ->Effect.runPromise
+      ->ignore
+    Base.connect(~name, ~channelSpecs, ~runtime, ~opts)
+  }
+}

package/src/adapter/EventCollector/LocalEventCollectorChannel.res.mjs

@@ -39,7 +39,51 @@ function Make(Bus) {
   };
 }
 
+function MakeProjection(Bus) {
+  let make = (param, eventTopics, param$1) => {
+    let eventTopicResources = Object.values(eventTopics).flatMap(outputs => outputs.resources);
+    return {
+      parts: undefined,
+      resources: eventTopicResources,
+      enqueueEvent: Pulumi.output((param, param$1, param$2) => Promise.resolve()),
+      handleChannelEvent: handleEvents => Pulumi.output((json, _ctx) => Effect.map(handleEvents(Stream.fromIterable([json])), () => {}))
+    };
+  };
+  let connect = (name, channelSpecs, runtime, param) => {
+    Effect.runPromise(Effect.flatMap(Deferred.await(runtime.parts.handlerDeferred), handler => Effect.sync(() => Bus.registerEventCollectorHandler(name, handler))));
+    channelSpecs.forEach(param => {
+      Object.values(param.eventTopics).forEach(topicOutputs => {
+        topicOutputs.resources.forEach(resource => {
+          resource.name.apply(topicName => {
+            let drainEffect = Effect.scoped(Effect.flatMap(Bus.subscribeToEventStream(topicName), stream => Stream.runForEach(stream, msg => Effect.zipRight(Effect.promise(async () => {
+              let handler = await Effect.runPromise(Deferred.await(runtime.parts.handlerDeferred));
+              return await handler(msg.json, undefined);
+            }), msg.done_))));
+            Effect.runFork(drainEffect);
+            Effect.runPromise(runtime.parts.subscriptionLatch.open);
+          });
+        });
+      });
+    });
+    return [];
+  };
+  let Base = {
+    make: make,
+    connect: connect
+  };
+  let connect$1 = (name, channelSpecs, runtime, opts) => {
+    Effect.runPromise(Effect.flatMap(Deferred.await(runtime.parts.handlerDeferred), handler => Effect.sync(() => Bus.registerProjectionCatchupHandler(name, handler))));
+    return connect(name, channelSpecs, runtime, opts);
+  };
+  return {
+    Base: Base,
+    make: make,
+    connect: connect$1
+  };
+}
+
 export {
   Make,
+  MakeProjection,
 }
 /* effect/Effect Not a pure module */

package/src/adapter/EventLog/EventLogStorage_InMemory.res

@@ -12,12 +12,16 @@ let makeMemoryStorage = (~name as _name, ~opts as _) => {
     ->Stm.commit
     ->Effect.runSync
 
+  // Keep-one snapshot per aggregate id. Node.js is single-threaded and snapshot
+  // writes are fire-and-forget upserts, so a plain dict suffices (no Stm).
+  let snapshots: dict<ReventlessCore.EventLog.snapshot> = Dict.make()
+
   let append: ReventlessCore.EventLog.append<string, JSON.t> = (seqNr, id, jsons) =>
     Stm.TRef.modify(eventsRef, events => {
       let existing = events->Dict.get(id)->Option.getOr([])
       let currentCount = existing->Array.length
       if seqNr != currentCount {
-        (Error("conflict"), events)
+        (Error(ReventlessCore.EventLog.Conflict), events)
       } else {
         events->Dict.set(id, existing->Array.concat(jsons))
         (Ok(), events)
@@ -28,10 +32,17 @@ let makeMemoryStorage = (~name as _name, ~opts as _) => {
 
   // The in-memory array is already resident; expose it as a stream for API uniformity.
   // The real performance gain from streaming comes from the DynamoDB adapter.
-  let replayStream: string => Stream.t<JSON.t, string, unit> = id =>
+  // Events are stored contiguously from seq 0, so the array index equals the
+  // sequence number and `fromSeq` is a plain slice offset.
+  let replayStream = (id, ~fromSeq=0) =>
     Stm.TRef.get(eventsRef)
     ->Stm.commit
-    ->Effect.map(events => events->Dict.get(id)->Option.getOr([]))
+    ->Effect.map(events =>
+      events
+      ->Dict.get(id)
+      ->Option.getOr([])
+      ->Array.filterWithIndex((_, i) => i >= fromSeq)
+    )
     ->Stream.fromEffect
     ->Stream.flatMap(arr => Stream.fromIterable(arr))
 
@@ -39,6 +50,14 @@ let makeMemoryStorage = (~name as _name, ~opts as _) => {
   let replay: ReventlessCore.EventLog.replay<string, JSON.t> = id =>
     replayStream(id)->Stream.runCollect->Effect.runPromise
 
+  let latestSnapshot: ReventlessCore.EventLog.latestSnapshot<string> = async id =>
+    Ok(snapshots->Dict.get(id))
+
+  let writeSnapshot: ReventlessCore.EventLog.writeSnapshot<string> = async (id, snap) => {
+    snapshots->Dict.set(id, snap)
+    Ok()
+  }
+
   // Appends each stream item sequentially to storage.
   // Node.js is single-threaded so a plain ref is safe for the seqNr counter.
   let appendStream: ReventlessCore.EventLog.appendStream<string, JSON.t> = (startingSeqNr, id, stream) => {
@@ -76,6 +95,8 @@ let makeMemoryStorage = (~name as _name, ~opts as _) => {
         replay,
         replayStream,
         appendStream,
+        latestSnapshot,
+        writeSnapshot,
       }),
     },
   )

package/src/adapter/EventLog/EventLogStorage_InMemory.res.mjs

@@ -13,6 +13,7 @@ import * as EventLogStorage_Sqlite$ReventlessLocal from "./EventLogStorage_Sqlit
 
 function makeMemoryStorage(_name, param) {
   let eventsRef = Effect.runSync(STM.commit(TRef.make({})));
+  let snapshots = {};
   let append = (seqNr, id, jsons) => Effect.runPromise(STM.commit(TRef.modify(eventsRef, events => {
     let existing = Stdlib_Option.getOr(events[id], []);
     let currentCount = existing.length;
@@ -20,7 +21,7 @@ function makeMemoryStorage(_name, param) {
       return [
         {
           TAG: "Error",
-          _0: "conflict"
+          _0: "Conflict"
         },
         events
       ];
@@ -35,8 +36,22 @@ function makeMemoryStorage(_name, param) {
       ];
     }
   })));
-  let replayStream = id => Stream$1.flatMap(Stream$1.fromEffect(Effect.map(STM.commit(TRef.get(eventsRef)), events => Stdlib_Option.getOr(events[id], []))), arr => Stream$1.fromIterable(arr));
-  let replay = id => Effect.runPromise(Stream.runCollect(replayStream(id)));
+  let replayStream = (id, fromSeqOpt) => {
+    let fromSeq = fromSeqOpt !== undefined ? fromSeqOpt : 0;
+    return Stream$1.flatMap(Stream$1.fromEffect(Effect.map(STM.commit(TRef.get(eventsRef)), events => Stdlib_Option.getOr(events[id], []).filter((param, i) => i >= fromSeq))), arr => Stream$1.fromIterable(arr));
+  };
+  let replay = id => Effect.runPromise(Stream.runCollect(replayStream(id, undefined)));
+  let latestSnapshot = async id => ({
+    TAG: "Ok",
+    _0: snapshots[id]
+  });
+  let writeSnapshot = async (id, snap) => {
+    snapshots[id] = snap;
+    return {
+      TAG: "Ok",
+      _0: undefined
+    };
+  };
   let appendStream = (startingSeqNr, id, stream) => {
     let seqNrRef = {
       contents: startingSeqNr
@@ -79,7 +94,9 @@ function makeMemoryStorage(_name, param) {
         append: append,
         replay: replay,
         replayStream: replayStream,
-        appendStream: appendStream
+        appendStream: appendStream,
+        latestSnapshot: latestSnapshot,
+        writeSnapshot: writeSnapshot
       })
     }
   ];

package/src/adapter/EventLog/EventLogStorage_Sqlite.res

@@ -16,10 +16,18 @@
 
 open ReventlessCore
 
-let ensureSchema = (db: SqliteDriver.t) =>
+let ensureSchema = (db: SqliteDriver.t) => {
   db->SqliteDriver.exec(
     "CREATE TABLE IF NOT EXISTS event_log (log_name TEXT NOT NULL, aggregate_id TEXT NOT NULL, seq_nr INTEGER NOT NULL, payload TEXT NOT NULL, PRIMARY KEY (log_name, aggregate_id, seq_nr))",
   )
+  // Keep-one aggregate-state snapshots (docs/plans/aggregate-snapshotting.md):
+  // one row per (log, aggregate), overwritten on every snapshot write. `state`
+  // is the JSON-encoded fold of events seq 0..seq_nr-1; `schema_hash` lets the
+  // consumer detect a state-schema drift and fall back to full replay.
+  db->SqliteDriver.exec(
+    "CREATE TABLE IF NOT EXISTS snapshot (log_name TEXT NOT NULL, aggregate_id TEXT NOT NULL, seq_nr INTEGER NOT NULL, state TEXT NOT NULL, schema_hash TEXT NOT NULL, PRIMARY KEY (log_name, aggregate_id))",
+  )
+}
 
 // Decodes the JSON column to a JSON.t. Returns JSON.Null on unparseable input
 // (should not happen for data we wrote ourselves).
@@ -53,15 +61,27 @@ let makeStorage = (~db: SqliteDriver.t, ~name: string, ~opts as _) => {
   let insertStmt = db->SqliteDriver.prepare(
     "INSERT INTO event_log(log_name, aggregate_id, seq_nr, payload) VALUES(?,?,?,?)",
   )
-  let countStmt = db->SqliteDriver.prepare(
-    "SELECT COUNT(*) AS c FROM event_log WHERE log_name=? AND aggregate_id=?",
+  // Expected next seq_nr for an aggregate. Events are always appended
+  // contiguously from seq 0 (the OCC check below forbids gaps), so
+  // `MAX(seq_nr)+1` equals the row count — but it reads the rightmost leaf of the
+  // (log_name, aggregate_id, seq_nr) PK index in O(log n) instead of `COUNT(*)`
+  // scanning every row for the aggregate. `COALESCE(…, -1)+1` yields 0 when empty.
+  let nextSeqStmt = db->SqliteDriver.prepare(
+    "SELECT COALESCE(MAX(seq_nr), -1) + 1 AS c FROM event_log WHERE log_name=? AND aggregate_id=?",
   )
   let selectByIdStmt = db->SqliteDriver.prepare(
-    "SELECT payload FROM event_log WHERE log_name=? AND aggregate_id=? ORDER BY seq_nr ASC",
+    "SELECT payload FROM event_log WHERE log_name=? AND aggregate_id=? AND seq_nr >= ? ORDER BY seq_nr ASC",
+  )
+  let lastRowidStmt = db->SqliteDriver.prepare("SELECT last_insert_rowid() AS r")
+  let upsertSnapshotStmt = db->SqliteDriver.prepare(
+    "INSERT OR REPLACE INTO snapshot(log_name, aggregate_id, seq_nr, state, schema_hash) VALUES(?,?,?,?,?)",
+  )
+  let selectSnapshotStmt = db->SqliteDriver.prepare(
+    "SELECT seq_nr, state, schema_hash FROM snapshot WHERE log_name=? AND aggregate_id=?",
   )
 
-  let currentCount = (id: string): int =>
-    switch countStmt->SqliteDriver.get([JSON.Encode.string(name), JSON.Encode.string(id)]) {
+  let expectedNextSeq = (id: string): int =>
+    switch nextSeqStmt->SqliteDriver.get([JSON.Encode.string(name), JSON.Encode.string(id)]) {
     | Some(row) =>
       switch row->Dict.get("c") {
       | Some(JSON.Number(n)) => Float.toInt(n)
@@ -70,10 +90,14 @@ let makeStorage = (~db: SqliteDriver.t, ~name: string, ~opts as _) => {
     | None => 0
     }
 
-  let append: EventLog.append<string, JSON.t> = async (seqNr, id, jsons) => {
+  // `track=false` for appendStream: bulk-replayed batches never flow through
+  // the EventTopic publish cycle, so they must not enter the projection pending
+  // set — an entry nobody resolves would pin the checkpoint low-watermark
+  // forever (see ProjectionCheckpoint.res).
+  let appendTracked = async (~track, seqNr, id, jsons: array<JSON.t>) => {
     try {
-      db->SqliteDriver.transaction(() => {
-        let existing = currentCount(id)
+      let lastRowid = db->SqliteDriver.transaction(() => {
+        let existing = expectedNextSeq(id)
         if seqNr != existing {
           throw(Failure("conflict"))
         }
@@ -85,17 +109,62 @@ let makeStorage = (~db: SqliteDriver.t, ~name: string, ~opts as _) => {
             JSON.Encode.string(JSON.stringify(json)),
           ])
         })
+        switch lastRowidStmt->SqliteDriver.get([]) {
+        | Some(row) =>
+          switch row->Dict.get("r") {
+          | Some(JSON.Number(n)) => Float.toInt(n)
+          | _ => 0
+          }
+        | None => 0
+        }
       })
+      // Register the committed batch as appended-but-not-yet-published for the
+      // projection checkpoint low-watermark; Platform's afterPublish hook
+      // resolves the msgIds once the publish cycle completes. A transaction's
+      // inserts get contiguous rowids ending at lastRowid.
+      if track && lastRowid > 0 {
+        let count = jsons->Array.length
+        ProjectionPending.trackAppended(
+          ~axis=ProjectionPending.Aggregate,
+          jsons
+          ->Array.mapWithIndex((json, i) =>
+            json
+            ->JSON.Decode.object
+            ->Option.flatMap(d => d->Dict.get("msgId"))
+            ->Option.flatMap(JSON.Decode.string)
+            ->Option.map(msgId => (msgId, lastRowid - count + 1 + i))
+          )
+          ->Array.filterMap(x => x),
+        )
+      }
       Ok()
     } catch {
-    | Failure(msg) => Error(msg)
-    | _ => Error("conflict")
+    // The deliberate seq_nr check above throws Failure("conflict"); a lost race
+    // shows up as a PRIMARY KEY violation on (log_name, aggregate_id, seq_nr) —
+    // also a genuine OCC conflict. Every OTHER failure (disk full, SQL error,
+    // locked db) is a real StorageFailure, not the retryable Conflict sentinel.
+    | Failure(msg) =>
+      msg == "conflict"
+        ? Error(ReventlessCore.EventLog.Conflict)
+        : Error(ReventlessCore.EventLog.StorageFailure(msg))
+    | exn =>
+      let msg = exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("")
+      msg->String.includes("constraint failed")
+        ? Error(ReventlessCore.EventLog.Conflict)
+        : Error(ReventlessCore.EventLog.StorageFailure(msg == "" ? "storage error" : msg))
     }
   }
 
-  let replayArray = (id: string): array<JSON.t> =>
+  let append: EventLog.append<string, JSON.t> = (seqNr, id, jsons) =>
+    appendTracked(~track=true, seqNr, id, jsons)
+
+  let replayArray = (id: string, ~fromSeq=0): array<JSON.t> =>
     selectByIdStmt
-    ->SqliteDriver.all([JSON.Encode.string(name), JSON.Encode.string(id)])
+    ->SqliteDriver.all([
+      JSON.Encode.string(name),
+      JSON.Encode.string(id),
+      JSON.Encode.int(fromSeq),
+    ])
     ->Array.map(decodePayload)
 
   let replay: EventLog.replay<string, JSON.t> = async id => replayArray(id)
@@ -103,23 +172,73 @@ let makeStorage = (~db: SqliteDriver.t, ~name: string, ~opts as _) => {
   // node:sqlite's iterate() could back a lazy stream, but the current Stream
   // API only has array-based fromIterable. Small events, dev-only backend →
   // materialise via all() and wrap. Swap to a lazy adapter if it matters.
-  let replayStream: string => Stream.t<JSON.t, string, unit> = id =>
-    replayArray(id)->Stream.fromIterable
-
-  let appendStream: EventLog.appendStream<string, JSON.t> = (startingSeqNr, id, stream) => {
-    let seqNrRef = ref(startingSeqNr)
-    stream->Stream.runForEach(json => {
-      let currentSeq = seqNrRef.contents
-      Effect.promise(() => append(currentSeq, id, [json]))->Effect.flatMap(result =>
+  let replayStream = (id, ~fromSeq=?) => replayArray(id, ~fromSeq=?fromSeq)->Stream.fromIterable
+
+  let latestSnapshot: EventLog.latestSnapshot<string> = async id =>
+    try {
+      switch selectSnapshotStmt->SqliteDriver.get([
+        JSON.Encode.string(name),
+        JSON.Encode.string(id),
+      ]) {
+      | Some(row) =>
+        switch (row->Dict.get("seq_nr"), row->Dict.get("state"), row->Dict.get("schema_hash")) {
+        | (Some(JSON.Number(seq)), Some(JSON.String(stateStr)), Some(JSON.String(schemaHash))) =>
+          Ok(
+            Some({
+              EventLog.seqNr: Float.toInt(seq),
+              state: JSON.parseOrThrow(stateStr),
+              schemaHash,
+            }),
+          )
+        | _ => Error("snapshot row has unexpected shape")
+        }
+      | None => Ok(None)
+      }
+    } catch {
+    | exn =>
+      Error(
+        exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("snapshot read error"),
+      )
+    }
+
+  let writeSnapshot: EventLog.writeSnapshot<string> = async (id, snap) =>
+    try {
+      upsertSnapshotStmt->SqliteDriver.run([
+        JSON.Encode.string(name),
+        JSON.Encode.string(id),
+        JSON.Encode.int(snap.seqNr),
+        JSON.Encode.string(JSON.stringify(snap.state)),
+        JSON.Encode.string(snap.schemaHash),
+      ])
+      Ok()
+    } catch {
+    | exn =>
+      Error(
+        exn
+        ->JsExn.fromException
+        ->Option.flatMap(JsExn.message)
+        ->Option.getOr("snapshot write error"),
+      )
+    }
+
+  // Collect the stream and hand the whole batch to `append`, which inserts every
+  // event under one transaction with a single OCC check (vs the old per-element
+  // append: one transaction + one seq lookup per event over a bulk replay). This
+  // also makes the replay atomic — all events land or none do, rather than
+  // leaving a partial prefix committed if a later element hit a raced PK.
+  let appendStream: EventLog.appendStream<string, JSON.t> = (startingSeqNr, id, stream) =>
+    stream
+    ->Stream.runCollect
+    ->Effect.flatMap(jsons =>
+      Effect.promise(() => appendTracked(~track=false, startingSeqNr, id, jsons))->Effect.flatMap(result =>
         switch result {
-        | Ok() =>
-          seqNrRef := currentSeq + 1
-          Effect.succeed()
-        | Error(msg) => Effect.fail(msg)
+        | Ok() => Effect.succeed()
+        // appendStream's error channel is a string; map the typed append error.
+        | Error(ReventlessCore.EventLog.Conflict) => Effect.fail("conflict")
+        | Error(StorageFailure(msg)) => Effect.fail(msg)
         }
       )
-    })
-  }
+    )
 
   (
     name,
@@ -131,6 +250,8 @@ let makeStorage = (~db: SqliteDriver.t, ~name: string, ~opts as _) => {
         replay,
         replayStream,
         appendStream,
+        latestSnapshot,
+        writeSnapshot,
       }),
     },
   )