effect 4.0.5 → 4.0.7

package/dist/eventlog/SqlEventJournal.js

@@ -1,15 +1,11 @@
 /**
- * SQL-backed persistence for the unstable event-log journal.
- *
- * This module implements `EventJournal` on top of a `SqlClient`. It stores event
- * entries as encoded bytes and stores per-remote sequence metadata in separate
- * tables, giving event-log programs a durable journal that can be replayed after
- * restart and synchronized with remote journals.
+ * SQL-backed event journal with per-remote upload checkpoints.
  *
  * @stability unstable
  * @since 4.0.0
  */
 import * as Arr from "../Array.js";
+import * as Clock from "../Clock.js";
 import * as Effect from "../Effect.js";
 import * as Uuid from "../internal/uuid.js";
 import * as Layer from "../Layer.js";
@@ -20,13 +16,26 @@ import * as SqlClient from "../sql/SqlClient.js";
 import * as SqlError from "../sql/SqlError.js";
 import * as SqlSchema from "../sql/SqlSchema.js";
 import * as EventJournal from "./EventJournal.js";
+import { ConflictIndex } from "./internal/conflicts.js";
 /**
  * Creates an `EventJournal` backed by a SQL database.
  *
  * **Details**
  *
- * The constructor creates the entry and remote metadata tables when needed,
- * persists local and remote entries, and uses the configured `SqlClient`.
+ * Maintains `entryTable`, `${entryTable}_sequence`, `${entryTable}_sources`,
+ * `${remotesTable}_checkpoints` and `remoteCursorsTable` (default
+ * `${remotesTable}_cursors`) using the configured `SqlClient`.
+ *
+ * **Gotchas**
+ *
+ * Creating or migrating the schema requires a writable connection. Migrated
+ * databases can be reopened read-only.
+ *
+ * Journals from earlier releases keep one `remotesTable` row per entry per
+ * remote. Opening one replaces those rows with entry sources and drops the
+ * table; an entry known to several remotes keeps one of them as its source.
+ *
+ * Upload acknowledgements do not advance past unseen entries.
  *
  * @stability unstable
  * @category constructors
@@ -36,183 +45,421 @@ export const make = options => Effect.gen(function* () {
   const sql = (yield* SqlClient.SqlClient).withoutTransforms();
   const entryTable = options?.entryTable ?? "effect_event_journal";
   const remotesTable = options?.remotesTable ?? "effect_event_remotes";
+  const remoteCursorsTable = options?.remoteCursorsTable ?? `${remotesTable}_cursors`;
+  const sequenceTable = `${entryTable}_sequence`;
+  const sourcesTable = `${entryTable}_sources`;
+  const checkpointsTable = `${remotesTable}_checkpoints`;
   const entryTableSql = sql(entryTable);
   const remotesTableSql = sql(remotesTable);
-  yield* sql.onDialectOrElse({
-    pg: () => sql`
-          CREATE TABLE IF NOT EXISTS ${entryTableSql} (
-            id BYTEA PRIMARY KEY,
-            event TEXT NOT NULL,
-            primary_key TEXT NOT NULL,
-            payload BYTEA NOT NULL,
-            timestamp BIGINT NOT NULL
-          )`,
-    mysql: () => sql`
-          CREATE TABLE IF NOT EXISTS ${entryTableSql} (
-            id BINARY(16) PRIMARY KEY,
-            event TEXT NOT NULL,
-            primary_key TEXT NOT NULL,
-            payload BLOB NOT NULL,
-            timestamp BIGINT NOT NULL
-          )`,
-    mssql: () => sql`
-          CREATE TABLE IF NOT EXISTS ${entryTableSql} (
-            id UNIQUEIDENTIFIER PRIMARY KEY,
-            event NVARCHAR(MAX) NOT NULL,
-            primary_key NVARCHAR(MAX) NOT NULL,
-            payload VARBINARY(MAX) NOT NULL,
-            timestamp BIGINT NOT NULL
-          )`,
-    orElse: () => sql`
-          CREATE TABLE IF NOT EXISTS ${entryTableSql} (
-            id BLOB PRIMARY KEY,
-            event TEXT NOT NULL,
-            primary_key TEXT NOT NULL,
-            payload BLOB NOT NULL,
-            timestamp INTEGER NOT NULL
-          )`
-  }).pipe(withTracerDisabled);
-  yield* sql.onDialectOrElse({
-    pg: () => sql`
-          CREATE TABLE IF NOT EXISTS ${remotesTableSql} (
-            remote_id BYTEA NOT NULL,
-            entry_id BYTEA NOT NULL,
-            sequence INT NOT NULL,
-            PRIMARY KEY (remote_id, entry_id)
-          )`,
-    mysql: () => sql`
-          CREATE TABLE IF NOT EXISTS ${remotesTableSql} (
-            remote_id BINARY(16) NOT NULL,
-            entry_id BINARY(16) NOT NULL,
-            sequence INT NOT NULL,
-            PRIMARY KEY (remote_id, entry_id)
-          )`,
-    mssql: () => sql`
-          CREATE TABLE IF NOT EXISTS ${remotesTableSql} (
-            remote_id UNIQUEIDENTIFIER NOT NULL,
-            entry_id UNIQUEIDENTIFIER NOT NULL,
-            sequence INT NOT NULL,
-            PRIMARY KEY (remote_id, entry_id)
-          )`,
-    orElse: () => sql`
-          CREATE TABLE IF NOT EXISTS ${remotesTableSql} (
-            remote_id BLOB NOT NULL,
-            entry_id BLOB NOT NULL,
-            sequence INT NOT NULL,
-            PRIMARY KEY (remote_id, entry_id)
-          )`
+  const sequenceTableSql = sql(sequenceTable);
+  const sourcesTableSql = sql(sourcesTable);
+  const checkpointsTableSql = sql(checkpointsTable);
+  const remoteCursorsTableSql = sql(remoteCursorsTable);
+  // to_regclass folds unquoted names to lowercase, so pass the quoted identifier.
+  const quoted = table => sql`${sql(table)}`.compile(true)[0];
+  const tableExists = table => sql.onDialectOrElse({
+    // Reading the catalog refreshes cached names after concurrent DDL.
+    pg: () => sql`SELECT 1 AS found FROM pg_class WHERE oid = to_regclass(${quoted(table)})`,
+    mysql: () => sql`SELECT 1 AS found FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ${table}`,
+    mssql: () => sql`SELECT 1 AS found WHERE OBJECT_ID(${table}, 'U') IS NOT NULL`,
+    orElse: () => sql`SELECT 1 AS found FROM sqlite_master WHERE type = 'table' AND name = ${table}`
+  }).pipe(Effect.map(rows => rows.length > 0));
+  const createSchema = Effect.gen(function* () {
+    const eventIndex = sql(`${entryTable}_event_idx`);
+    const timestampIndex = sql(`${entryTable}_timestamp_idx`);
+    yield* sql.onDialectOrElse({
+      pg: () => sql`
+            CREATE TABLE IF NOT EXISTS ${entryTableSql} (
+              id BYTEA PRIMARY KEY,
+              event TEXT NOT NULL,
+              primary_key TEXT NOT NULL,
+              payload BYTEA NOT NULL,
+              timestamp BIGINT NOT NULL
+            )`,
+      mysql: () => sql`
+            CREATE TABLE IF NOT EXISTS ${entryTableSql} (
+              id BINARY(16) PRIMARY KEY,
+              event TEXT NOT NULL,
+              primary_key TEXT NOT NULL,
+              payload BLOB NOT NULL,
+              timestamp BIGINT NOT NULL
+            )`,
+      mssql: () => sql`
+            CREATE TABLE IF NOT EXISTS ${entryTableSql} (
+              id UNIQUEIDENTIFIER PRIMARY KEY,
+              event NVARCHAR(MAX) NOT NULL,
+              primary_key NVARCHAR(MAX) NOT NULL,
+              payload VARBINARY(MAX) NOT NULL,
+              timestamp BIGINT NOT NULL
+            )`,
+      orElse: () => sql`
+            CREATE TABLE IF NOT EXISTS ${entryTableSql} (
+              id BLOB PRIMARY KEY,
+              event TEXT NOT NULL,
+              primary_key TEXT NOT NULL,
+              payload BLOB NOT NULL,
+              timestamp INTEGER NOT NULL
+            )`
+    });
+    // Insertion sequences let uploads include older entries imported later.
+    yield* sql.onDialectOrElse({
+      pg: () => sql`
+            CREATE TABLE IF NOT EXISTS ${sequenceTableSql} (
+              sequence BIGSERIAL PRIMARY KEY,
+              entry_id BYTEA NOT NULL UNIQUE
+            )`,
+      mysql: () => sql`
+            CREATE TABLE IF NOT EXISTS ${sequenceTableSql} (
+              sequence BIGINT AUTO_INCREMENT PRIMARY KEY,
+              entry_id BINARY(16) NOT NULL UNIQUE
+            )`,
+      mssql: () => sql`
+            CREATE TABLE ${sequenceTableSql} (
+              sequence BIGINT IDENTITY(1,1) PRIMARY KEY,
+              entry_id UNIQUEIDENTIFIER NOT NULL UNIQUE
+            )`,
+      orElse: () => sql`
+            CREATE TABLE IF NOT EXISTS ${sequenceTableSql} (
+              sequence INTEGER PRIMARY KEY,
+              entry_id BLOB NOT NULL UNIQUE
+            )`
+    });
+    yield* sql.onDialectOrElse({
+      pg: () => sql`
+            CREATE TABLE IF NOT EXISTS ${checkpointsTableSql} (
+              remote_id BYTEA PRIMARY KEY,
+              sequence BIGINT NOT NULL
+            )`,
+      mysql: () => sql`
+            CREATE TABLE IF NOT EXISTS ${checkpointsTableSql} (
+              remote_id BINARY(16) PRIMARY KEY,
+              sequence BIGINT NOT NULL
+            )`,
+      mssql: () => sql`
+            CREATE TABLE ${checkpointsTableSql} (
+              remote_id UNIQUEIDENTIFIER PRIMARY KEY,
+              sequence BIGINT NOT NULL
+            )`,
+      orElse: () => sql`
+            CREATE TABLE IF NOT EXISTS ${checkpointsTableSql} (
+              remote_id BLOB PRIMARY KEY,
+              sequence INTEGER NOT NULL
+            )`
+    });
+    yield* sql.onDialectOrElse({
+      pg: () => sql`
+            CREATE TABLE IF NOT EXISTS ${remoteCursorsTableSql} (
+              remote_id BYTEA PRIMARY KEY,
+              next_sequence INT NOT NULL
+            )`,
+      mysql: () => sql`
+            CREATE TABLE IF NOT EXISTS ${remoteCursorsTableSql} (
+              remote_id BINARY(16) PRIMARY KEY,
+              next_sequence INT NOT NULL
+            )`,
+      mssql: () => sql`
+            CREATE TABLE ${remoteCursorsTableSql} (
+              remote_id UNIQUEIDENTIFIER PRIMARY KEY,
+              next_sequence INT NOT NULL
+            )`,
+      orElse: () => sql`
+            CREATE TABLE IF NOT EXISTS ${remoteCursorsTableSql} (
+              remote_id BLOB PRIMARY KEY,
+              next_sequence INT NOT NULL
+            )`
+    });
+    // The remote each imported entry came from, so uploads skip it.
+    yield* sql.onDialectOrElse({
+      pg: () => sql`
+            CREATE TABLE IF NOT EXISTS ${sourcesTableSql} (
+              entry_id BYTEA PRIMARY KEY,
+              remote_id BYTEA NOT NULL
+            )`,
+      mysql: () => sql`
+            CREATE TABLE IF NOT EXISTS ${sourcesTableSql} (
+              entry_id BINARY(16) PRIMARY KEY,
+              remote_id BINARY(16) NOT NULL
+            )`,
+      mssql: () => sql`
+            CREATE TABLE ${sourcesTableSql} (
+              entry_id UNIQUEIDENTIFIER PRIMARY KEY,
+              remote_id UNIQUEIDENTIFIER NOT NULL
+            )`,
+      orElse: () => sql`
+            CREATE TABLE IF NOT EXISTS ${sourcesTableSql} (
+              entry_id BLOB PRIMARY KEY,
+              remote_id BLOB NOT NULL
+            )`
+    });
+    yield* sql.onDialectOrElse({
+      // MySQL indexes TEXT columns by prefix only.
+      mysql: () => Effect.andThen(sql`CREATE INDEX ${eventIndex} ON ${entryTableSql} (event(191), primary_key(191), timestamp)`, sql`CREATE INDEX ${timestampIndex} ON ${entryTableSql} (timestamp)`),
+      // B-tree entries are limited to about 2.7 KB, so index hashes of the
+      // unbounded columns.
+      pg: () => Effect.andThen(sql`CREATE INDEX IF NOT EXISTS ${eventIndex} ON ${entryTableSql} (md5(event), md5(primary_key), timestamp)`, sql`CREATE INDEX IF NOT EXISTS ${timestampIndex} ON ${entryTableSql} (timestamp)`),
+      // NVARCHAR(MAX) columns cannot be index keys.
+      mssql: () => sql`CREATE INDEX ${timestampIndex} ON ${entryTableSql} (timestamp)`,
+      orElse: () => Effect.andThen(sql`CREATE INDEX IF NOT EXISTS ${eventIndex} ON ${entryTableSql} (event, primary_key, timestamp)`, sql`CREATE INDEX IF NOT EXISTS ${timestampIndex} ON ${entryTableSql} (timestamp)`)
+    });
+    // Number existing entries in creation order. `WHERE true` keeps SQLite
+    // from parsing ON CONFLICT as a join constraint.
+    yield* sql`
+        INSERT INTO ${sequenceTableSql} (entry_id)
+        SELECT id FROM ${entryTableSql} WHERE true ORDER BY timestamp ASC, id ASC
+        ON CONFLICT DO NOTHING
+      `;
+  });
+  // Earlier releases kept a remotes row per entry per remote. An entry known
+  // to several remotes keeps one of them as its source, and a remote without
+  // a cursor resumes after its highest downloaded sequence.
+  const importRemotes = Effect.gen(function* () {
+    yield* sql`
+        INSERT INTO ${sourcesTableSql} (entry_id, remote_id)
+        SELECT entry_id, remote_id FROM ${remotesTableSql} WHERE true
+        ON CONFLICT DO NOTHING
+      `;
+    yield* sql`
+        INSERT INTO ${remoteCursorsTableSql} (remote_id, next_sequence)
+        SELECT remote_id, MAX(sequence) + 1 FROM ${remotesTableSql} GROUP BY remote_id
+        ON CONFLICT DO NOTHING
+      `;
+    yield* sql`DROP TABLE ${remotesTableSql}`;
+  });
+  // Both steps run in one transaction, so a complete schema has the sources
+  // table and no remotes table. Checking that first lets complete databases
+  // open read-only; PostgreSQL also locks and rechecks so concurrent opens
+  // migrate once.
+  const schemaState = Effect.all({
+    sources: tableExists(sourcesTable),
+    remotes: tableExists(remotesTable)
+  });
+  const isComplete = state => state.sources && !state.remotes;
+  yield* Effect.gen(function* () {
+    if (isComplete(yield* schemaState)) return;
+    yield* Effect.gen(function* () {
+      yield* sql.onDialectOrElse({
+        pg: () => sql`SELECT pg_advisory_xact_lock(hashtext(${quoted(entryTable)}))`,
+        orElse: () => Effect.void
+      });
+      const state = yield* schemaState;
+      if (!state.sources) yield* createSchema;
+      if (state.remotes) yield* importRemotes;
+    }).pipe(sql.withTransaction);
   }).pipe(withTracerDisabled);
   const decodeEntryRows = Schema.decodeUnknownEffect(EntryRowArray);
   const decodeEntryIdRows = Schema.decodeUnknownEffect(EntryIdRowArray);
   const toEntries = rows => rows.map(toEntry);
-  const insertEntry = SqlSchema.void({
-    Request: EntryRow,
-    execute: entry => sql`INSERT INTO ${entryTableSql} ${sql.insert(entry)} ON CONFLICT DO NOTHING`
-  });
+  // Sort timestamp ties by id bytes consistently across dialects.
+  const toOrderedEntries = rows => rows.map(toEntry).sort(EventJournal.Entry.Order);
   const insertEntries = SqlSchema.void({
     Request: EntryRowArray,
-    execute: entries => sql`INSERT INTO ${entryTableSql} ${sql.insert(entries)} ON CONFLICT DO NOTHING`
+    execute: entries => forEachChunk(entries, entryColumns, chunk => sql`INSERT INTO ${entryTableSql} ${sql.insert(chunk)} ON CONFLICT DO NOTHING`)
+  });
+  const insertSources = (entries, remoteId) => forEachChunk(entries, 2, chunk => {
+    const rows = chunk.map(entry => ({
+      entry_id: entry.id,
+      remote_id: remoteId
+    }));
+    return sql`INSERT INTO ${sourcesTableSql} ${sql.insert(rows)} ON CONFLICT DO NOTHING`;
   });
-  const insertRemotes = SqlSchema.void({
-    Request: RemoteRowArray,
-    execute: entries => sql`INSERT INTO ${remotesTableSql} ${sql.insert(entries)} ON CONFLICT DO NOTHING`
+  // Serialize writers so checkpoints cannot skip out-of-order commits.
+  // Lock before callbacks in both paths to avoid projection-row deadlocks.
+  const lockWriters = sql.onDialectOrElse({
+    pg: () => Effect.asVoid(sql`LOCK TABLE ${sequenceTableSql} IN SHARE ROW EXCLUSIVE MODE`),
+    orElse: () => Effect.void
   });
+  const insertSequence = entries => forEachChunk(entries, 1, chunk => {
+    const rows = chunk.map(entry => ({
+      entry_id: entry.id
+    }));
+    return sql`INSERT INTO ${sequenceTableSql} ${sql.insert(rows)} ON CONFLICT DO NOTHING`;
+  });
+  const decodeCursorRows = Schema.decodeUnknownEffect(CursorRowArray);
+  const readCursor = remoteId => sql`SELECT next_sequence FROM ${remoteCursorsTableSql} WHERE remote_id = ${remoteId}`.pipe(Effect.flatMap(decodeCursorRows), Effect.map(rows => rows.length > 0 ? rows[0].next_sequence : 0));
+  // Cursors only move forward.
+  const writeCursor = (remoteId, next) => sql`
+        INSERT INTO ${remoteCursorsTableSql} ${sql.insert({
+    remote_id: remoteId,
+    next_sequence: next
+  })}
+        ON CONFLICT (remote_id) DO UPDATE SET next_sequence = excluded.next_sequence
+        WHERE ${remoteCursorsTableSql}.next_sequence < excluded.next_sequence
+      `;
   const pubsub = yield* PubSub.unbounded();
+  const clock = yield* Clock.Clock;
   const writeFromRemote = Effect.fnUntraced(function* (options) {
     const entries = options.entries.map(remoteEntry => remoteEntry.entry);
-    const remoteRows = options.entries.map(remoteEntry => ({
-      remote_id: options.remoteId,
-      entry_id: remoteEntry.entry.id,
-      sequence: remoteEntry.remoteSequence
-    }));
-    const existingIds = new Set();
-    if (entries.length > 0) {
-      yield* sql`SELECT id FROM ${entryTableSql} WHERE ${sql.in("id", entries.map(entry => entry.id))}`.pipe(Effect.flatMap(decodeEntryIdRows), Effect.tap(rows => Effect.sync(() => {
-        for (const row of rows) {
-          existingIds.add(Uuid.stringify(row.id));
-        }
-      })));
-    }
-    if (entries.length > 0) {
-      yield* insertEntries(entries.map(toEntryRow));
-    }
-    if (remoteRows.length > 0) {
-      yield* insertRemotes(remoteRows);
-    }
-    const uncommitted = options.entries.filter(entry => !existingIds.has(entry.entry.idString));
+    yield* lockWriters;
+    const existingRows = yield* forEachChunk(entries, 1, chunk => sql`SELECT id FROM ${entryTableSql} WHERE ${sql.in("id", chunk.map(entry => entry.id))}`).pipe(Effect.flatMap(decodeEntryIdRows));
+    const existingIds = new Set(existingRows.map(row => Uuid.stringify(row.id)));
+    // Overlapping batches can repeat an entry; the first copy wins.
+    const incomingIds = new Set();
+    const uncommitted = options.entries.filter(entry => {
+      const idString = entry.entry.idString;
+      if (existingIds.has(idString) || incomingIds.has(idString)) return false;
+      incomingIds.add(idString);
+      return true;
+    });
     const duplicateEntries = options.entries.filter(entry => existingIds.has(entry.entry.idString)).map(entry => entry.entry);
     const compacted = options.compact ? yield* options.compact(uncommitted) : uncommitted.map(remoteEntry => remoteEntry.entry);
-    for (const entry of compacted) {
-      const conflicts = yield* sql`
-            SELECT *
-            FROM ${entryTableSql}
-            WHERE event = ${entry.event} AND
-                  primary_key = ${entry.primaryKey} AND
-                  timestamp >= ${entry.createdAtMillis}
-            ORDER BY timestamp ASC
-          `.pipe(Effect.flatMap(decodeEntryRows), Effect.map(toEntries));
-      yield* options.effect({
-        entry,
-        conflicts
-      });
+    if (Arr.isReadonlyArrayNonEmpty(compacted)) {
+      const index = new ConflictIndex(compacted);
+      const conflicts = groups => {
+        const events = Array.from(new Set(groups.map(group => group.event)));
+        const primaryKeys = Array.from(new Set(groups.map(group => group.primaryKey)));
+        return sql`SELECT * FROM ${entryTableSql} WHERE ${sql.and([
+        // Match the hash index; exact predicates below rule out hash collisions.
+        ...sql.onDialectOrElse({
+          pg: () => [sql`md5(event) IN (${sql.csv(events.map(event => sql`md5(${event})`))})`, sql`md5(primary_key) IN (${sql.csv(primaryKeys.map(primaryKey => sql`md5(${primaryKey})`))})`],
+          orElse: () => []
+        }), sql.in("event", events), sql.in("primary_key", primaryKeys),
+        // A coarse filter: the index compares ids exactly.
+        sql`timestamp >= ${index.oldestMillis}`])}`;
+      };
+      // Each group binds its event and key, twice on PostgreSQL, and one
+      // parameter is left for the timestamp.
+      const groupsPerChunk = Math.floor((maxParameters - 1) / sql.onDialectOrElse({
+        pg: () => 4,
+        orElse: () => 2
+      }));
+      const stored = yield* Effect.forEach(Arr.chunksOf(index.groups(), groupsPerChunk), conflicts).pipe(Effect.flatMap(rows => decodeEntryRows(rows.flat())), Effect.map(toEntries));
+      // Chunks can share an event and a key, so a row can be returned twice.
+      const added = new Set();
+      for (const entry of stored) {
+        if (added.has(entry.idString)) continue;
+        added.add(entry.idString);
+        index.add(entry);
+      }
+      yield* options.effect(index.toBatch());
+    }
+    // Keep the incoming batch out of conflict queries and persist only after
+    // compaction and every callback have succeeded.
+    if (uncommitted.length > 0) {
+      const imported = uncommitted.map(remoteEntry => remoteEntry.entry);
+      yield* insertEntries(imported.map(toEntryRow));
+      yield* insertSequence(imported);
+      yield* insertSources(imported, options.remoteId);
     }
+    // The change stream is ordered, so every earlier sequence is now known.
+    if (options.entries.length > 0) {
+      yield* writeCursor(options.remoteId, EventJournal.nextSequenceAfter(options.entries));
+    }
+    yield* EventJournal.warnIfAhead(clock, uncommitted.map(remoteEntry => remoteEntry.entry.id));
     return {
       duplicateEntries
     };
   });
+  // Other processes share the database but not the entry id sequence. Ids in
+  // the latest millisecond are few, and comparing them here avoids ordering
+  // id bytes in SQL.
+  const latestEntryIds = sql`
+      SELECT id FROM ${entryTableSql} WHERE timestamp = (SELECT MAX(timestamp) FROM ${entryTableSql})
+    `.pipe(Effect.flatMap(decodeEntryIdRows), Effect.map(rows => rows.map(row => row.id)));
+  const writeMany = ({
+    effect,
+    entries
+  }) => Effect.uninterruptibleMask(restore => Effect.gen(function* () {
+    yield* lockWriters;
+    // Rows visible here include an enclosing transaction's uncommitted
+    // ones, which local ids must also sort after.
+    const committed = yield* EventJournal.makeEntries(entries, yield* latestEntryIds);
+    const result = yield* restore(effect(committed)).pipe(Effect.mapError(error => new CallbackFailure(error)));
+    yield* insertEntries(committed.map(toEntryRow));
+    yield* insertSequence(committed);
+    return {
+      result,
+      committed
+    };
+  }).pipe(sql.withTransaction, Effect.mapError(error => error instanceof CallbackFailure ? error.error : error instanceof EventJournal.EventJournalError ? error : new EventJournal.EventJournalError({
+    cause: error,
+    method: "writeMany"
+  })), Effect.flatMap(({
+    committed,
+    result
+  }) => Effect.as(PubSub.publishAll(pubsub, committed), result)))).pipe(withTracerDisabled);
   return EventJournal.EventJournal.of({
-    entries: sql`SELECT * FROM ${entryTableSql} ORDER BY timestamp ASC`.pipe(withTracerDisabled, Effect.flatMap(decodeEntryRows), Effect.map(toEntries), Effect.mapError(cause => new EventJournal.EventJournalError({
+    entries: sql`SELECT * FROM ${entryTableSql} ORDER BY timestamp ASC`.pipe(withTracerDisabled, Effect.flatMap(decodeEntryRows), Effect.map(toOrderedEntries), Effect.mapError(cause => new EventJournal.EventJournalError({
       cause,
       method: "entries"
     }))),
-    write: Effect.fnUntraced(function* ({
+    entriesById: ids => forEachChunk(ids, 1, chunk => sql`SELECT * FROM ${entryTableSql} WHERE ${sql.in("id", chunk)}`).pipe(withTracerDisabled, Effect.flatMap(decodeEntryRows), Effect.map(rows => {
+      const byId = new Map(toEntries(rows).map(entry => [entry.idString, entry]));
+      return ids.map(id => byId.get(Uuid.stringify(id)));
+    }), Effect.mapError(cause => new EventJournal.EventJournalError({
+      cause,
+      method: "entriesById"
+    }))),
+    write: ({
       effect,
-      event,
-      payload,
-      primaryKey
-    }) {
-      const entry = new EventJournal.Entry({
-        id: EventJournal.makeEntryIdUnsafe(),
-        event,
-        primaryKey,
-        payload
-      }, {
-        disableChecks: true
-      });
-      return yield* Effect.uninterruptibleMask(restore => restore(effect(entry)).pipe(Effect.tap(insertEntry(toEntryRow(entry)).pipe(Effect.mapError(cause => new EventJournal.EventJournalError({
-        cause,
-        method: "write"
-      })))), Effect.tap(PubSub.publish(pubsub, entry))));
-    }, withTracerDisabled),
-    writeFromRemote: options => writeFromRemote(options).pipe(withTracerDisabled, Effect.catchIf(e => e._tag !== "EventJournalError", cause => Effect.fail(new EventJournal.EventJournalError({
+      ...entry
+    }) => writeMany({
+      entries: [entry],
+      effect: entries => effect(entries[0])
+    }),
+    writeMany,
+    writeFromRemote: options => writeFromRemote(options).pipe(sql.withTransaction, withTracerDisabled, Effect.catchIf(e => e._tag !== "EventJournalError", cause => Effect.fail(new EventJournal.EventJournalError({
       cause,
       method: "writeFromRemote"
     })))),
     withRemoteUncommited: Effect.fnUntraced(function* (remoteId, f) {
-      const entries = yield* sql`
-            SELECT *
-            FROM ${entryTableSql}
-            WHERE id NOT IN (SELECT entry_id FROM ${remotesTableSql} WHERE remote_id = ${remoteId})
-            ORDER BY timestamp ASC
-          `.pipe(Effect.flatMap(decodeEntryRows), Effect.map(toEntries), Effect.mapError(cause => new EventJournal.EventJournalError({
+      const mapError = Effect.mapError(cause => new EventJournal.EventJournalError({
         cause,
         method: "withRemoteUncommited"
-      })));
-      if (!Arr.isReadonlyArrayNonEmpty(entries)) return yield* Effect.succeedNone;
-      return yield* Effect.asSome(f(entries));
+      }));
+      // Capture an upper bound so entries committed during upload remain pending.
+      const [{
+        checkpoint,
+        latest
+      }] = yield* sql`
+            SELECT
+              COALESCE((SELECT sequence FROM ${checkpointsTableSql} WHERE remote_id = ${remoteId}), 0) AS checkpoint,
+              COALESCE((SELECT MAX(sequence) FROM ${sequenceTableSql}), 0) AS latest
+          `.pipe(Effect.flatMap(decodeUploadBounds), mapError);
+      if (latest <= checkpoint) return yield* Effect.succeedNone;
+      const entries = yield* sql`
+            SELECT e.*
+            FROM ${sequenceTableSql} s
+            JOIN ${entryTableSql} e ON e.id = s.entry_id
+            WHERE s.sequence > ${checkpoint} AND
+                  s.sequence <= ${latest} AND
+                  NOT EXISTS (
+                    SELECT 1 FROM ${sourcesTableSql} src WHERE src.entry_id = s.entry_id AND src.remote_id = ${remoteId}
+                  )
+          `.pipe(Effect.flatMap(decodeEntryRows), Effect.map(toOrderedEntries), mapError);
+      // Uploads can complete out of order; the checkpoint only moves forward.
+      const saveCheckpoint = sql`
+            INSERT INTO ${checkpointsTableSql} ${sql.insert({
+        remote_id: remoteId,
+        sequence: latest
+      })}
+            ON CONFLICT (remote_id) DO UPDATE SET sequence = excluded.sequence
+            WHERE ${checkpointsTableSql}.sequence < excluded.sequence
+          `.pipe(mapError);
+      return yield* Effect.uninterruptibleMask(restore => restore(Arr.isReadonlyArrayNonEmpty(entries) ? Effect.asSome(f(entries)) : Effect.succeedNone).pipe(Effect.tap(saveCheckpoint)));
     }, withTracerDisabled),
-    nextRemoteSequence: remoteId => sql`SELECT MAX(sequence) AS max FROM ${remotesTableSql} WHERE remote_id = ${remoteId}`.pipe(Effect.map(rows => {
-      const value = rows[0]?.max;
-      if (value === null || value === undefined) return 0;
-      return Number(value) + 1;
-    }), withTracerDisabled, Effect.mapError(cause => new EventJournal.EventJournalError({
+    writeRemoteAcks: Effect.fnUntraced(function* ({
+      acks,
+      firstSequence,
+      remoteId
+    }) {
+      if (acks.length === 0) return;
+      const next = yield* readCursor(remoteId);
+      yield* writeCursor(remoteId, EventJournal.advanceSequence(next, acks, firstSequence));
+    }, _ => sql.withTransaction(_), _ => withTracerDisabled(_), Effect.mapError(cause => new EventJournal.EventJournalError({
+      cause,
+      method: "writeRemoteAcks"
+    }))),
+    nextRemoteSequence: remoteId => readCursor(remoteId).pipe(withTracerDisabled, Effect.mapError(cause => new EventJournal.EventJournalError({
       cause,
       method: "nextRemoteSequence"
     }))),
     changes: PubSub.subscribe(pubsub),
     destroy: Effect.gen(function* () {
       yield* sql`DROP TABLE ${entryTableSql}`;
-      yield* sql`DROP TABLE ${remotesTableSql}`;
+      yield* sql`DROP TABLE ${sequenceTableSql}`;
+      yield* sql`DROP TABLE ${sourcesTableSql}`;
+      yield* sql`DROP TABLE ${checkpointsTableSql}`;
+      yield* sql`DROP TABLE ${remoteCursorsTableSql}`;
     }).pipe(withTracerDisabled, Effect.mapError(cause => new EventJournal.EventJournalError({
       cause,
       method: "destroy"
@@ -223,25 +470,11 @@ export const make = options => Effect.gen(function* () {
   });
 });
 /**
- * Provides `EventJournal` using the SQL-backed implementation created by
- * `make`.
- *
- * **When to use**
- *
- * Use when composing a Layer graph that should provide a persistent SQL-backed
- * `EventJournal` from an existing `SqlClient` service.
- *
- * **Details**
- *
- * The layer delegates to `make(options)`, so the same optional `entryTable` and
- * `remotesTable` settings are used and construction requires `SqlClient` and
- * may fail with `SqlError`.
+ * Provides a SQL-backed `EventJournal` from an existing `SqlClient` service.
  *
  * **Gotchas**
  *
- * Layer construction performs the same minimal `CREATE TABLE IF NOT EXISTS`
- * setup as `make`; manage indexes and schema migrations outside this layer when
- * your SQL schema needs more than the built-in tables.
+ * Creating or migrating the schema requires a writable connection.
  *
  * @see {@link make} for constructing the SQL-backed service directly
  * @see {@link EventJournal.layerMemory} for an in-memory `EventJournal` layer
@@ -272,6 +505,11 @@ const EntryRow = /*#__PURE__*/Schema.Struct({
   timestamp: SqlNatural
 });
 const EntryRowArray = /*#__PURE__*/Schema.Array(EntryRow);
+const entryColumns = /*#__PURE__*/Object.keys(EntryRow.fields).length;
+const decodeUploadBounds = /*#__PURE__*/Schema.decodeUnknownEffect(/*#__PURE__*/Schema.Tuple([/*#__PURE__*/Schema.Struct({
+  checkpoint: SqlNatural,
+  latest: SqlNatural
+})]));
 const EntryIdRowArray = /*#__PURE__*/Schema.Array(/*#__PURE__*/Schema.Struct({
   id: SqlEntryId
 }));
@@ -290,11 +528,21 @@ const toEntryRow = entry => ({
   payload: entry.payload,
   timestamp: EventJournal.entryIdMillis(entry.id)
 });
-const RemoteRow = /*#__PURE__*/Schema.Struct({
-  remote_id: EventJournal.RemoteId,
-  entry_id: EventJournal.EntryId,
-  sequence: Schema.Natural
-});
-const RemoteRowArray = /*#__PURE__*/Schema.Array(RemoteRow);
+const CursorRowArray = /*#__PURE__*/Schema.Array(/*#__PURE__*/Schema.Struct({
+  next_sequence: SqlNatural
+}));
 const withTracerDisabled = /*#__PURE__*/Effect.withTracerEnabled(false);
+// Statements bind at most this many parameters: Cloudflare Durable Object
+// SQLite's per-statement limit, the lowest of the supported databases.
+const maxParameters = 100;
+// Runs `f` in order over chunks of `values`, each binding `parametersPerValue`
+// parameters per value, and concatenates the rows.
+const forEachChunk = (values, parametersPerValue, f) => Effect.map(Effect.forEach(Arr.chunksOf(values, Math.floor(maxParameters / parametersPerValue)), f), rows => rows.flat());
+// Preserve callback errors without wrapping them as journal SQL errors.
+class CallbackFailure {
+  error;
+  constructor(error) {
+    this.error = error;
+  }
+}
 //# sourceMappingURL=SqlEventJournal.js.map