@livestore/common 0.4.0-dev.9 → 0.5.0-dev.0

package/src/leader-thread/RejectedPushError.ts

@@ -0,0 +1,119 @@
+/**
+ * Push validation errors returned by {@link LeaderSyncProcessor.push}.
+ *
+ * All errors share a common {@link RejectedPushErrorTypeId} so consumers can catch the
+ * family as a group via {@link isRejectedPushError} instead of matching individual tags.
+ * Recovery is the same in every case: the client should rebase and retry.
+ *
+ * @module
+ */
+import { Predicate, Schema } from '@livestore/utils/effect'
+
+import { EventSequenceNumber } from '../schema/mod.ts'
+
+export const RejectedPushErrorTypeId = '~@livestore/common/RejectedPushError' as const
+
+/**
+ * A pushed batch of events failed validation because its sequence numbers are not strictly increasing.
+ *
+ * @remarks
+ *
+ * This is a defensive check — callers are expected to construct monotonic event batches.
+ * The client should rebase and retry.
+ */
+export class NonMonotonicBatchError extends Schema.TaggedError<NonMonotonicBatchError>(
+  `${RejectedPushErrorTypeId}/NonMonotonicBatchError`,
+)('NonMonotonicBatchError', {
+  /** The sequence number that broke the monotonic invariant (i.e. the one that is >= the next). */
+  precedingSeqNum: EventSequenceNumber.Client.Composite,
+  /** The sequence number that was expected to be greater than `precedingSeqNum`. */
+  violatingSeqNum: EventSequenceNumber.Client.Composite,
+  /** The index in the batch where the violation occurred. */
+  violationIndex: Schema.Finite,
+  /** The session that produced the malformed batch. */
+  sessionId: Schema.String,
+}) {
+  readonly [RejectedPushErrorTypeId] = RejectedPushErrorTypeId
+
+  override get message(): string {
+    return `Pushed events' sequence numbers are not strictly increasing at index ${this.violationIndex} (session ${this.sessionId}): ${EventSequenceNumber.Client.toString(this.precedingSeqNum)} >= ${EventSequenceNumber.Client.toString(this.violatingSeqNum)}`
+  }
+}
+
+/** A pushed batch skips an event required to connect it to the leader's admitted push prefix. */
+export class NonContiguousBatchError extends Schema.TaggedError<NonContiguousBatchError>(
+  `${RejectedPushErrorTypeId}/NonContiguousBatchError`,
+)('NonContiguousBatchError', {
+  expectedSeqNum: EventSequenceNumber.Client.Composite,
+  providedSeqNum: EventSequenceNumber.Client.Composite,
+  expectedParentSeqNum: EventSequenceNumber.Client.Composite,
+  providedParentSeqNum: EventSequenceNumber.Client.Composite,
+  violationIndex: Schema.Finite,
+  sessionId: Schema.String,
+}) {
+  readonly [RejectedPushErrorTypeId] = RejectedPushErrorTypeId
+
+  override get message(): string {
+    return `Pushed events are not contiguous at index ${this.violationIndex} (session ${this.sessionId}): expected ${EventSequenceNumber.Client.toString(this.expectedSeqNum)} after ${EventSequenceNumber.Client.toString(this.expectedParentSeqNum)}, got ${EventSequenceNumber.Client.toString(this.providedSeqNum)} after ${EventSequenceNumber.Client.toString(this.providedParentSeqNum)}`
+  }
+}
+
+/**
+ * A pushed batch of events failed validation because its rebase generation is older than the leader's current rebase generation.
+ *
+ * @remarks
+ *
+ * This happens when events were enqueued before a backend-pull-triggered rebase incremented the generation.
+ */
+export class StaleRebaseGenerationError extends Schema.TaggedError<StaleRebaseGenerationError>(
+  `${RejectedPushErrorTypeId}/StaleRebaseGenerationError`,
+)('StaleRebaseGenerationError', {
+  /** The leader's current rebase generation. */
+  currentRebaseGeneration: Schema.Finite,
+  /** The rebase generation carried by the dropped events. */
+  providedRebaseGeneration: Schema.Finite,
+  /** The session that produced the stale batch. */
+  sessionId: Schema.String,
+}) {
+  readonly [RejectedPushErrorTypeId] = RejectedPushErrorTypeId
+
+  override get message(): string {
+    return `Pushed events have stale rebase generation (session ${this.sessionId}): expected >= ${this.currentRebaseGeneration}, got ${this.providedRebaseGeneration}`
+  }
+}
+
+/**
+ * A pushed batch of events was rejected because the leader's push head has already advanced
+ * past the batch's first event.
+ *
+ * @remarks
+ *
+ * This occurs when another client session (or a backend pull) has pushed events that the current
+ * session hasn't seen yet.
+ */
+export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>(
+  `${RejectedPushErrorTypeId}/LeaderAheadError`,
+)('LeaderAheadError', {
+  minimumExpectedNum: EventSequenceNumber.Client.Composite,
+  providedNum: EventSequenceNumber.Client.Composite,
+  /** The session that produced the stale batch. */
+  sessionId: Schema.String,
+}) {
+  readonly [RejectedPushErrorTypeId] = RejectedPushErrorTypeId
+
+  override get message(): string {
+    return `Leader push head is ahead of batch (session ${this.sessionId}): expected > ${EventSequenceNumber.Client.toString(this.minimumExpectedNum)}, got ${EventSequenceNumber.Client.toString(this.providedNum)}`
+  }
+}
+
+export const RejectedPushError = Schema.Union([
+  LeaderAheadError,
+  NonContiguousBatchError,
+  NonMonotonicBatchError,
+  StaleRebaseGenerationError,
+])
+
+export type RejectedPushError = typeof RejectedPushError.Type
+
+export const isRejectedPushError = (u: unknown): u is RejectedPushError =>
+  Predicate.hasProperty(u, RejectedPushErrorTypeId)

package/src/leader-thread/connection.ts

@@ -62,7 +62,7 @@ export const configureConnection = (sqliteDb: SqliteDb, { foreignKeys, lockingMo
     -- disable WAL until we have it working properly
     -- PRAGMA journal_mode=WAL;
     PRAGMA page_size=8192;
-    PRAGMA foreign_keys=${foreignKeys ? 'ON' : 'OFF'};
+    PRAGMA foreign_keys=${foreignKeys === true ? 'ON' : 'OFF'};
     ${lockingMode === undefined ? '' : sql`PRAGMA locking_mode=${lockingMode};`}
   `,
     {},
@@ -83,14 +83,18 @@ export const execSql = (sqliteDb: SqliteDb, sql: string, bind: BindValues) => {
   )
 }
 
-// const selectSqlPrepared = <T>(stmt: PreparedStatement, bind: BindValues) => {
-//   const bindValues = prepareBindValues(bind, stmt.sql)
-//   return Effect.try({
-//     try: () => stmt.select<T>(bindValues),
-//     catch: (cause) =>
-//       new SqliteError({ cause, query: { bindValues, sql: stmt.sql }, code: (cause as WaSqlite.SQLiteError).code }),
-//   })
-// }
+export const selectSql = <T>(sqliteDb: SqliteDb, sql: string, bind: BindValues) => {
+  const bindValues = prepareBindValues(bind, sql)
+  return Effect.try({
+    try: () => sqliteDb.select<T>(sql, bindValues),
+    catch: (cause) =>
+      new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
+  }).pipe(
+    Effect.withSpan(`@livestore/common:selectSql`, {
+      attributes: { 'span.label': sql, sql, bindValueKeys: Object.keys(bindValues) },
+    }),
+  )
+}
 
 // TODO actually use prepared statements
 export const execSqlPrepared = (sqliteDb: SqliteDb, sql: string, bindValues: PreparedBindValues) => {

package/src/leader-thread/eventlog.ts

@@ -2,22 +2,22 @@ import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
 import { Effect, Option, Schema } from '@livestore/utils/effect'
 
 import type { SqliteDb } from '../adapter-types.ts'
-import * as EventSequenceNumber from '../schema/EventSequenceNumber.ts'
-import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
+import { migrateTable } from '../schema-management/migrations.ts'
+import * as EventSequenceNumber from '../schema/EventSequenceNumber/mod.ts'
+import * as LiveStoreEvent from '../schema/LiveStoreEvent/mod.ts'
 import {
   EVENTLOG_META_TABLE,
   eventlogMetaTable,
   eventlogSystemTables,
   SYNC_STATUS_TABLE,
-  sessionChangesetMetaTable,
-} from '../schema/state/sqlite/system-tables.ts'
-import { migrateTable } from '../schema-management/migrations.ts'
+} from '../schema/state/sqlite/system-tables/eventlog-tables.ts'
+import { sessionChangesetMetaTable } from '../schema/state/sqlite/system-tables/state-tables.ts'
 import { insertRow, updateRows } from '../sql-queries/sql-queries.ts'
 import type { PreparedBindValues } from '../util.ts'
 import { sql } from '../util.ts'
 import { execSql } from './connection.ts'
-import type { InitialSyncInfo } from './types.ts'
-import { LeaderThreadCtx } from './types.ts'
+import type { InitialSyncInfo, StreamEventsOptions } from './types.ts'
+import { LeaderThreadCtx, STREAM_EVENTS_BATCH_SIZE_DEFAULT } from './types.ts'
 
 export const initEventlogDb = (dbEventlog: SqliteDb) =>
   Effect.gen(function* () {
@@ -34,7 +34,7 @@ export const initEventlogDb = (dbEventlog: SqliteDb) =>
     yield* execSql(
       dbEventlog,
       sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
-          SELECT ${EventSequenceNumber.ROOT.global}
+          SELECT ${EventSequenceNumber.Client.ROOT.global}
           WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
       {},
     )
@@ -51,22 +51,23 @@ export const getEventsSince = ({
 }: {
   dbEventlog: SqliteDb
   dbState: SqliteDb
-  since: EventSequenceNumber.EventSequenceNumber
-}): ReadonlyArray<LiveStoreEvent.EncodedWithMeta> => {
+  since: EventSequenceNumber.Client.Composite
+}): ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta> => {
   const pendingEvents = dbEventlog.select(eventlogMetaTable.where('seqNumGlobal', '>=', since.global))
 
   const sessionChangesetRowsDecoded = dbState.select(
     sessionChangesetMetaTable.where('seqNumGlobal', '>=', since.global),
   )
 
+  // Create a Map for O(1) lookup instead of O(n) find
+  const sessionChangesetMap = new Map(
+    sessionChangesetRowsDecoded.map((row) => [`${row.seqNumGlobal}:${row.seqNumClient}`, row]),
+  )
+
   return pendingEvents
     .map((eventlogEvent) => {
-      const sessionChangeset = sessionChangesetRowsDecoded.find(
-        (readModelEvent) =>
-          readModelEvent.seqNumGlobal === eventlogEvent.seqNumGlobal &&
-          readModelEvent.seqNumClient === eventlogEvent.seqNumClient,
-      )
-      return LiveStoreEvent.EncodedWithMeta.make({
+      const sessionChangeset = sessionChangesetMap.get(`${eventlogEvent.seqNumGlobal}:${eventlogEvent.seqNumClient}`)
+      return LiveStoreEvent.Client.EncodedWithMeta.make({
         name: eventlogEvent.name,
         args: eventlogEvent.argsJson,
         seqNum: {
@@ -83,7 +84,7 @@ export const getEventsSince = ({
         sessionId: eventlogEvent.sessionId,
         meta: {
           sessionChangeset:
-            sessionChangeset && sessionChangeset.changeset !== null
+            sessionChangeset !== undefined && sessionChangeset.changeset !== null
               ? {
                   _tag: 'sessionChangeset' as const,
                   data: sessionChangeset.changeset,
@@ -96,35 +97,111 @@ export const getEventsSince = ({
         },
       })
     })
-    .filter((_) => EventSequenceNumber.compare(_.seqNum, since) > 0)
-    .sort((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
+    .filter((_) => EventSequenceNumber.Client.compare(_.seqNum, since) > 0)
+    .toSorted((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
 }
 
-export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.EventSequenceNumber => {
+export const getEventsFromEventlog = ({
+  dbEventlog,
+  options,
+}: {
+  dbEventlog: SqliteDb
+  options: StreamEventsOptions
+}): Effect.Effect<ReadonlyArray<LiveStoreEvent.Client.Encoded>> =>
+  Effect.gen(function* () {
+    const since = options.since ?? EventSequenceNumber.Client.ROOT
+    const batchSize = options.batchSize ?? STREAM_EVENTS_BATCH_SIZE_DEFAULT
+
+    const makeQuery = () => {
+      let query = eventlogMetaTable.where('seqNumGlobal', '>', since.global)
+
+      if (options.until !== undefined) {
+        query = query.where('seqNumGlobal', '<=', options.until.global)
+      }
+
+      if (options.filter !== undefined && options.filter.length > 0) {
+        query = query.where({ name: { op: 'IN', value: options.filter } })
+      }
+
+      if (options.clientIds !== undefined && options.clientIds.length > 0) {
+        query = query.where({ clientId: { op: 'IN', value: options.clientIds } })
+      }
+
+      if (options.sessionIds !== undefined && options.sessionIds.length > 0) {
+        query = query.where({ sessionId: { op: 'IN', value: options.sessionIds } })
+      }
+
+      if (options.includeClientOnly !== true) {
+        query = query.where('seqNumClient', '<=', EventSequenceNumber.Client.DEFAULT)
+      }
+
+      return query
+        .orderBy([
+          { col: 'seqNumGlobal', direction: 'asc' },
+          { col: 'seqNumClient', direction: 'asc' },
+        ])
+        .limit(batchSize)
+    }
+
+    const eventlogEvents = yield* Effect.sync(() => dbEventlog.select(makeQuery()))
+
+    if (eventlogEvents.length === 0) {
+      return []
+    }
+
+    const spanAttributes = {
+      'livestore.eventLog.since': since.global,
+      'livestore.eventLog.until': options.until?.global,
+    }
+
+    return yield* Effect.sync(() => {
+      const encodedEvents = eventlogEvents.map((eventlogEvent) => {
+        return LiveStoreEvent.Client.Encoded.make({
+          name: eventlogEvent.name,
+          args: eventlogEvent.argsJson,
+          seqNum: {
+            global: eventlogEvent.seqNumGlobal,
+            client: eventlogEvent.seqNumClient,
+            rebaseGeneration: eventlogEvent.seqNumRebaseGeneration,
+          },
+          parentSeqNum: {
+            global: eventlogEvent.parentSeqNumGlobal,
+            client: eventlogEvent.parentSeqNumClient,
+            rebaseGeneration: eventlogEvent.parentSeqNumRebaseGeneration,
+          },
+          clientId: eventlogEvent.clientId,
+          sessionId: eventlogEvent.sessionId,
+        })
+      })
+
+      return encodedEvents
+    }).pipe(Effect.withSpan('@livestore/common:eventlog:getEventsFromEventlog', { attributes: spanAttributes }))
+  })
+
+export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.Client.Composite => {
   const res = dbEventlog.select<{
-    seqNumGlobal: EventSequenceNumber.GlobalEventSequenceNumber
-    seqNumClient: EventSequenceNumber.ClientEventSequenceNumber
+    seqNumGlobal: EventSequenceNumber.Global.Type
+    seqNumClient: EventSequenceNumber.Client.Type
     seqNumRebaseGeneration: number
   }>(
     sql`select seqNumGlobal, seqNumClient, seqNumRebaseGeneration from ${EVENTLOG_META_TABLE} order by seqNumGlobal DESC, seqNumClient DESC limit 1`,
   )[0]
 
-  return res
+  return res !== undefined
     ? { global: res.seqNumGlobal, client: res.seqNumClient, rebaseGeneration: res.seqNumRebaseGeneration }
-    : EventSequenceNumber.ROOT
+    : EventSequenceNumber.Client.ROOT
 }
 
-export const getBackendHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.GlobalEventSequenceNumber =>
-  dbEventlog.select<{ head: EventSequenceNumber.GlobalEventSequenceNumber }>(
-    sql`select head from ${SYNC_STATUS_TABLE}`,
-  )[0]?.head ?? EventSequenceNumber.ROOT.global
+export const getBackendHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.Global.Type =>
+  dbEventlog.select<{ head: EventSequenceNumber.Global.Type }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ??
+  EventSequenceNumber.Client.ROOT.global
 
 // TODO use prepared statements
-export const updateBackendHead = (dbEventlog: SqliteDb, head: EventSequenceNumber.EventSequenceNumber) =>
+export const updateBackendHead = (dbEventlog: SqliteDb, head: EventSequenceNumber.Client.Composite) =>
   dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
 
 export const getBackendIdFromDb = (dbEventlog: SqliteDb): Option.Option<string> =>
-  Option.fromNullable(
+  Option.fromNullishOr(
     dbEventlog.select<{ backendId: string | null }>(sql`select backendId from ${SYNC_STATUS_TABLE}`)[0]?.backendId,
   )
 
@@ -132,7 +209,7 @@ export const updateBackendId = (dbEventlog: SqliteDb, backendId: string) =>
   dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET backendId = '${backendId}'`)
 
 export const insertIntoEventlog = (
-  eventEncoded: LiveStoreEvent.EncodedWithMeta,
+  eventEncoded: LiveStoreEvent.Client.EncodedWithMeta,
   dbEventlog: SqliteDb,
   eventDefSchemaHash: number,
   clientId: string,
@@ -140,7 +217,7 @@ export const insertIntoEventlog = (
 ) =>
   Effect.gen(function* () {
     // Check history consistency during LS_DEV
-    if (LS_DEV && eventEncoded.parentSeqNum.global !== EventSequenceNumber.ROOT.global) {
+    if (LS_DEV === true && eventEncoded.parentSeqNum.global !== EventSequenceNumber.Client.ROOT.global) {
       const parentEventExists =
         dbEventlog.select<{ count: number }>(
           `SELECT COUNT(*) as count FROM ${EVENTLOG_META_TABLE} WHERE seqNumGlobal = ? AND seqNumClient = ?`,
@@ -180,7 +257,7 @@ export const insertIntoEventlog = (
     dbEventlog.debug.head = eventEncoded.seqNum
   })
 
-export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>) =>
+export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>) =>
   Effect.gen(function* () {
     const { dbEventlog } = yield* LeaderThreadCtx
 
@@ -200,25 +277,21 @@ export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.EncodedWi
     }
   })
 
-export const getSyncBackendCursorInfo = ({
-  remoteHead,
-}: {
-  remoteHead: EventSequenceNumber.GlobalEventSequenceNumber
-}) =>
+export const getSyncBackendCursorInfo = ({ remoteHead }: { remoteHead: EventSequenceNumber.Global.Type }) =>
   Effect.gen(function* () {
     const { dbEventlog } = yield* LeaderThreadCtx
 
-    if (remoteHead === EventSequenceNumber.ROOT.global) return Option.none()
+    if (remoteHead === EventSequenceNumber.Client.ROOT.global) return Option.none()
 
     const EventlogQuerySchema = Schema.Struct({
-      syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
+      syncMetadataJson: Schema.fromJsonString(Schema.toCodecJson(Schema.Option(Schema.Json))),
     }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
 
     const syncMetadataOption = yield* Effect.sync(() =>
       dbEventlog.select<{ syncMetadataJson: string }>(
         sql`SELECT syncMetadataJson FROM ${EVENTLOG_META_TABLE} WHERE seqNumGlobal = ${remoteHead} ORDER BY seqNumClient ASC LIMIT 1`,
       ),
-    ).pipe(Effect.andThen(Schema.decode(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
+    ).pipe(Effect.andThen(Schema.decodeEffect(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
 
     return Option.some({
       eventSequenceNumber: remoteHead,