@livestore/common 0.4.0-dev.7 → 0.4.0-dev.9

package/src/leader-thread/LeaderSyncProcessor.ts

@@ -4,6 +4,7 @@ import {
   BucketQueue,
   Cause,
   Deferred,
+  Duration,
   Effect,
   Exit,
   FiberHandle,
@@ -13,6 +14,7 @@ import {
   pipe,
   Queue,
   ReadonlyArray,
+  Schedule,
   Stream,
   Subscribable,
   SubscriptionRef,
@@ -102,7 +104,10 @@ export const makeLeaderSyncProcessor = ({
      */
     backendPushBatchSize?: number
   }
-  /** * Whether the sync backend should reactively pull new events from the sync backend */
+  /**
+   * Whether the sync backend should reactively pull new events from the sync backend
+   * When `false`, the sync processor will only do an initial pull
+   */
   livePull: boolean
   testing: {
     delays?: {
@@ -281,7 +286,7 @@ export const makeLeaderSyncProcessor = ({
         },
       }).pipe(Effect.catchAllCause(maybeShutdownOnError), Effect.forkScoped)
 
-      const backendPushingFiberHandle = yield* FiberHandle.make<undefined, never>()
+      const backendPushingFiberHandle = yield* FiberHandle.make<void, never>()
       const backendPushingEffect = backgroundBackendPushing({
         syncBackendPushQueue,
         otelSpan,
@@ -686,6 +691,13 @@ const backgroundBackendPulling = ({
             mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
           })
 
+          // Ensure push fiber is active after advance by restarting with current pending (non-client) events
+          const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
+            const { eventDef } = getEventDef(schema, event.name)
+            return eventDef.options.clientOnly === false
+          })
+          yield* restartBackendPushing(globalPendingEvents)
+
           yield* connectedClientSessionPullQueues.offer({
             payload: SyncState.payloadFromMergeResult(mergeResult),
             leaderHead: mergeResult.newSyncState.localHead,
@@ -756,6 +768,9 @@ const backgroundBackendPulling = ({
       Stream.runDrain,
       Effect.interruptible,
     )
+
+    // Should only ever happen when livePull is false
+    yield* Effect.logDebug('backend-pulling finished', { livePull })
   }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'))
 
 const backgroundBackendPushing = ({
@@ -789,25 +804,53 @@ const backgroundBackendPushing = ({
         batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
       })
 
-      // TODO handle push errors (should only happen during concurrent pull+push)
-      const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
+      // Push with declarative retry/backoff using Effect schedules
+      // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
+      // - Delay clamped at 30s (continues retrying at 30s)
+      // - Resets automatically after successful push
+      // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
+
+      // Only retry for transient UnexpectedError cases
+      const isRetryable = (err: InvalidPushError | IsOfflineError) =>
+        err._tag === 'InvalidPushError' && err.cause._tag === 'LiveStore.UnexpectedError'
+
+      // Input: InvalidPushError | IsOfflineError, Output: Duration
+      const retrySchedule: Schedule.Schedule<Duration.DurationInput, InvalidPushError | IsOfflineError> =
+        Schedule.exponential(Duration.seconds(1)).pipe(
+          Schedule.andThenEither(Schedule.spaced(Duration.seconds(30))), // clamp at 30 second intervals
+          Schedule.compose(Schedule.elapsed),
+          Schedule.whileInput(isRetryable),
+        )
+
+      yield* Effect.gen(function* () {
+        const iteration = yield* Schedule.CurrentIterationMetadata
 
-      if (pushResult._tag === 'Left') {
-        if (
-          pushResult.left._tag === 'InvalidPushError' &&
-          // server ahead errors are gracefully handled
-          pushResult.left.cause._tag !== 'ServerAheadError'
-        ) {
-          return yield* pushResult.left
+        const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
+
+        const retries = iteration.recurrence
+        if (retries > 0 && pushResult._tag === 'Right') {
+          otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length })
         }
 
-        if (LS_DEV) {
-          yield* Effect.logDebug('handled backend-push-error', { error: pushResult.left.toString() })
+        if (pushResult._tag === 'Left') {
+          otelSpan?.addEvent('backend-push-error', {
+            error: pushResult.left.toString(),
+            retries,
+            batchSize: queueItems.length,
+          })
+          const error = pushResult.left
+          if (
+            error._tag === 'IsOfflineError' ||
+            (error._tag === 'InvalidPushError' && error.cause._tag === 'ServerAheadError')
+          ) {
+            // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
+            yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error })
+            return yield* Effect.never
+          }
+
+          return yield* error
         }
-        otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() })
-        // wait for interrupt caused by background pulling which will then restart pushing
-        return yield* Effect.never
-      }
+      }).pipe(Effect.retry(retrySchedule))
     }
   }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'))
 

package/src/leader-thread/eventlog.ts

@@ -149,7 +149,7 @@ export const insertIntoEventlog = (
 
       if (parentEventExists === false) {
         shouldNeverHappen(
-          `Parent mutation ${eventEncoded.parentSeqNum.global},${eventEncoded.parentSeqNum.client} does not exist`,
+          `Parent event ${eventEncoded.parentSeqNum.global},${eventEncoded.parentSeqNum.client} does not exist in eventlog`,
         )
       }
     }

package/src/leader-thread/types.ts

@@ -150,6 +150,7 @@ export interface LeaderSyncProcessor {
     options?: {
       /**
        * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
+       * `true` doesn't mean the events have been pushed to the sync backend.
        * @default false
        */
       waitForProcessing?: boolean

package/src/schema/state/sqlite/client-document-def.ts

@@ -1,6 +1,6 @@
 import { shouldNeverHappen } from '@livestore/utils'
 import type { Option, Types } from '@livestore/utils/effect'
-import { Schema, SchemaAST } from '@livestore/utils/effect'
+import { Schema } from '@livestore/utils/effect'
 
 import { SessionIdSymbol } from '../../../adapter-types.ts'
 import { sql } from '../../../util.ts'
@@ -273,7 +273,7 @@ export const deriveEventAndMaterializer = ({
     }
 
     // Override the full value if it's not an object or no partial set is allowed
-    const schemaProps = SchemaAST.getPropertySignatures(valueSchema.ast)
+    const schemaProps = Schema.getResolvedPropertySignatures(valueSchema)
     if (schemaProps.length === 0 || partialSet === false) {
       const valueColJsonSchema = Schema.parseJson(valueSchema)
       const encodedInsertValue = Schema.encodeSyncDebug(valueColJsonSchema)(value ?? defaultValue)

package/src/schema/state/sqlite/column-annotations.ts

@@ -1,5 +1,5 @@
 import type { Schema } from '@livestore/utils/effect'
-import { dual } from '@livestore/utils/effect'
+import { dual, Option, SchemaAST } from '@livestore/utils/effect'
 import type { SqliteDsl } from './db-schema/mod.ts'
 
 export const PrimaryKeyId = Symbol.for('livestore/state/sqlite/annotations/primary-key')
@@ -32,7 +32,7 @@ Here are the knobs you can turn per-column when you CREATE TABLE (or ALTER TABLE
  * Adds a primary key annotation to a schema.
  */
 export const withPrimaryKey = <T extends Schema.Schema.All>(schema: T) =>
-  schema.annotations({ [PrimaryKeyId]: true }) as T
+  applyAnnotations(schema, { [PrimaryKeyId]: true })
 
 /**
  * Adds a column type annotation to a schema.
@@ -43,19 +43,19 @@ export const withColumnType: {
   <T extends Schema.Schema.All>(schema: T, type: SqliteDsl.FieldColumnType): T
 } = dual(2, <T extends Schema.Schema.All>(schema: T, type: SqliteDsl.FieldColumnType) => {
   validateSchemaColumnTypeCompatibility(schema, type)
-  return schema.annotations({ [ColumnType]: type }) as T
+  return applyAnnotations(schema, { [ColumnType]: type })
 })
 
 /**
  * Adds an auto-increment annotation to a schema.
  */
 export const withAutoIncrement = <T extends Schema.Schema.All>(schema: T) =>
-  schema.annotations({ [AutoIncrement]: true }) as T
+  applyAnnotations(schema, { [AutoIncrement]: true })
 
 /**
  * Adds a unique constraint annotation to a schema.
  */
-export const withUnique = <T extends Schema.Schema.All>(schema: T) => schema.annotations({ [Unique]: true }) as T
+export const withUnique = <T extends Schema.Schema.All>(schema: T) => applyAnnotations(schema, { [Unique]: true })
 
 /**
  * Adds a default value annotation to a schema.
@@ -64,7 +64,7 @@ export const withDefault: {
   // TODO make type safe
   <T extends Schema.Schema.All>(schema: T, value: unknown): T
   (value: unknown): <T extends Schema.Schema.All>(schema: T) => T
-} = dual(2, <T extends Schema.Schema.All>(schema: T, value: unknown) => schema.annotations({ [Default]: value }) as T)
+} = dual(2, <T extends Schema.Schema.All>(schema: T, value: unknown) => applyAnnotations(schema, { [Default]: value }))
 
 /**
  * Validates that a schema is compatible with the specified SQLite column type
@@ -75,3 +75,13 @@ const validateSchemaColumnTypeCompatibility = (
 ): void => {
   // TODO actually implement this
 }
+
+const applyAnnotations = <T extends Schema.Schema.All>(schema: T, overrides: Record<PropertyKey, unknown>): T => {
+  const identifier = SchemaAST.getIdentifierAnnotation(schema.ast)
+  const shouldPreserveIdentifier = Option.isSome(identifier) && !(SchemaAST.IdentifierAnnotationId in overrides)
+  const annotations: Record<PropertyKey, unknown> = shouldPreserveIdentifier
+    ? { ...overrides, [SchemaAST.IdentifierAnnotationId]: identifier.value }
+    : overrides
+
+  return schema.annotations(annotations) as T
+}

package/src/schema/state/sqlite/query-builder/impl.test.ts

@@ -683,6 +683,127 @@ describe('query builder', () => {
       expect(pattern1).toEqual(pattern2)
     })
   })
+
+  describe('schema transforms', () => {
+    const Flat = Schema.Struct({
+      id: Schema.String.pipe(State.SQLite.withPrimaryKey),
+      contactFirstName: Schema.String,
+      contactLastName: Schema.String,
+      contactEmail: Schema.String.pipe(State.SQLite.withUnique),
+    })
+
+    const Nested = Schema.transform(
+      Flat,
+      Schema.Struct({
+        id: Schema.String,
+        contact: Schema.Struct({
+          firstName: Schema.String,
+          lastName: Schema.String,
+          email: Schema.String,
+        }),
+      }),
+      {
+        decode: ({ id, contactFirstName, contactLastName, contactEmail }) => ({
+          id,
+          contact: {
+            firstName: contactFirstName,
+            lastName: contactLastName,
+            email: contactEmail,
+          },
+        }),
+        encode: ({ id, contact }) => ({
+          id,
+          contactFirstName: contact.firstName,
+          contactLastName: contact.lastName,
+          contactEmail: contact.email,
+        }),
+      },
+    )
+
+    const makeContactsTable = () =>
+      State.SQLite.table({
+        name: 'contacts',
+        schema: Nested,
+        // schema: Flat,
+      })
+
+    it('exposes flattened insert type while schema type is nested', () => {
+      const contactsTable = makeContactsTable()
+
+      type InsertInput = Parameters<(typeof contactsTable)['insert']>[0]
+      type NestedType = Schema.Schema.Type<typeof Nested>
+
+      type Assert<T extends true> = T
+
+      type InsertKeys = keyof InsertInput
+      type NestedKeys = keyof NestedType
+
+      type _InsertHasFlattenedColumns = Assert<
+        'contactFirstName' extends InsertKeys
+          ? 'contactLastName' extends InsertKeys
+            ? 'contactEmail' extends InsertKeys
+              ? true
+              : false
+            : false
+          : false
+      >
+
+      type _InsertDoesNotExposeNested = Assert<Extract<'contact', InsertKeys> extends never ? true : false>
+
+      type _SchemaTypeIsNested = Assert<'contact' extends NestedKeys ? true : false>
+
+      void contactsTable
+    })
+
+    it('fails to encode nested inserts because flat columns are required', () => {
+      const contactsTable = makeContactsTable()
+
+      expect(
+        contactsTable
+          // TODO in the future we should use decoded types here instead of encoded
+          .insert({
+            id: 'person-1',
+            contactFirstName: 'Ada',
+            contactLastName: 'Lovelace',
+            contactEmail: 'ada@example.com',
+          })
+          .asSql(),
+      ).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "person-1",
+            "Ada",
+            "Lovelace",
+            "ada@example.com",
+          ],
+          "query": "INSERT INTO 'contacts' (id, contactFirstName, contactLastName, contactEmail) VALUES (?, ?, ?, ?)",
+          "usedTables": Set {
+            "contacts",
+          },
+        }
+      `)
+    })
+
+    it('fails to encode nested inserts because flat columns are required', () => {
+      const contactsTable = makeContactsTable()
+
+      expect(() =>
+        contactsTable
+          .insert({
+            id: 'person-1',
+            // @ts-expect-error
+            contact: {
+              firstName: 'Ada',
+              lastName: 'Lovelace',
+              email: 'ada@example.com',
+            },
+          })
+          .asSql(),
+      ).toThrowErrorMatchingInlineSnapshot(`
+        [ParseError: contacts\n└─ ["contactFirstName"]\n   └─ is missing]
+      `)
+    })
+  })
 })
 
 // TODO nested queries

package/src/schema/state/sqlite/table-def.test.ts

@@ -178,6 +178,59 @@ describe('table function overloads', () => {
     expect(userTable.sqliteDef.columns.age.columnType).toBe('integer')
   })
 
+  it('should support schemas that transform flat columns into nested types', () => {
+    const Flat = Schema.Struct({
+      id: Schema.String.pipe(State.SQLite.withPrimaryKey),
+      contactFirstName: Schema.String,
+      contactLastName: Schema.String,
+      contactEmail: Schema.String.pipe(State.SQLite.withUnique),
+    })
+
+    const Nested = Schema.transform(
+      Flat,
+      Schema.Struct({
+        id: Schema.String,
+        contact: Schema.Struct({
+          firstName: Schema.String,
+          lastName: Schema.String,
+          email: Schema.String,
+        }),
+      }),
+      {
+        decode: ({ id, contactFirstName, contactLastName, contactEmail }) => ({
+          id,
+          contact: {
+            firstName: contactFirstName,
+            lastName: contactLastName,
+            email: contactEmail,
+          },
+        }),
+        encode: ({ id, contact }) => ({
+          id,
+          contactFirstName: contact.firstName,
+          contactLastName: contact.lastName,
+          contactEmail: contact.email,
+        }),
+      },
+    )
+
+    const contactsTable = State.SQLite.table({
+      name: 'contacts',
+      schema: Nested,
+    })
+
+    const columns = contactsTable.sqliteDef.columns
+
+    expect(Object.keys(columns)).toEqual(['id', 'contactFirstName', 'contactLastName', 'contactEmail'])
+    expect(columns.id.primaryKey).toBe(true)
+    expect(columns.contactEmail.columnType).toBe('text')
+    expect(contactsTable.sqliteDef.indexes).toContainEqual({
+      name: 'idx_contacts_contactEmail_unique',
+      columns: ['contactEmail'],
+      isUnique: true,
+    })
+  })
+
   it('should extract table name from Schema.Class identifier', () => {
     class TodoItem extends Schema.Class<TodoItem>('TodoItem')({
       id: Schema.String,

package/src/schema/state/sqlite/table-def.ts

@@ -1,5 +1,5 @@
 import { type Nullable, shouldNeverHappen } from '@livestore/utils'
-import { Option, type Schema, SchemaAST, type Types } from '@livestore/utils/effect'
+import { Option, Schema, SchemaAST, type Types } from '@livestore/utils/effect'
 
 import { getColumnDefForSchema, schemaFieldsToColumns } from './column-def.ts'
 import { SqliteDsl } from './db-schema/mod.ts'
@@ -221,7 +221,7 @@ export function table<
     ) as SqliteDsl.Columns
     additionalIndexes = []
   } else if ('schema' in args) {
-    const result = schemaFieldsToColumns(SchemaAST.getPropertySignatures(args.schema.ast))
+    const result = schemaFieldsToColumns(Schema.getResolvedPropertySignatures(args.schema))
     columns = result.columns
 
     // We'll set tableName first, then use it for index names
@@ -381,12 +381,13 @@ export declare namespace SchemaToColumns {
   export type ColumnDefForType<TEncoded, TType> = SqliteDsl.ColumnDefinition<TEncoded, TType>
 
   // Create columns type from schema Type and Encoded
-  export type FromTypes<TType, TEncoded> = TType extends Record<string, any>
-    ? TEncoded extends Record<string, any>
-      ? {
-          [K in keyof TType & keyof TEncoded]: ColumnDefForType<TEncoded[K], TType[K]>
-        }
-      : SqliteDsl.Columns
+  export type FromTypes<TType, TEncoded> = TEncoded extends Record<string, any>
+    ? {
+        [K in keyof TEncoded]-?: ColumnDefForType<
+          TEncoded[K],
+          TType extends Record<string, any> ? (K extends keyof TType ? TType[K] : TEncoded[K]) : TEncoded[K]
+        >
+      }
     : SqliteDsl.Columns
 }
 

package/src/sync/ClientSessionSyncProcessor.ts

@@ -94,7 +94,7 @@ export const makeClientSessionSyncProcessor = ({
     }),
   }
 
-  /** Only used for debugging / observability, it's not relied upon for correctness of the sync processor. */
+  /** Only used for debugging / observability / testing, it's not relied upon for correctness of the sync processor. */
   const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
   const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) =>
     getEventDef(schema, eventEncoded.name).eventDef.options.clientOnly
@@ -240,7 +240,6 @@ export const makeClientSessionSyncProcessor = ({
           }
 
           syncStateRef.current = mergeResult.newSyncState
-          yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
 
           if (mergeResult._tag === 'rebase') {
             span.addEvent('merge:pull:rebase', {
@@ -298,7 +297,11 @@ export const makeClientSessionSyncProcessor = ({
             debugInfo.advanceCount++
           }
 
-          if (mergeResult.newEvents.length === 0) return
+          if (mergeResult.newEvents.length === 0) {
+            // If there are no new events, we need to update the sync state as well
+            yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
+            return
+          }
 
           const writeTables = new Set<string>()
           for (const event of mergeResult.newEvents) {
@@ -321,6 +324,9 @@ export const makeClientSessionSyncProcessor = ({
           }
 
           refreshTables(writeTables)
+
+          // We're only triggering the sync state update after all events have been materialized
+          yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
         }).pipe(
           Effect.tapCauseLogPretty,
           Effect.catchAllCause((cause) => clientSession.shutdown(Exit.failCause(cause))),

package/src/sync/mock-sync-backend.ts

@@ -1,7 +1,8 @@
 import type { Schema, Scope } from '@livestore/utils/effect'
 import { Effect, Mailbox, Option, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
-import type { UnexpectedError } from '../errors.ts'
+import { UnexpectedError } from '../errors.ts'
 import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
+import { InvalidPushError } from './errors.ts'
 import * as SyncBackend from './sync-backend.ts'
 import { validatePushPayload } from './validate-push-payload.ts'
 
@@ -11,41 +12,118 @@ export interface MockSyncBackend {
   disconnect: Effect.Effect<void>
   makeSyncBackend: Effect.Effect<SyncBackend.SyncBackend, UnexpectedError, Scope.Scope>
   advance: (...batch: LiveStoreEvent.AnyEncodedGlobal[]) => Effect.Effect<void>
+  /** Fail the next N push calls with an InvalidPushError (or custom error) */
+  failNextPushes: (
+    count: number,
+    error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
+  ) => Effect.Effect<void>
 }
 
-export const makeMockSyncBackend: Effect.Effect<MockSyncBackend, UnexpectedError, Scope.Scope> = Effect.gen(
-  function* () {
+export interface MockSyncBackendOptions {
+  /** Chunk size for non-live pulls; defaults to 100 */
+  nonLiveChunkSize?: number
+  /** Initial connected state; defaults to false */
+  startConnected?: boolean
+  // TODO add a "flaky" mode to simulate transient network / server failures for pull/push
+}
+
+export const makeMockSyncBackend = (
+  options?: MockSyncBackendOptions,
+): Effect.Effect<MockSyncBackend, UnexpectedError, Scope.Scope> =>
+  Effect.gen(function* () {
     const syncEventSequenceNumberRef = { current: EventSequenceNumber.ROOT.global }
     const syncPullQueue = yield* Queue.unbounded<LiveStoreEvent.AnyEncodedGlobal>()
     const pushedEventsQueue = yield* Mailbox.make<LiveStoreEvent.AnyEncodedGlobal>()
-    const syncIsConnectedRef = yield* SubscriptionRef.make(true)
+    const syncIsConnectedRef = yield* SubscriptionRef.make(options?.startConnected ?? false)
+    const allEventsRef: { current: LiveStoreEvent.AnyEncodedGlobal[] } = { current: [] }
 
     const span = yield* Effect.currentSpan.pipe(Effect.orDie)
 
     const semaphore = yield* Effect.makeSemaphore(1)
 
+    // TODO improve the API and implementation of simulating errors
+    const failCounterRef = yield* SubscriptionRef.make(0)
+    const failEffectRef = yield* SubscriptionRef.make<
+      ((batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>) | undefined
+    >(undefined)
+
     const makeSyncBackend = Effect.gen(function* () {
+      const nonLiveChunkSize = Math.max(1, options?.nonLiveChunkSize ?? 100)
+
+      // TODO consider making offline state actively error pull/push.
+      // Currently, offline only reflects in `isConnected`, while operations still succeed,
+      // mirroring how some real providers behave during transient disconnects.
       return SyncBackend.of<Schema.JsonValue>({
         isConnected: syncIsConnectedRef,
-        connect: Effect.void,
+        connect: SubscriptionRef.set(syncIsConnectedRef, true),
         ping: Effect.void,
-        pull: () =>
-          Stream.fromQueue(syncPullQueue).pipe(
-            Stream.chunks,
-            Stream.map((chunk) => ({
-              batch: [...chunk].map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
-              pageInfo: SyncBackend.pageInfoNoMore,
-            })),
-            Stream.withSpan('MockSyncBackend:pull', { parent: span }),
-          ),
+        pull: (cursor, options) =>
+          (options?.live
+            ? Stream.concat(
+                Stream.make(SyncBackend.pullResItemEmpty()),
+                Stream.fromQueue(syncPullQueue).pipe(
+                  Stream.chunks,
+                  Stream.map((chunk) => ({
+                    batch: [...chunk].map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
+                    pageInfo: SyncBackend.pageInfoNoMore,
+                  })),
+                ),
+              )
+            : Stream.fromEffect(
+                Effect.sync(() => {
+                  const lastSeen = cursor.pipe(
+                    Option.match({
+                      onNone: () => EventSequenceNumber.ROOT.global,
+                      onSome: (_) => _.eventSequenceNumber,
+                    }),
+                  )
+                  // All events with seqNum greater than lastSeen
+                  const slice = allEventsRef.current.filter((e) => e.seqNum > lastSeen)
+                  // Split into configured chunk size
+                  const chunks: { events: LiveStoreEvent.AnyEncodedGlobal[]; remaining: number }[] = []
+                  for (let i = 0; i < slice.length; i += nonLiveChunkSize) {
+                    const end = Math.min(i + nonLiveChunkSize, slice.length)
+                    const remaining = Math.max(slice.length - end, 0)
+                    chunks.push({ events: slice.slice(i, end), remaining })
+                  }
+                  if (chunks.length === 0) {
+                    chunks.push({ events: [], remaining: 0 })
+                  }
+                  return chunks
+                }),
+              ).pipe(
+                Stream.flatMap((chunks) =>
+                  Stream.fromIterable(chunks).pipe(
+                    Stream.map(({ events, remaining }) => ({
+                      batch: events.map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
+                      pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
+                    })),
+                  ),
+                ),
+              )
+          ).pipe(Stream.withSpan('MockSyncBackend:pull', { parent: span })),
         push: (batch) =>
           Effect.gen(function* () {
             yield* validatePushPayload(batch, syncEventSequenceNumberRef.current)
 
+            const remaining = yield* SubscriptionRef.get(failCounterRef)
+            if (remaining > 0) {
+              const maybeFail = yield* SubscriptionRef.get(failEffectRef)
+              // decrement counter first
+              yield* SubscriptionRef.set(failCounterRef, remaining - 1)
+              if (maybeFail) {
+                return yield* maybeFail(batch)
+              }
+              return yield* new InvalidPushError({
+                cause: new UnexpectedError({ cause: new Error('MockSyncBackend: simulated push failure') }),
+              })
+            }
+
             yield* Effect.sleep(10).pipe(Effect.withSpan('MockSyncBackend:push:sleep')) // Simulate network latency
 
             yield* pushedEventsQueue.offerAll(batch)
             yield* syncPullQueue.offerAll(batch)
+            allEventsRef.current = allEventsRef.current.concat(batch)
 
             syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
           }).pipe(
@@ -71,6 +149,7 @@ export const makeMockSyncBackend: Effect.Effect<MockSyncBackend, UnexpectedError
     const advance = (...batch: LiveStoreEvent.AnyEncodedGlobal[]) =>
       Effect.gen(function* () {
         syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
+        allEventsRef.current = allEventsRef.current.concat(batch)
         yield* syncPullQueue.offerAll(batch)
       }).pipe(
         Effect.withSpan('MockSyncBackend:advance', {
@@ -83,6 +162,15 @@ export const makeMockSyncBackend: Effect.Effect<MockSyncBackend, UnexpectedError
     const connect = SubscriptionRef.set(syncIsConnectedRef, true)
     const disconnect = SubscriptionRef.set(syncIsConnectedRef, false)
 
+    const failNextPushes = (
+      count: number,
+      error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
+    ) =>
+      Effect.gen(function* () {
+        yield* SubscriptionRef.set(failCounterRef, count)
+        yield* SubscriptionRef.set(failEffectRef, error)
+      })
+
     return {
       syncEventSequenceNumberRef,
       syncPullQueue,
@@ -91,6 +179,6 @@ export const makeMockSyncBackend: Effect.Effect<MockSyncBackend, UnexpectedError
       disconnect,
       makeSyncBackend,
       advance,
+      failNextPushes,
     }
-  },
-).pipe(Effect.withSpanScoped('MockSyncBackend'))
+  }).pipe(Effect.withSpanScoped('MockSyncBackend'))