@livestore/common 0.4.0-dev.9 → 0.4.0

package/src/sync/mock-sync-backend.ts

@@ -1,21 +1,29 @@
 import type { Schema, Scope } from '@livestore/utils/effect'
-import { Effect, Mailbox, Option, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
-import { UnexpectedError } from '../errors.ts'
+import { Effect, Mailbox, Option, Queue, Ref, Stream, SubscriptionRef } from '@livestore/utils/effect'
+
+import { UnknownError } from '../errors.ts'
 import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
-import { InvalidPushError } from './errors.ts'
+import { type BackendIdMismatchError, type ServerAheadError } from './errors.ts'
 import * as SyncBackend from './sync-backend.ts'
 import { validatePushPayload } from './validate-push-payload.ts'
 
 export interface MockSyncBackend {
-  pushedEvents: Stream.Stream<LiveStoreEvent.AnyEncodedGlobal>
+  pushedEvents: Stream.Stream<LiveStoreEvent.Global.Encoded>
   connect: Effect.Effect<void>
   disconnect: Effect.Effect<void>
-  makeSyncBackend: Effect.Effect<SyncBackend.SyncBackend, UnexpectedError, Scope.Scope>
-  advance: (...batch: LiveStoreEvent.AnyEncodedGlobal[]) => Effect.Effect<void>
-  /** Fail the next N push calls with an InvalidPushError (or custom error) */
+  makeSyncBackend: Effect.Effect<SyncBackend.SyncBackend, UnknownError, Scope.Scope>
+  advance: (...batch: LiveStoreEvent.Global.Encoded[]) => Effect.Effect<void>
+  /** Fail the next N push calls with an UnknownError, ServerAheadError, BackendIdMismatchError, or custom error */
   failNextPushes: (
     count: number,
-    error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
+    error?: (
+      batch: ReadonlyArray<LiveStoreEvent.Global.Encoded>,
+    ) => Effect.Effect<never, UnknownError | ServerAheadError | BackendIdMismatchError>,
+  ) => Effect.Effect<void>
+  /** Fail the next N pull calls with an UnknownError, BackendIdMismatchError, or custom error */
+  failNextPulls: (
+    count: number,
+    error?: () => Effect.Effect<never, UnknownError | BackendIdMismatchError>,
   ) => Effect.Effect<void>
 }
 
@@ -29,109 +37,138 @@ export interface MockSyncBackendOptions {
 
 export const makeMockSyncBackend = (
   options?: MockSyncBackendOptions,
-): Effect.Effect<MockSyncBackend, UnexpectedError, Scope.Scope> =>
+): Effect.Effect<MockSyncBackend, UnknownError, Scope.Scope> =>
   Effect.gen(function* () {
-    const syncEventSequenceNumberRef = { current: EventSequenceNumber.ROOT.global }
-    const syncPullQueue = yield* Queue.unbounded<LiveStoreEvent.AnyEncodedGlobal>()
-    const pushedEventsQueue = yield* Mailbox.make<LiveStoreEvent.AnyEncodedGlobal>()
+    const span = yield* Effect.currentSpan.pipe(Effect.orDie)
+    const semaphore = yield* Effect.makeSemaphore(1)
+
+    // State refs
+    const syncHeadRef = yield* Ref.make(EventSequenceNumber.Client.ROOT.global)
+    const allEventsRef = yield* Ref.make<LiveStoreEvent.Global.Encoded[]>([])
     const syncIsConnectedRef = yield* SubscriptionRef.make(options?.startConnected ?? false)
-    const allEventsRef: { current: LiveStoreEvent.AnyEncodedGlobal[] } = { current: [] }
 
-    const span = yield* Effect.currentSpan.pipe(Effect.orDie)
+    // Queues for streaming
+    const syncPullQueue = yield* Queue.unbounded<LiveStoreEvent.Global.Encoded>()
+    const pushedEventsQueue = yield* Mailbox.make<LiveStoreEvent.Global.Encoded>()
+
+    // Failure simulation state
+    const failPushRef = yield* Ref.make<
+      FailureState<UnknownError | ServerAheadError | BackendIdMismatchError, [ReadonlyArray<LiveStoreEvent.Global.Encoded>]>
+    >({ remaining: 0, error: undefined })
+    const failPullRef = yield* Ref.make<FailureState<UnknownError | BackendIdMismatchError, []>>({
+      remaining: 0,
+      error: undefined,
+    })
 
-    const semaphore = yield* Effect.makeSemaphore(1)
+    const nonLiveChunkSize = Math.max(1, options?.nonLiveChunkSize ?? 100)
+
+    /** Check and consume a simulated failure, returning the error effect if one should fire */
+    const checkFailure = <E, Args extends unknown[]>(
+      ref: Ref.Ref<FailureState<E, Args>>,
+      defaultError: E,
+      ...args: Args
+    ): Effect.Effect<void, E> =>
+      Ref.modify(ref, (state) => {
+        if (state.remaining <= 0) {
+          return [Option.none(), state] as const
+        }
+        const error = state.error?.(...args) ?? Effect.fail(defaultError)
+        return [Option.some(error), { ...state, remaining: state.remaining - 1 }] as const
+      }).pipe(
+        Effect.flatMap(
+          Option.match({
+            onNone: () => Effect.void,
+            onSome: (errorEffect) => errorEffect,
+          }),
+        ),
+      )
 
-    // TODO improve the API and implementation of simulating errors
-    const failCounterRef = yield* SubscriptionRef.make(0)
-    const failEffectRef = yield* SubscriptionRef.make<
-      ((batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>) | undefined
-    >(undefined)
+    const pullNonLive = (cursor: Option.Option<{ eventSequenceNumber: EventSequenceNumber.Global.Type }>) =>
+      Effect.gen(function* () {
+        const lastSeen = Option.match(cursor, {
+          onNone: () => EventSequenceNumber.Client.ROOT.global,
+          onSome: (_) => _.eventSequenceNumber,
+        })
+        const allEvents = yield* Ref.get(allEventsRef)
+        const slice = allEvents.filter((e) => e.seqNum > lastSeen)
+
+        // Split into chunks with remaining count for pageInfo
+        const chunks: Array<{ events: LiveStoreEvent.Global.Encoded[]; remaining: number }> = []
+        for (let i = 0; i < slice.length; i += nonLiveChunkSize) {
+          const end = Math.min(i + nonLiveChunkSize, slice.length)
+          chunks.push({
+            events: slice.slice(i, end),
+            remaining: Math.max(slice.length - end, 0),
+          })
+        }
+        // Always return at least one empty chunk
+        if (chunks.length === 0) {
+          chunks.push({ events: [], remaining: 0 })
+        }
+        return chunks
+      }).pipe(
+        Effect.map((chunks) =>
+          Stream.fromIterable(chunks).pipe(
+            Stream.map(({ events, remaining }) => ({
+              batch: events.map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
+              pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
+            })),
+          ),
+        ),
+        Stream.fromEffect,
+        Stream.flatten(),
+      )
 
-    const makeSyncBackend = Effect.gen(function* () {
-      const nonLiveChunkSize = Math.max(1, options?.nonLiveChunkSize ?? 100)
+    const pullLive = Stream.concat(
+      Stream.make(SyncBackend.pullResItemEmpty()),
+      Stream.fromQueue(syncPullQueue).pipe(
+        Stream.chunks,
+        Stream.map((chunk) => ({
+          batch: [...chunk].map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
+          pageInfo: SyncBackend.pageInfoNoMore,
+        })),
+      ),
+    )
 
+    const makeSyncBackend = Effect.gen(function* () {
       // TODO consider making offline state actively error pull/push.
       // Currently, offline only reflects in `isConnected`, while operations still succeed,
       // mirroring how some real providers behave during transient disconnects.
-      return SyncBackend.of<Schema.JsonValue>({
+      return SyncBackend.of({
         isConnected: syncIsConnectedRef,
         connect: SubscriptionRef.set(syncIsConnectedRef, true),
         ping: Effect.void,
-        pull: (cursor, options) =>
-          (options?.live
-            ? Stream.concat(
-                Stream.make(SyncBackend.pullResItemEmpty()),
-                Stream.fromQueue(syncPullQueue).pipe(
-                  Stream.chunks,
-                  Stream.map((chunk) => ({
-                    batch: [...chunk].map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
-                    pageInfo: SyncBackend.pageInfoNoMore,
-                  })),
-                ),
-              )
-            : Stream.fromEffect(
-                Effect.sync(() => {
-                  const lastSeen = cursor.pipe(
-                    Option.match({
-                      onNone: () => EventSequenceNumber.ROOT.global,
-                      onSome: (_) => _.eventSequenceNumber,
-                    }),
-                  )
-                  // All events with seqNum greater than lastSeen
-                  const slice = allEventsRef.current.filter((e) => e.seqNum > lastSeen)
-                  // Split into configured chunk size
-                  const chunks: { events: LiveStoreEvent.AnyEncodedGlobal[]; remaining: number }[] = []
-                  for (let i = 0; i < slice.length; i += nonLiveChunkSize) {
-                    const end = Math.min(i + nonLiveChunkSize, slice.length)
-                    const remaining = Math.max(slice.length - end, 0)
-                    chunks.push({ events: slice.slice(i, end), remaining })
-                  }
-                  if (chunks.length === 0) {
-                    chunks.push({ events: [], remaining: 0 })
-                  }
-                  return chunks
-                }),
-              ).pipe(
-                Stream.flatMap((chunks) =>
-                  Stream.fromIterable(chunks).pipe(
-                    Stream.map(({ events, remaining }) => ({
-                      batch: events.map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
-                      pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
-                    })),
-                  ),
-                ),
-              )
-          ).pipe(Stream.withSpan('MockSyncBackend:pull', { parent: span })),
+        pull: (cursor, pullOptions) =>
+          Stream.fromEffect(
+            checkFailure(
+              failPullRef,
+              new UnknownError({ cause: new Error('MockSyncBackend: simulated pull failure') }),
+            ),
+          ).pipe(
+            Stream.flatMap(() => (pullOptions?.live === true ? pullLive : pullNonLive(cursor))),
+            Stream.withSpan('MockSyncBackend:pull', { parent: span }),
+          ),
         push: (batch) =>
           Effect.gen(function* () {
-            yield* validatePushPayload(batch, syncEventSequenceNumberRef.current)
-
-            const remaining = yield* SubscriptionRef.get(failCounterRef)
-            if (remaining > 0) {
-              const maybeFail = yield* SubscriptionRef.get(failEffectRef)
-              // decrement counter first
-              yield* SubscriptionRef.set(failCounterRef, remaining - 1)
-              if (maybeFail) {
-                return yield* maybeFail(batch)
-              }
-              return yield* new InvalidPushError({
-                cause: new UnexpectedError({ cause: new Error('MockSyncBackend: simulated push failure') }),
-              })
-            }
+            const currentHead = yield* Ref.get(syncHeadRef)
+            yield* validatePushPayload(batch, currentHead)
+
+            yield* checkFailure(
+              failPushRef,
+              new UnknownError({ cause: new Error('MockSyncBackend: simulated push failure') }),
+              batch,
+            )
 
             yield* Effect.sleep(10).pipe(Effect.withSpan('MockSyncBackend:push:sleep')) // Simulate network latency
 
             yield* pushedEventsQueue.offerAll(batch)
             yield* syncPullQueue.offerAll(batch)
-            allEventsRef.current = allEventsRef.current.concat(batch)
-
-            syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
+            yield* Ref.update(allEventsRef, (events) => events.concat(batch))
+            yield* Ref.set(syncHeadRef, batch.at(-1)!.seqNum)
           }).pipe(
             Effect.withSpan('MockSyncBackend:push', {
               parent: span,
-              attributes: {
-                nums: batch.map((_) => _.seqNum),
-              },
+              attributes: { nums: batch.map((_) => _.seqNum) },
             }),
             semaphore.withPermits(1),
           ),
@@ -146,10 +183,10 @@ export const makeMockSyncBackend = (
       })
     })
 
-    const advance = (...batch: LiveStoreEvent.AnyEncodedGlobal[]) =>
+    const advance = (...batch: LiveStoreEvent.Global.Encoded[]) =>
       Effect.gen(function* () {
-        syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
-        allEventsRef.current = allEventsRef.current.concat(batch)
+        yield* Ref.set(syncHeadRef, batch.at(-1)!.seqNum)
+        yield* Ref.update(allEventsRef, (events) => events.concat(batch))
         yield* syncPullQueue.offerAll(batch)
       }).pipe(
         Effect.withSpan('MockSyncBackend:advance', {
@@ -159,26 +196,31 @@ export const makeMockSyncBackend = (
         semaphore.withPermits(1),
       )
 
-    const connect = SubscriptionRef.set(syncIsConnectedRef, true)
-    const disconnect = SubscriptionRef.set(syncIsConnectedRef, false)
-
     const failNextPushes = (
       count: number,
-      error?: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>) => Effect.Effect<never, InvalidPushError>,
-    ) =>
-      Effect.gen(function* () {
-        yield* SubscriptionRef.set(failCounterRef, count)
-        yield* SubscriptionRef.set(failEffectRef, error)
-      })
+      error?: (
+        batch: ReadonlyArray<LiveStoreEvent.Global.Encoded>,
+      ) => Effect.Effect<never, UnknownError | ServerAheadError | BackendIdMismatchError>,
+    ) => Ref.set(failPushRef, { remaining: count, error })
+
+    const failNextPulls = (
+      count: number,
+      error?: () => Effect.Effect<never, UnknownError | BackendIdMismatchError>,
+    ) => Ref.set(failPullRef, { remaining: count, error })
 
     return {
-      syncEventSequenceNumberRef,
-      syncPullQueue,
       pushedEvents: Mailbox.toStream(pushedEventsQueue),
-      connect,
-      disconnect,
+      connect: SubscriptionRef.set(syncIsConnectedRef, true),
+      disconnect: SubscriptionRef.set(syncIsConnectedRef, false),
       makeSyncBackend,
       advance,
       failNextPushes,
+      failNextPulls,
     }
   }).pipe(Effect.withSpanScoped('MockSyncBackend'))
+
+/** Internal state for simulating failures */
+interface FailureState<E, Args extends unknown[]> {
+  remaining: number
+  error: ((...args: Args) => Effect.Effect<never, E>) | undefined
+}

package/src/sync/next/compact-events.ts

@@ -1,6 +1,5 @@
 import { EventSequenceNumber } from '../../schema/mod.ts'
 import { replacesFacts } from './facts.ts'
-import { graphologyDag } from './graphology_.ts'
 import type { HistoryDag } from './history-dag-common.ts'
 import { emptyHistoryDag } from './history-dag-common.ts'
 
@@ -17,7 +16,7 @@ export const compactEvents = (inputDag: HistoryDag): { dag: HistoryDag; compacte
   const dag = inputDag.copy()
   const compactedEventCount = 0
 
-  const orderedEventSequenceNumberStrs = graphologyDag.topologicalSort(dag).reverse()
+  const orderedEventSequenceNumberStrs = dag.topologicalNodeIds().toReversed()
 
   // drop root
   orderedEventSequenceNumberStrs.pop()
@@ -30,7 +29,7 @@ export const compactEvents = (inputDag: HistoryDag): { dag: HistoryDag; compacte
     const subDagsForEvent = Array.from(makeSubDagsForEvent(dag, eventNumStr))
     for (const subDag of subDagsForEvent) {
       let shouldRetry = true
-      while (shouldRetry) {
+      while (shouldRetry === true) {
         const subDagsInHistory = findSubDagsInHistory(dag, subDag, eventNumStr)
 
         // console.debug(
@@ -65,7 +64,7 @@ export const compactEvents = (inputDag: HistoryDag): { dag: HistoryDag; compacte
   return { dag, compactedEventCount }
 }
 
-function* makeSubDagsForEvent(inputDag: HistoryDag, eventNumStr: string): Generator<HistoryDag> {
+const makeSubDagsForEvent = function* (inputDag: HistoryDag, eventNumStr: string): Generator<HistoryDag> {
   /** Map from eventNumStr to array of eventNumStrs that are dependencies */
   let nextIterationEls: Map<string, string[]> = new Map([[eventNumStr, []]])
   let previousDag: HistoryDag | undefined
@@ -116,7 +115,7 @@ const findSubDagsInHistory = (
   const subDags: HistoryDag[] = []
   const allOutsideDependencies: string[][] = []
 
-  for (const eventNumStr of graphologyDag.topologicalSort(inputDag)) {
+  for (const eventNumStr of inputDag.topologicalNodeIds()) {
     if (eventNumStr === upToExclEventSequenceNumberStr) {
       break
     }
@@ -132,7 +131,7 @@ const findSubDagsInHistory = (
         allOutsideDependencies.push(outsideDependencies)
       }
 
-      if (outsideDependencies.length === 0 && dagReplacesDag(subDag, targetSubDag)) {
+      if (outsideDependencies.length === 0 && dagReplacesDag(subDag, targetSubDag) === true) {
         subDags.push(subDag)
       } else {
         break
@@ -172,7 +171,7 @@ const dagDependsOnDag = (dagA: HistoryDag, dagB: HistoryDag, inputDag: HistoryDa
     for (const edgeEntryA of inputDag.inboundEdgeEntries(nodeAIdStr)) {
       if (edgeEntryA.attributes.type === 'facts') {
         const depNodeIdStr = edgeEntryA.target
-        if (dagB.hasNode(depNodeIdStr)) {
+        if (dagB.hasNode(depNodeIdStr) === true) {
           return true
         }
       }
@@ -189,8 +188,8 @@ const dagReplacesDag = (dagA: HistoryDag, dagB: HistoryDag): boolean => {
   }
 
   // TODO write tests that covers deterministic order when DAGs have branches
-  const nodeEntriesA = graphologyDag.topologicalSort(dagA).map((nodeId) => dagA.getNodeAttributes(nodeId))
-  const nodeEntriesB = graphologyDag.topologicalSort(dagB).map((nodeId) => dagB.getNodeAttributes(nodeId))
+  const nodeEntriesA = dagA.topologicalNodeIds().map((nodeId) => dagA.getNodeAttributes(nodeId))
+  const nodeEntriesB = dagB.topologicalNodeIds().map((nodeId) => dagB.getNodeAttributes(nodeId))
 
   for (let i = 0; i < nodeEntriesA.length; i++) {
     const nodeA = nodeEntriesA[i]!
@@ -207,14 +206,14 @@ const dagReplacesDag = (dagA: HistoryDag, dagB: HistoryDag): boolean => {
 const removeEvent = (dag: HistoryDag, eventNumStr: string) => {
   // console.debug('removing event', eventNumStr)
   const event = dag.getNodeAttributes(eventNumStr)
-  const parentSeqNumStr = EventSequenceNumber.toString(event.parentSeqNum)
+  const parentSeqNumStr = EventSequenceNumber.Client.toString(event.parentSeqNum)
   const childEdges = dag.outboundEdgeEntries(eventNumStr)
 
   for (const childEdge of childEdges) {
     if (childEdge.attributes.type === 'parent') {
       const childEvent = dag.getNodeAttributes(childEdge.target)
       childEvent.parentSeqNum = { ...event.parentSeqNum }
-      dag.addEdge(parentSeqNumStr, EventSequenceNumber.toString(childEvent.seqNum), { type: 'parent' })
+      dag.addEdge(parentSeqNumStr, EventSequenceNumber.Client.toString(childEvent.seqNum), { type: 'parent' })
     }
   }
 

package/src/sync/next/facts.ts

@@ -6,14 +6,13 @@ import type {
   EventDefFactsGroup,
   EventDefFactsSnapshot,
   FactsCallback,
-} from '../../schema/EventDef.ts'
-import type * as EventSequenceNumber from '../../schema/EventSequenceNumber.ts'
-import { graphologyDag } from './graphology_.ts'
+} from '../../schema/EventDef/mod.ts'
+import type * as EventSequenceNumber from '../../schema/EventSequenceNumber/mod.ts'
 import { EMPTY_FACT_VALUE, type HistoryDag, type HistoryDagNode } from './history-dag-common.ts'
 
 export const factsSnapshotForEvents = (
   events: HistoryDagNode[],
-  endEventSequenceNumber: EventSequenceNumber.EventSequenceNumber,
+  endEventSequenceNumber: EventSequenceNumber.Client.Composite,
 ): EventDefFactsSnapshot => {
   const facts = new Map<string, any>()
 
@@ -30,11 +29,11 @@ export const factsSnapshotForEvents = (
 
 export const factsSnapshotForDag = (
   dag: HistoryDag,
-  endEventSequenceNumber: EventSequenceNumber.EventSequenceNumber | undefined,
+  endEventSequenceNumber: EventSequenceNumber.Client.Composite | undefined,
 ): EventDefFactsSnapshot => {
   const facts = new Map<string, any>()
 
-  const orderedEventSequenceNumberStrs = graphologyDag.topologicalSort(dag)
+  const orderedEventSequenceNumberStrs = dag.topologicalNodeIds()
 
   for (let i = 0; i < orderedEventSequenceNumberStrs.length; i++) {
     const event = dag.getNodeAttributes(orderedEventSequenceNumberStrs[i]!)
@@ -131,7 +130,7 @@ const isSubSetMapByValue = (setA: EventDefFacts, setB: EventDefFacts) => {
 /** Check if setA is a subset of setB */
 const isSubSetMapByKey = (setA: EventDefFacts, setB: EventDefFacts) => {
   for (const [key, _value] of setA) {
-    if (!setB.has(key)) {
+    if (setB.has(key) === false) {
       return false
     }
   }
@@ -169,7 +168,7 @@ export const factsToString = (facts: EventDefFacts) => {
 
 export const factsIntersect = (setA: EventDefFacts, setB: EventDefFacts): boolean => {
   for (const [key, _value] of setA) {
-    if (setB.has(key)) {
+    if (setB.has(key) === true) {
       return true
     }
   }
@@ -200,7 +199,7 @@ export const getFactsGroupForEventArgs = ({
         }
       }
 
-      notYetImplemented(`getFactsGroupForEventArgs: ${prop.toString()} is not yet implemented`)
+      return notYetImplemented(`getFactsGroupForEventArgs: ${prop.toString()} is not yet implemented`)
     },
   })
 
@@ -217,9 +216,9 @@ export const getFactsGroupForEventArgs = ({
     return map
   }
   const facts = {
-    modifySet: factsRes?.modify.set ? iterableToMap(factsRes.modify.set) : new Map(),
-    modifyUnset: factsRes?.modify.unset ? iterableToMap(factsRes.modify.unset) : new Map(),
-    depRequire: factsRes?.require ? iterableToMap(factsRes.require) : new Map(),
+    modifySet: factsRes?.modify.set !== undefined ? iterableToMap(factsRes.modify.set) : new Map(),
+    modifyUnset: factsRes?.modify.unset !== undefined ? iterableToMap(factsRes.modify.unset) : new Map(),
+    depRequire: factsRes?.require !== undefined ? iterableToMap(factsRes.require) : new Map(),
     depRead,
   }
 
@@ -227,8 +226,8 @@ export const getFactsGroupForEventArgs = ({
 }
 
 export const compareEventSequenceNumbers = (
-  a: EventSequenceNumber.EventSequenceNumber,
-  b: EventSequenceNumber.EventSequenceNumber,
+  a: EventSequenceNumber.Client.Composite,
+  b: EventSequenceNumber.Client.Composite,
 ) => {
   if (a.global !== b.global) {
     return a.global - b.global