@livestore/common 0.4.0-dev.2 → 0.4.0-dev.5

package/src/sync/mock-sync-backend.ts

@@ -0,0 +1,96 @@
+import type { Schema, Scope } from '@livestore/utils/effect'
+import { Effect, Mailbox, Option, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import type { UnexpectedError } from '../errors.ts'
+import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
+import * as SyncBackend from './sync-backend.ts'
+import { validatePushPayload } from './validate-push-payload.ts'
+
+export interface MockSyncBackend {
+  pushedEvents: Stream.Stream<LiveStoreEvent.AnyEncodedGlobal>
+  connect: Effect.Effect<void>
+  disconnect: Effect.Effect<void>
+  makeSyncBackend: Effect.Effect<SyncBackend.SyncBackend, UnexpectedError, Scope.Scope>
+  advance: (...batch: LiveStoreEvent.AnyEncodedGlobal[]) => Effect.Effect<void>
+}
+
+export const makeMockSyncBackend: Effect.Effect<MockSyncBackend, UnexpectedError, Scope.Scope> = Effect.gen(
+  function* () {
+    const syncEventSequenceNumberRef = { current: EventSequenceNumber.ROOT.global }
+    const syncPullQueue = yield* Queue.unbounded<LiveStoreEvent.AnyEncodedGlobal>()
+    const pushedEventsQueue = yield* Mailbox.make<LiveStoreEvent.AnyEncodedGlobal>()
+    const syncIsConnectedRef = yield* SubscriptionRef.make(true)
+
+    const span = yield* Effect.currentSpan.pipe(Effect.orDie)
+
+    const semaphore = yield* Effect.makeSemaphore(1)
+
+    const makeSyncBackend = Effect.gen(function* () {
+      return SyncBackend.of<Schema.JsonValue>({
+        isConnected: syncIsConnectedRef,
+        connect: Effect.void,
+        ping: Effect.void,
+        pull: () =>
+          Stream.fromQueue(syncPullQueue).pipe(
+            Stream.chunks,
+            Stream.map((chunk) => ({
+              batch: [...chunk].map((eventEncoded) => ({ eventEncoded, metadata: Option.none() })),
+              pageInfo: SyncBackend.pageInfoNoMore,
+            })),
+            Stream.withSpan('MockSyncBackend:pull', { parent: span }),
+          ),
+        push: (batch) =>
+          Effect.gen(function* () {
+            yield* validatePushPayload(batch, syncEventSequenceNumberRef.current)
+
+            yield* Effect.sleep(10).pipe(Effect.withSpan('MockSyncBackend:push:sleep')) // Simulate network latency
+
+            yield* pushedEventsQueue.offerAll(batch)
+            yield* syncPullQueue.offerAll(batch)
+
+            syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
+          }).pipe(
+            Effect.withSpan('MockSyncBackend:push', {
+              parent: span,
+              attributes: {
+                nums: batch.map((_) => _.seqNum),
+              },
+            }),
+            semaphore.withPermits(1),
+          ),
+        metadata: {
+          name: '@livestore/mock-sync',
+          description: 'Just a mock sync backend',
+        },
+        supports: {
+          pullPageInfoKnown: true,
+          pullLive: true,
+        },
+      })
+    })
+
+    const advance = (...batch: LiveStoreEvent.AnyEncodedGlobal[]) =>
+      Effect.gen(function* () {
+        syncEventSequenceNumberRef.current = batch.at(-1)!.seqNum
+        yield* syncPullQueue.offerAll(batch)
+      }).pipe(
+        Effect.withSpan('MockSyncBackend:advance', {
+          parent: span,
+          attributes: { nums: batch.map((_) => _.seqNum) },
+        }),
+        semaphore.withPermits(1),
+      )
+
+    const connect = SubscriptionRef.set(syncIsConnectedRef, true)
+    const disconnect = SubscriptionRef.set(syncIsConnectedRef, false)
+
+    return {
+      syncEventSequenceNumberRef,
+      syncPullQueue,
+      pushedEvents: Mailbox.toStream(pushedEventsQueue),
+      connect,
+      disconnect,
+      makeSyncBackend,
+      advance,
+    }
+  },
+).pipe(Effect.withSpanScoped('MockSyncBackend'))

package/src/sync/next/history-dag.ts

@@ -18,7 +18,9 @@ export const historyDagFromNodes = (dagNodes: HistoryDagNode[], options?: { skip
 
   const dag = emptyHistoryDag()
 
-  dagNodes.forEach((node) => dag.addNode(EventSequenceNumber.toString(node.seqNum), node))
+  dagNodes.forEach((node) => {
+    dag.addNode(EventSequenceNumber.toString(node.seqNum), node)
+  })
 
   dagNodes.forEach((node) => {
     if (EventSequenceNumber.toString(node.parentSeqNum) !== EventSequenceNumber.toString(rootParentNum)) {

package/src/sync/sync-backend-kv.ts

@@ -0,0 +1,22 @@
+import { Effect, KeyValueStore, Option } from '@livestore/utils/effect'
+import { UnexpectedError } from '../errors.ts'
+
+export const makeBackendIdHelper = Effect.gen(function* () {
+  const kv = yield* KeyValueStore.KeyValueStore
+
+  const backendIdKey = `backendId`
+  const backendIdRef = { current: yield* kv.get(backendIdKey).pipe(UnexpectedError.mapToUnexpectedError) }
+
+  const setBackendId = (backendId: string) =>
+    Effect.gen(function* () {
+      if (backendIdRef.current._tag === 'None' || backendIdRef.current.value !== backendId) {
+        backendIdRef.current = Option.some(backendId)
+        yield* kv.set(backendIdKey, backendId)
+      }
+    }).pipe(UnexpectedError.mapToUnexpectedError)
+
+  return {
+    lazySet: setBackendId,
+    get: () => backendIdRef.current,
+  }
+})

package/src/sync/sync-backend.ts

@@ -0,0 +1,137 @@
+import {
+  type Cause,
+  type Effect,
+  type HttpClient,
+  type KeyValueStore,
+  Option,
+  Schema,
+  type Scope,
+  type Stream,
+  type SubscriptionRef,
+} from '@livestore/utils/effect'
+import type { UnexpectedError } from '../adapter-types.ts'
+import type * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
+import type { EventSequenceNumber } from '../schema/mod.ts'
+import type { InvalidPullError, InvalidPushError, IsOfflineError } from './errors.ts'
+
+export * from './sync-backend-kv.ts'
+
+/**
+ * Those arguments can be used to implement multi-tenancy etc and are passed in from the store.
+ */
+export type MakeBackendArgs = {
+  storeId: string
+  clientId: string
+  payload: Schema.JsonValue | undefined
+}
+
+// TODO rename to `SyncProviderClientConstructor`
+export type SyncBackendConstructor<TSyncMetadata = Schema.JsonValue> = (
+  args: MakeBackendArgs,
+) => Effect.Effect<
+  SyncBackend<TSyncMetadata>,
+  UnexpectedError,
+  Scope.Scope | HttpClient.HttpClient | KeyValueStore.KeyValueStore
+>
+
+// TODO add more runtime sync metadata/metrics
+// - latency histogram
+// - number of events pushed/pulled
+// - dynamic sync backend data;
+//   - data center location (e.g. colo on CF workers)
+
+// TODO rename to `SyncProviderClient`
+export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
+  /**
+   * Can be implemented to prepare a connection to the sync backend to speed up the first pull/push.
+   */
+  connect: Effect.Effect<void, IsOfflineError | UnexpectedError, Scope.Scope>
+  pull: (
+    cursor: Option.Option<{
+      eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+      /** Metadata is needed by some sync backends */
+      metadata: Option.Option<TSyncMetadata>
+    }>,
+    options?: {
+      /**
+       * If true, the sync backend will return a stream of events that have been pushed after the cursor.
+       *
+       * @default false
+       */
+      live?: boolean
+    },
+  ) => Stream.Stream<PullResItem<TSyncMetadata>, IsOfflineError | InvalidPullError>
+  // TODO support transactions (i.e. group of mutation events which need to be applied together)
+  push: (
+    /**
+     * Constraints for batch:
+     * - Number of events: 1-100
+     * - sequence numbers must be in ascending order
+     * */
+    batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>,
+  ) => Effect.Effect<void, IsOfflineError | InvalidPushError>
+  ping: Effect.Effect<void, IsOfflineError | UnexpectedError | Cause.TimeoutException>
+  // TODO also expose latency information additionally to whether the backend is connected
+  isConnected: SubscriptionRef.SubscriptionRef<boolean>
+  /**
+   * Metadata describing the sync backend. (Currently only used by devtools.)
+   */
+  metadata: { name: string; description: string } & Record<string, Schema.JsonValue>
+  /** Information about the sync backend capabilities. */
+  supports: {
+    /**
+     * Whether the sync backend supports the `hasMore` field in the pull response.
+     */
+    pullPageInfoKnown: boolean
+    /**
+     * Whether the sync backend supports the `live` option for the pull method and thus
+     * long-lived, reactive pull streams.
+     */
+    pullLive: boolean
+  }
+}
+
+export const PullResPageInfo = Schema.Union(
+  Schema.TaggedStruct('MoreUnknown', {}),
+  Schema.TaggedStruct('MoreKnown', {
+    remaining: Schema.Number,
+  }),
+  Schema.TaggedStruct('NoMore', {}),
+)
+
+export type PullResPageInfo = typeof PullResPageInfo.Type
+
+export const pageInfoNoMore: PullResPageInfo = { _tag: 'NoMore' } as const
+export const pageInfoMoreUnknown: PullResPageInfo = { _tag: 'MoreUnknown' } as const
+export const pageInfoMoreKnown = (remaining: number): PullResPageInfo => ({ _tag: 'MoreKnown', remaining })
+
+export const pullResItemEmpty = <TSyncMetadata = Schema.JsonValue>(): PullResItem<TSyncMetadata> => ({
+  batch: [],
+  pageInfo: pageInfoNoMore,
+})
+
+export interface PullResItem<TSyncMetadata = Schema.JsonValue> {
+  batch: ReadonlyArray<{
+    eventEncoded: LiveStoreEvent.AnyEncodedGlobal
+    metadata: Option.Option<TSyncMetadata>
+  }>
+  pageInfo: PullResPageInfo
+}
+
+export const of = <TSyncMetadata = Schema.JsonValue>(obj: SyncBackend<TSyncMetadata>) => obj
+
+/**
+ * Useful to continue pulling from the last event in the batch.
+ */
+export const cursorFromPullResItem = <TSyncMetadata = Schema.JsonValue>(
+  item: PullResItem<TSyncMetadata>,
+): Option.Option<{
+  eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
+  metadata: Option.Option<TSyncMetadata>
+}> => {
+  const lastEvent = item.batch.at(-1)
+  if (!lastEvent) {
+    return Option.none()
+  }
+  return Option.some({ eventSequenceNumber: lastEvent.eventEncoded.seqNum, metadata: lastEvent.metadata })
+}

package/src/sync/sync.ts

@@ -1,19 +1,8 @@
-import type { Effect, HttpClient, Option, Scope, Stream, SubscriptionRef } from '@livestore/utils/effect'
-import { Schema } from '@livestore/utils/effect'
+export * from './errors.ts'
+export * as SyncBackend from './sync-backend.ts'
 
-import type { UnexpectedError } from '../adapter-types.ts'
 import type { InitialSyncOptions } from '../leader-thread/types.ts'
-import * as EventSequenceNumber from '../schema/EventSequenceNumber.ts'
-import type * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
-
-/**
- * Those arguments can be used to implement multi-tenancy etc and are passed in from the store.
- */
-export type MakeBackendArgs = {
-  storeId: string
-  clientId: string
-  payload: Schema.JsonValue | undefined
-}
+import type { SyncBackendConstructor } from './sync-backend.ts'
 
 export type SyncOptions = {
   backend?: SyncBackendConstructor<any>
@@ -29,81 +18,9 @@ export type SyncOptions = {
    * @default 'ignore'
    * */
   onSyncError?: 'shutdown' | 'ignore'
-}
-
-// TODO rename to `SyncProviderClientConstructor`
-export type SyncBackendConstructor<TSyncMetadata = Schema.JsonValue> = (
-  args: MakeBackendArgs,
-) => Effect.Effect<SyncBackend<TSyncMetadata>, UnexpectedError, Scope.Scope | HttpClient.HttpClient>
-
-// TODO add more runtime sync metadata
-// - latency histogram
-// - number of events pushed/pulled
-// - dynamic sync backend data;
-//   - data center location (e.g. colo on CF workers)
-
-// TODO rename to `SyncProviderClient`
-export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
   /**
-   * Can be implemented to prepare a connection to the sync backend to speed up the first pull/push.
+   * Whether the sync backend should reactively pull new events from the sync backend
+   * @default true
    */
-  connect: Effect.Effect<void, IsOfflineError | UnexpectedError, HttpClient.HttpClient | Scope.Scope>
-  pull: (
-    args: Option.Option<{
-      cursor: EventSequenceNumber.EventSequenceNumber
-      metadata: Option.Option<TSyncMetadata>
-    }>,
-  ) => Stream.Stream<
-    {
-      batch: ReadonlyArray<{
-        eventEncoded: LiveStoreEvent.AnyEncodedGlobal
-        metadata: Option.Option<TSyncMetadata>
-      }>
-      remaining: number
-    },
-    IsOfflineError | InvalidPullError,
-    HttpClient.HttpClient
-  >
-  // TODO support transactions (i.e. group of mutation events which need to be applied together)
-  push: (
-    /**
-     * Constraints for batch:
-     * - Number of events: 1-100
-     * - sequence numbers must be in ascending order
-     * */
-    batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>,
-  ) => Effect.Effect<void, IsOfflineError | InvalidPushError, HttpClient.HttpClient>
-  isConnected: SubscriptionRef.SubscriptionRef<boolean>
-  /**
-   * Metadata describing the sync backend. (Currently only used by devtools.)
-   */
-  metadata: { name: string; description: string } & Record<string, Schema.JsonValue>
+  livePull?: boolean
 }
-
-export class IsOfflineError extends Schema.TaggedError<IsOfflineError>()('IsOfflineError', {}) {}
-
-// TODO gt rid of this error in favour of SyncError
-export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('InvalidPushError', {
-  reason: Schema.Union(
-    Schema.TaggedStruct('Unexpected', {
-      message: Schema.String,
-    }),
-    Schema.TaggedStruct('ServerAhead', {
-      minimumExpectedNum: Schema.Number,
-      providedNum: Schema.Number,
-    }),
-  ),
-}) {}
-
-// TODO gt rid of this error in favour of SyncError
-export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
-  message: Schema.String,
-}) {}
-
-// TODO gt rid of this error in favour of SyncError
-export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
-  minimumExpectedNum: EventSequenceNumber.EventSequenceNumber,
-  providedNum: EventSequenceNumber.EventSequenceNumber,
-  /** Generation number the client session should use for subsequent pushes */
-  // nextGeneration: Schema.Number,
-}) {}

package/src/sync/validate-push-payload.ts

@@ -1,7 +1,7 @@
 import { Effect } from '@livestore/utils/effect'
 
-import type { EventSequenceNumber, LiveStoreEvent } from '../schema/mod.ts'
-import { InvalidPushError } from './sync.ts'
+import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
+import { InvalidPushError, ServerAheadError } from './sync.ts'
 
 // TODO proper batch validation
 export const validatePushPayload = (
@@ -11,11 +11,10 @@ export const validatePushPayload = (
   Effect.gen(function* () {
     if (batch[0]!.seqNum <= currentEventSequenceNumber) {
       return yield* InvalidPushError.make({
-        reason: {
-          _tag: 'ServerAhead',
-          minimumExpectedNum: currentEventSequenceNumber + 1,
-          providedNum: batch[0]!.seqNum,
-        },
+        cause: new ServerAheadError({
+          minimumExpectedNum: EventSequenceNumber.globalEventSequenceNumber(currentEventSequenceNumber + 1),
+          providedNum: EventSequenceNumber.globalEventSequenceNumber(batch[0]!.seqNum),
+        }),
       })
     }
   })

package/src/version.ts

@@ -2,7 +2,7 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.4.0-dev.2' as const
+export const liveStoreVersion = '0.4.0-dev.5' as const
 
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.
@@ -11,4 +11,4 @@ export const liveStoreVersion = '0.4.0-dev.2' as const
  * While LiveStore is in beta, this might happen more frequently.
  * In the future, LiveStore will provide a migration path for older database files to avoid the impression of data loss.
  */
-export const liveStoreStorageFormatVersion = 5
+export const liveStoreStorageFormatVersion = 6