@livestore/sync-electric 0.3.0-dev.23 → 0.3.0-dev.25

package/src/index.ts

@@ -1,4 +1,4 @@
-import type { IsOfflineError, SyncBackend } from '@livestore/common'
+import type { IsOfflineError, SyncBackend, SyncBackendConstructor } from '@livestore/common'
 import { InvalidPullError, InvalidPushError } from '@livestore/common'
 import type { EventId } from '@livestore/common/schema'
 import { MutationEvent } from '@livestore/common/schema'
@@ -67,7 +67,7 @@ const MutationEventGlobalFromStringRecord = Schema.Struct({
   mutation: Schema.String,
   args: Schema.parseJson(Schema.Any),
   clientId: Schema.String,
-  sessionId: Schema.optional(Schema.String),
+  sessionId: Schema.String,
 }).pipe(
   Schema.transform(MutationEvent.AnyEncodedGlobal, {
     decode: (_) => _,
@@ -101,11 +101,16 @@ export const syncBackend = {} as any
 
 export const syncBackendOptions = <TOptions extends SyncBackendOptions>(options: TOptions) => options
 
+/**
+ * This function should be called in a trusted environment (e.g. a proxy server) as it
+ * requires access to senstive information (e.g. `apiSecret` / `sourceSecret`).
+ */
 export const makeElectricUrl = ({
   electricHost,
   searchParams: providedSearchParams,
   sourceId,
   sourceSecret,
+  apiSecret,
 }: {
   electricHost: string
   /**
@@ -119,6 +124,8 @@ export const makeElectricUrl = ({
   sourceId?: string
   /** Needed for Electric Cloud */
   sourceSecret?: string
+  /** For self-hosted ElectricSQL */
+  apiSecret?: string
 }) => {
   const endpointUrl = `${electricHost}/v1/shape`
   const argsResult = Schema.decodeUnknownEither(Schema.Struct({ args: Schema.parseJson(ApiSchema.PullPayload) }))(
@@ -126,7 +133,11 @@ export const makeElectricUrl = ({
   )
 
   if (argsResult._tag === 'Left') {
-    return shouldNeverHappen('Invalid search params provided to makeElectricUrl', providedSearchParams)
+    return shouldNeverHappen(
+      'Invalid search params provided to makeElectricUrl',
+      providedSearchParams,
+      Object.fromEntries(providedSearchParams.entries()),
+    )
   }
 
   const args = argsResult.right.args
@@ -139,6 +150,9 @@ export const makeElectricUrl = ({
   if (sourceSecret !== undefined) {
     searchParams.set('source_secret', sourceSecret)
   }
+  if (apiSecret !== undefined) {
+    searchParams.set('api_secret', apiSecret)
+  }
   if (args.handle._tag === 'None') {
     searchParams.set('offset', '-1')
   } else {
@@ -153,7 +167,6 @@ export const makeElectricUrl = ({
 }
 
 export interface SyncBackendOptions {
-  storeId: string
   /**
    * The endpoint to pull/push events. Pull is a `GET` request, push is a `POST` request.
    * Usually this endpoint is part of your API layer to proxy requests to the Electric server
@@ -182,153 +195,167 @@ type SyncMetadata = {
   handle: string
 }
 
-export const makeSyncBackend = ({
-  storeId,
-  endpoint,
-}: SyncBackendOptions): Effect.Effect<SyncBackend<SyncMetadata>, never, Scope.Scope> =>
-  Effect.gen(function* () {
-    const isConnected = yield* SubscriptionRef.make(true)
-    const pullEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.pull
-    const pushEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.push
-
-    // TODO check whether we still need this
-    const pendingPushDeferredMap = new Map<EventId.GlobalEventId, Deferred.Deferred<SyncMetadata>>()
-
-    const pull = (
-      handle: Option.Option<SyncMetadata>,
-    ): Effect.Effect<
-      Option.Option<
-        readonly [
-          Chunk.Chunk<{ metadata: Option.Option<SyncMetadata>; mutationEventEncoded: MutationEvent.AnyEncodedGlobal }>,
-          Option.Option<SyncMetadata>,
-        ]
-      >,
-      InvalidPullError | IsOfflineError,
-      HttpClient.HttpClient
-    > =>
-      Effect.gen(function* () {
-        const argsJson = yield* Schema.encode(Schema.parseJson(ApiSchema.PullPayload))(
-          ApiSchema.PullPayload.make({ storeId, handle }),
-        )
-        const url = `${pullEndpoint}?args=${argsJson}`
-
-        const resp = yield* HttpClient.get(url)
-
-        const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp)
-        const nextHandle = {
-          offset: headers['electric-offset'],
-          handle: headers['electric-handle'],
-        }
-
-        // TODO handle case where Electric shape is not found for a given handle
-        // https://electric-sql.com/openapi.html#/paths/~1v1~1shape/get
-        // {
-        // "message": "The shape associated with this shape_handle and offset was not found. Resync to fetch the latest shape",
-        // "shape_handle": "2494_84241",
-        // "offset": "-1"
-        // }
-        if (resp.status === 409) {
-          // TODO: implementation plan:
-          // start pulling events from scratch with the new handle and ignore the "old events"
-          // until we found a new event, then, continue with the new handle
-          return notYetImplemented(`Electric shape not found for handle ${nextHandle.handle}`)
-        }
-
-        // Electric completes the long-poll request after ~20 seconds with a 204 status
-        // In this case we just retry where we left off
-        if (resp.status === 204) {
-          return Option.some([Chunk.empty(), Option.some(nextHandle)] as const)
-        }
-
-        const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem), {
-          onExcessProperty: 'preserve',
-        })(resp)
-
-        const items = body
-          .filter((item) => item.value !== undefined && (item.headers as any).operation === 'insert')
-          .map((item) => ({
-            metadata: Option.some({ offset: nextHandle.offset!, handle: nextHandle.handle }),
-            mutationEventEncoded: item.value! as MutationEvent.AnyEncodedGlobal,
-          }))
-
-        // // TODO implement proper `remaining` handling
-        // remaining: 0,
+export const makeSyncBackend =
+  ({ endpoint }: SyncBackendOptions): SyncBackendConstructor<SyncMetadata> =>
+  ({ storeId, payload }) =>
+    Effect.gen(function* () {
+      const isConnected = yield* SubscriptionRef.make(true)
+      const pullEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.pull
+      const pushEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.push
+
+      // TODO check whether we still need this
+      const pendingPushDeferredMap = new Map<EventId.GlobalEventId, Deferred.Deferred<SyncMetadata>>()
+
+      const pull = (
+        handle: Option.Option<SyncMetadata>,
+      ): Effect.Effect<
+        Option.Option<
+          readonly [
+            Chunk.Chunk<{
+              metadata: Option.Option<SyncMetadata>
+              mutationEventEncoded: MutationEvent.AnyEncodedGlobal
+            }>,
+            Option.Option<SyncMetadata>,
+          ]
+        >,
+        InvalidPullError | IsOfflineError,
+        HttpClient.HttpClient
+      > =>
+        Effect.gen(function* () {
+          const argsJson = yield* Schema.encode(Schema.parseJson(ApiSchema.PullPayload))(
+            ApiSchema.PullPayload.make({ storeId, handle }),
+          )
+          const url = `${pullEndpoint}?args=${argsJson}`
 
-        // if (listenForNew === false && items.length === 0) {
-        //   return Option.none()
-        // }
+          const resp = yield* HttpClient.get(url)
 
-        for (const item of items) {
-          const deferred = pendingPushDeferredMap.get(item.mutationEventEncoded.id)
-          if (deferred !== undefined) {
-            yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata))
+          if (resp.status === 401) {
+            return yield* InvalidPullError.make({
+              message: `Unauthorized (401): Couldn't connect to ElectricSQL`,
+            })
           }
-        }
 
-        return Option.some([Chunk.fromIterable(items), Option.some(nextHandle)] as const)
-      }).pipe(
-        Effect.scoped,
-        Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })),
-      )
-
-    return {
-      pull: (args) =>
-        Stream.unfoldChunkEffect(
-          args.pipe(
-            Option.map((_) => _.metadata),
-            Option.flatten,
-          ),
-          (metadataOption) => pull(metadataOption),
-        ).pipe(
-          Stream.chunks,
-          Stream.map((chunk) => ({ batch: [...chunk], remaining: 0 })),
-        ),
+          const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp)
+          const nextHandle = {
+            offset: headers['electric-offset'],
+            handle: headers['electric-handle'],
+          }
 
-      push: (batch) =>
-        Effect.gen(function* () {
-          const deferreds: Deferred.Deferred<SyncMetadata>[] = []
-          for (const mutationEventEncoded of batch) {
-            const deferred = yield* Deferred.make<SyncMetadata>()
-            pendingPushDeferredMap.set(mutationEventEncoded.id, deferred)
-            deferreds.push(deferred)
+          // TODO handle case where Electric shape is not found for a given handle
+          // https://electric-sql.com/openapi.html#/paths/~1v1~1shape/get
+          // {
+          // "message": "The shape associated with this shape_handle and offset was not found. Resync to fetch the latest shape",
+          // "shape_handle": "2494_84241",
+          // "offset": "-1"
+          // }
+          if (resp.status === 409) {
+            // TODO: implementation plan:
+            // start pulling events from scratch with the new handle and ignore the "old events"
+            // until we found a new event, then, continue with the new handle
+            return notYetImplemented(`Electric shape not found for handle ${nextHandle.handle}`)
           }
 
-          const resp = yield* HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(
-            HttpClientRequest.post(pushEndpoint),
-            ApiSchema.PushPayload.make({ storeId, batch }),
-          ).pipe(
-            Effect.andThen(HttpClient.execute),
-            Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))),
-            Effect.scoped,
-            Effect.mapError((cause) =>
-              InvalidPushError.make({ reason: { _tag: 'Unexpected', message: cause.toString() } }),
-            ),
-          )
+          // Electric completes the long-poll request after ~20 seconds with a 204 status
+          // In this case we just retry where we left off
+          if (resp.status === 204) {
+            return Option.some([Chunk.empty(), Option.some(nextHandle)] as const)
+          }
 
-          if (!resp.success) {
-            yield* InvalidPushError.make({ reason: { _tag: 'Unexpected', message: 'Push failed' } })
+          const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem), {
+            onExcessProperty: 'preserve',
+          })(resp)
+
+          const items = body
+            .filter((item) => item.value !== undefined && (item.headers as any).operation === 'insert')
+            .map((item) => ({
+              metadata: Option.some({ offset: nextHandle.offset!, handle: nextHandle.handle }),
+              mutationEventEncoded: item.value! as MutationEvent.AnyEncodedGlobal,
+            }))
+
+          // // TODO implement proper `remaining` handling
+          // remaining: 0,
+
+          // if (listenForNew === false && items.length === 0) {
+          //   return Option.none()
+          // }
+
+          for (const item of items) {
+            const deferred = pendingPushDeferredMap.get(item.mutationEventEncoded.id)
+            if (deferred !== undefined) {
+              yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata))
+            }
           }
 
-          const metadata = yield* Effect.all(deferreds, { concurrency: 'unbounded' }).pipe(
-            Effect.map((_) => _.map(Option.some)),
-          )
+          return Option.some([Chunk.fromIterable(items), Option.some(nextHandle)] as const)
+        }).pipe(
+          Effect.scoped,
+          Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })),
+        )
 
-          for (const mutationEventEncoded of batch) {
-            pendingPushDeferredMap.delete(mutationEventEncoded.id)
-          }
+      return {
+        pull: (args) =>
+          Stream.unfoldChunkEffect(
+            args.pipe(
+              Option.map((_) => _.metadata),
+              Option.flatten,
+            ),
+            (metadataOption) => pull(metadataOption),
+          ).pipe(
+            Stream.chunks,
+            Stream.map((chunk) => ({ batch: [...chunk], remaining: 0 })),
+          ),
 
-          return { metadata }
-        }),
-      isConnected,
-    } satisfies SyncBackend<SyncMetadata>
-  })
+        push: (batch) =>
+          Effect.gen(function* () {
+            const deferreds: Deferred.Deferred<SyncMetadata>[] = []
+            for (const mutationEventEncoded of batch) {
+              const deferred = yield* Deferred.make<SyncMetadata>()
+              pendingPushDeferredMap.set(mutationEventEncoded.id, deferred)
+              deferreds.push(deferred)
+            }
+
+            const resp = yield* HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(
+              HttpClientRequest.post(pushEndpoint),
+              ApiSchema.PushPayload.make({ storeId, batch }),
+            ).pipe(
+              Effect.andThen(HttpClient.execute),
+              Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))),
+              Effect.scoped,
+              Effect.mapError((cause) =>
+                InvalidPushError.make({ reason: { _tag: 'Unexpected', message: cause.toString() } }),
+              ),
+            )
+
+            if (!resp.success) {
+              yield* InvalidPushError.make({ reason: { _tag: 'Unexpected', message: 'Push failed' } })
+            }
+
+            const metadata = yield* Effect.all(deferreds, { concurrency: 'unbounded' }).pipe(
+              Effect.map((_) => _.map(Option.some)),
+            )
+
+            for (const mutationEventEncoded of batch) {
+              pendingPushDeferredMap.delete(mutationEventEncoded.id)
+            }
+
+            return { metadata }
+          }),
+        isConnected,
+        metadata: {
+          name: '@livestore/sync-electric',
+          description: 'LiveStore sync backend implementation using ElectricSQL',
+          protocol: 'http',
+          endpoint,
+        },
+      } satisfies SyncBackend<SyncMetadata>
+    })
 
 /**
  * Needs to be bumped when the storage format changes (e.g. mutationLogTable schema changes)
  *
  * Changing this version number will lead to a "soft reset".
  */
-export const PERSISTENCE_FORMAT_VERSION = 3
+export const PERSISTENCE_FORMAT_VERSION = 4
 
 export const toTableName = (storeId: string) => {
   const escapedStoreId = storeId.replaceAll(/[^a-zA-Z0-9_]/g, '_')