@livestore/sync-electric 0.3.0-dev.2 → 0.3.0-dev.22

package/src/index.ts

@@ -1,7 +1,8 @@
-import type { SyncBackend, SyncBackendOptionsBase } from '@livestore/common'
+import type { IsOfflineError, SyncBackend } from '@livestore/common'
 import { InvalidPullError, InvalidPushError } from '@livestore/common'
 import type { EventId } from '@livestore/common/schema'
 import { MutationEvent } from '@livestore/common/schema'
+import { notYetImplemented, shouldNeverHappen } from '@livestore/utils'
 import type { Scope } from '@livestore/utils/effect'
 import {
   Chunk,
@@ -16,137 +17,238 @@ import {
   SubscriptionRef,
 } from '@livestore/utils/effect'
 
+import * as ApiSchema from './api-schema.js'
+
+export * as ApiSchema from './api-schema.js'
+
 /*
 Example data:
 
-[{"key":"\"public\".\"events\"/\"1\"","value":{"id":"1","mutation":"test","args_json":"{\"test\":\"test\"}","schema_hash":"1","created_at":"2024-09-07T10:05:31.445Z"},"headers":{"operation":"insert","relation":["public","events"]},"offset":"0_0"}
-,{"key":"\"public\".\"events\"/\"1725703554783\"","value":{"id":"1725703554783","mutation":"test","args_json":"{\"test\":\"test\"}","schema_hash":"1","created_at":"2024-09-07T10:05:54.783Z"},"headers":{"operation":"insert","relation":["public","events"]},"offset":"0_0"}
-,{"headers":{"control":"up-to-date"}}]
+[
+    {
+        "value": {
+            "args": "{\"id\": \"127c3df4-0855-4587-ae75-14463f4a3aa0\", \"text\": \"1\"}",
+            "clientId": "S_YOa",
+            "id": "0",
+            "mutation": "todoCreated",
+            "parentId": "-1"
+        },
+        "key": "\"public\".\"events_9069baf0_b3e6_42f7_980f_188416eab3fx3\"/\"0\"",
+        "headers": {
+            "last": true,
+            "relation": [
+                "public",
+                "events_9069baf0_b3e6_42f7_980f_188416eab3fx3"
+            ],
+            "operation": "insert",
+            "lsn": 27294160,
+            "op_position": 0,
+            "txids": [
+                753
+            ]
+        }
+    },
+    {
+        "headers": {
+            "control": "up-to-date",
+            "global_last_seen_lsn": 27294160
+        }
+    }
+]
+
 
 Also see: https://github.com/electric-sql/electric/blob/main/packages/typescript-client/src/client.ts
 
 */
 
+const MutationEventGlobalFromStringRecord = Schema.Struct({
+  id: Schema.NumberFromString,
+  parentId: Schema.NumberFromString,
+  mutation: Schema.String,
+  args: Schema.parseJson(Schema.Any),
+  clientId: Schema.String,
+  sessionId: Schema.optional(Schema.String),
+}).pipe(
+  Schema.transform(MutationEvent.AnyEncodedGlobal, {
+    decode: (_) => _,
+    encode: (_) => _,
+  }),
+)
+
 const ResponseItem = Schema.Struct({
-  /** Postgres path (e.g. "public.events/1") */
+  /** Postgres path (e.g. `"public"."events_9069baf0_b3e6_42f7_980f_188416eab3fx3"/"0"`) */
   key: Schema.optional(Schema.String),
-  value: Schema.optional(MutationEvent.EncodedAny),
-  headers: Schema.Record({ key: Schema.String, value: Schema.Any }),
-  offset: Schema.optional(Schema.String),
+  value: Schema.optional(MutationEventGlobalFromStringRecord),
+  headers: Schema.Union(
+    Schema.Struct({
+      operation: Schema.Union(Schema.Literal('insert'), Schema.Literal('update'), Schema.Literal('delete')),
+      relation: Schema.Array(Schema.String),
+    }),
+    Schema.Struct({
+      control: Schema.String,
+    }),
+  ),
 })
 
 const ResponseHeaders = Schema.Struct({
-  'x-electric-shape-id': Schema.String,
-  // 'x-electric-schema': Schema.parseJson(Schema.Any),
+  'electric-handle': Schema.String,
+  // 'electric-schema': Schema.parseJson(Schema.Any),
   /** e.g. 26799576_0 */
-  'x-electric-chunk-last-offset': Schema.String,
+  'electric-offset': Schema.String,
 })
 
 export const syncBackend = {} as any
 
-export const ApiPushEventPayload = Schema.TaggedStruct('sync-electric.PushEvent', {
-  roomId: Schema.String,
-  batch: Schema.Array(MutationEvent.EncodedAny),
-})
-
-export const ApiInitRoomPayload = Schema.TaggedStruct('sync-electric.InitRoom', {
-  roomId: Schema.String,
-})
-
-export const ApiPayload = Schema.Union(ApiPushEventPayload, ApiInitRoomPayload)
-
 export const syncBackendOptions = <TOptions extends SyncBackendOptions>(options: TOptions) => options
 
-export interface SyncBackendOptions extends SyncBackendOptionsBase {
-  type: 'electric'
-  /**
-   * The host of the Electric server
-   *
-   * @example "https://localhost:3000"
-   */
+export const makeElectricUrl = ({
+  electricHost,
+  searchParams: providedSearchParams,
+  sourceId,
+  sourceSecret,
+}: {
   electricHost: string
-  roomId: string
   /**
-   * The POST endpoint to push events to
-   *
-   * @example "/api/push-event"
-   * @example "https://api.myapp.com/push-event"
+   * Needed to extract information from the search params which the `@livestore/sync-electric`
+   * client implementation automatically adds:
+   * - `handle`: the ElectricSQL handle
+   * - `storeId`: the Livestore storeId
    */
-  pushEventEndpoint: string
+  searchParams: URLSearchParams
+  /** Needed for Electric Cloud */
+  sourceId?: string
+  /** Needed for Electric Cloud */
+  sourceSecret?: string
+}) => {
+  const endpointUrl = `${electricHost}/v1/shape`
+  const argsResult = Schema.decodeUnknownEither(Schema.Struct({ args: Schema.parseJson(ApiSchema.PullPayload) }))(
+    Object.fromEntries(providedSearchParams.entries()),
+  )
+
+  if (argsResult._tag === 'Left') {
+    return shouldNeverHappen('Invalid search params provided to makeElectricUrl', providedSearchParams)
+  }
+
+  const args = argsResult.right.args
+  const tableName = toTableName(args.storeId)
+  const searchParams = new URLSearchParams()
+  searchParams.set('table', tableName)
+  if (sourceId !== undefined) {
+    searchParams.set('source_id', sourceId)
+  }
+  if (sourceSecret !== undefined) {
+    searchParams.set('source_secret', sourceSecret)
+  }
+  if (args.handle._tag === 'None') {
+    searchParams.set('offset', '-1')
+  } else {
+    searchParams.set('offset', args.handle.value.offset)
+    searchParams.set('handle', args.handle.value.handle)
+    searchParams.set('live', 'true')
+  }
+
+  const url = `${endpointUrl}?${searchParams.toString()}`
+
+  return { url, storeId: args.storeId, needsInit: args.handle._tag === 'None' }
 }
 
-interface LiveStoreGlobalElectric {
-  syncBackend: SyncBackendOptions
+export interface SyncBackendOptions {
+  storeId: string
+  /**
+   * The endpoint to pull/push events. Pull is a `GET` request, push is a `POST` request.
+   * Usually this endpoint is part of your API layer to proxy requests to the Electric server
+   * e.g. to implement auth, rate limiting, etc.
+   *
+   * @example "/api/electric"
+   * @example { push: "/api/push-event", pull: "/api/pull-event" }
+   */
+  endpoint:
+    | string
+    | {
+        push: string
+        pull: string
+      }
 }
 
-declare global {
-  interface LiveStoreGlobal extends LiveStoreGlobalElectric {}
-}
+export const SyncMetadata = Schema.Struct({
+  offset: Schema.String,
+  // TODO move this into some kind of "global" sync metadata as it's the same for each event
+  handle: Schema.String,
+})
 
 type SyncMetadata = {
   offset: string
   // TODO move this into some kind of "global" sync metadata as it's the same for each event
-  shapeId: string
+  handle: string
 }
 
 export const makeSyncBackend = ({
-  electricHost,
-  roomId,
-  pushEventEndpoint,
+  storeId,
+  endpoint,
 }: SyncBackendOptions): Effect.Effect<SyncBackend<SyncMetadata>, never, Scope.Scope> =>
   Effect.gen(function* () {
-    const endpointUrl = `${electricHost}/v1/shape/events_${roomId}`
-
     const isConnected = yield* SubscriptionRef.make(true)
-
-    const initRoom = HttpClientRequest.schemaBodyJson(ApiInitRoomPayload)(
-      HttpClientRequest.post(pushEventEndpoint),
-      ApiInitRoomPayload.make({ roomId }),
-    ).pipe(Effect.andThen(HttpClient.execute))
+    const pullEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.pull
+    const pushEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.push
 
     // TODO check whether we still need this
-    const pendingPushDeferredMap = new Map<string, Deferred.Deferred<SyncMetadata>>()
-
-    const pull = (args: Option.Option<SyncMetadata>) =>
+    const pendingPushDeferredMap = new Map<EventId.GlobalEventId, Deferred.Deferred<SyncMetadata>>()
+
+    const pull = (
+      handle: Option.Option<SyncMetadata>,
+    ): Effect.Effect<
+      Option.Option<
+        readonly [
+          Chunk.Chunk<{ metadata: Option.Option<SyncMetadata>; mutationEventEncoded: MutationEvent.AnyEncodedGlobal }>,
+          Option.Option<SyncMetadata>,
+        ]
+      >,
+      InvalidPullError | IsOfflineError,
+      HttpClient.HttpClient
+    > =>
       Effect.gen(function* () {
-        const url =
-          args._tag === 'None'
-            ? `${endpointUrl}?offset=-1`
-            : `${endpointUrl}?offset=${args.value.offset}&shape_id=${args.value.shapeId}&live=true`
-
-        const resp = yield* HttpClient.get(url).pipe(
-          Effect.tapErrorTag('ResponseError', (error) =>
-            // TODO handle 409 error when the shapeId you request no longer exists for whatever reason.
-            // The correct behavior here is to refetch the shape from scratch and to reset the local state.
-            error.response.status === 400 ? initRoom : Effect.fail(error),
-          ),
-          Effect.retry({ times: 1 }),
+        const argsJson = yield* Schema.encode(Schema.parseJson(ApiSchema.PullPayload))(
+          ApiSchema.PullPayload.make({ storeId, handle }),
         )
+        const url = `${pullEndpoint}?args=${argsJson}`
+
+        const resp = yield* HttpClient.get(url)
 
         const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp)
-        const nextCursor = {
-          offset: headers['x-electric-chunk-last-offset'],
-          shapeId: headers['x-electric-shape-id'],
+        const nextHandle = {
+          offset: headers['electric-offset'],
+          handle: headers['electric-handle'],
+        }
+
+        // TODO handle case where Electric shape is not found for a given handle
+        // https://electric-sql.com/openapi.html#/paths/~1v1~1shape/get
+        // {
+        // "message": "The shape associated with this shape_handle and offset was not found. Resync to fetch the latest shape",
+        // "shape_handle": "2494_84241",
+        // "offset": "-1"
+        // }
+        if (resp.status === 409) {
+          // TODO: implementation plan:
+          // start pulling events from scratch with the new handle and ignore the "old events"
+          // until we found a new event, then, continue with the new handle
+          return notYetImplemented(`Electric shape not found for handle ${nextHandle.handle}`)
         }
 
         // Electric completes the long-poll request after ~20 seconds with a 204 status
         // In this case we just retry where we left off
         if (resp.status === 204) {
-          return Option.some([Chunk.empty(), Option.some(nextCursor)] as const)
+          return Option.some([Chunk.empty(), Option.some(nextHandle)] as const)
         }
 
-        const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp)
+        const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem), {
+          onExcessProperty: 'preserve',
+        })(resp)
 
         const items = body
-          .filter((item) => item.value !== undefined)
+          .filter((item) => item.value !== undefined && (item.headers as any).operation === 'insert')
           .map((item) => ({
-            metadata: Option.some({ offset: item.offset!, shapeId: nextCursor.shapeId }),
-            mutationEventEncoded: {
-              mutation: item.value!.mutation,
-              args: JSON.parse(item.value!.args),
-              id: item.value!.id,
-              parentId: item.value!.parentId,
-            },
+            metadata: Option.some({ offset: nextHandle.offset!, handle: nextHandle.handle }),
+            mutationEventEncoded: item.value! as MutationEvent.AnyEncodedGlobal,
           }))
 
         // // TODO implement proper `remaining` handling
@@ -156,16 +258,14 @@ export const makeSyncBackend = ({
         //   return Option.none()
         // }
 
-        const [newItems, pendingPushItems] = Chunk.fromIterable(items).pipe(
-          Chunk.partition((item) => pendingPushDeferredMap.has(eventIdToString(item.mutationEventEncoded.id))),
-        )
-
-        for (const item of pendingPushItems) {
-          const deferred = pendingPushDeferredMap.get(eventIdToString(item.mutationEventEncoded.id))!
-          yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata))
+        for (const item of items) {
+          const deferred = pendingPushDeferredMap.get(item.mutationEventEncoded.id)
+          if (deferred !== undefined) {
+            yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata))
+          }
         }
 
-        return Option.some([newItems, Option.some(nextCursor)] as const)
+        return Option.some([Chunk.fromIterable(items), Option.some(nextHandle)] as const)
       }).pipe(
         Effect.scoped,
         Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })),
@@ -189,13 +289,13 @@ export const makeSyncBackend = ({
           const deferreds: Deferred.Deferred<SyncMetadata>[] = []
           for (const mutationEventEncoded of batch) {
             const deferred = yield* Deferred.make<SyncMetadata>()
-            pendingPushDeferredMap.set(eventIdToString(mutationEventEncoded.id), deferred)
+            pendingPushDeferredMap.set(mutationEventEncoded.id, deferred)
             deferreds.push(deferred)
           }
 
-          const resp = yield* HttpClientRequest.schemaBodyJson(ApiPushEventPayload)(
-            HttpClientRequest.post(pushEventEndpoint),
-            ApiPushEventPayload.make({ roomId, batch }),
+          const resp = yield* HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(
+            HttpClientRequest.post(pushEndpoint),
+            ApiSchema.PushPayload.make({ storeId, batch }),
           ).pipe(
             Effect.andThen(HttpClient.execute),
             Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))),
@@ -214,7 +314,7 @@ export const makeSyncBackend = ({
           )
 
           for (const mutationEventEncoded of batch) {
-            pendingPushDeferredMap.delete(eventIdToString(mutationEventEncoded.id))
+            pendingPushDeferredMap.delete(mutationEventEncoded.id)
           }
 
           return { metadata }
@@ -223,4 +323,14 @@ export const makeSyncBackend = ({
     } satisfies SyncBackend<SyncMetadata>
   })
 
-const eventIdToString = (eventId: EventId.EventId) => `${eventId.global}_${eventId.local}`
+/**
+ * Needs to be bumped when the storage format changes (e.g. mutationLogTable schema changes)
+ *
+ * Changing this version number will lead to a "soft reset".
+ */
+export const PERSISTENCE_FORMAT_VERSION = 3
+
+export const toTableName = (storeId: string) => {
+  const escapedStoreId = storeId.replaceAll(/[^a-zA-Z0-9_]/g, '_')
+  return `mutation_log_${PERSISTENCE_FORMAT_VERSION}_${escapedStoreId}`
+}