@livestore/sync-electric 0.3.0-dev.5 → 0.3.0-dev.50

package/src/index.ts

@@ -1,11 +1,9 @@
-import type { SyncBackend, SyncBackendOptionsBase } from '@livestore/common'
-import { InvalidPullError, InvalidPushError } from '@livestore/common'
-import type { EventId } from '@livestore/common/schema'
-import { MutationEvent } from '@livestore/common/schema'
-import type { Scope } from '@livestore/utils/effect'
+import type { IsOfflineError, SyncBackend, SyncBackendConstructor } from '@livestore/common'
+import { InvalidPullError, InvalidPushError, UnexpectedError } from '@livestore/common'
+import { LiveStoreEvent } from '@livestore/common/schema'
+import { notYetImplemented, shouldNeverHappen } from '@livestore/utils'
 import {
   Chunk,
-  Deferred,
   Effect,
   HttpClient,
   HttpClientRequest,
@@ -16,209 +14,336 @@ import {
   SubscriptionRef,
 } from '@livestore/utils/effect'
 
+import * as ApiSchema from './api-schema.js'
+
+export * as ApiSchema from './api-schema.js'
+
 /*
 Example data:
 
-[{"key":"\"public\".\"events\"/\"1\"","value":{"id":"1","mutation":"test","args_json":"{\"test\":\"test\"}","schema_hash":"1","created_at":"2024-09-07T10:05:31.445Z"},"headers":{"operation":"insert","relation":["public","events"]},"offset":"0_0"}
-,{"key":"\"public\".\"events\"/\"1725703554783\"","value":{"id":"1725703554783","mutation":"test","args_json":"{\"test\":\"test\"}","schema_hash":"1","created_at":"2024-09-07T10:05:54.783Z"},"headers":{"operation":"insert","relation":["public","events"]},"offset":"0_0"}
-,{"headers":{"control":"up-to-date"}}]
+[
+    {
+        "value": {
+            "args": "{\"id\": \"127c3df4-0855-4587-ae75-14463f4a3aa0\", \"text\": \"1\"}",
+            "clientId": "S_YOa",
+            "id": "0",
+            "name": "todoCreated",
+            "parentSeqNum": "-1"
+        },
+        "key": "\"public\".\"events_9069baf0_b3e6_42f7_980f_188416eab3fx3\"/\"0\"",
+        "headers": {
+            "last": true,
+            "relation": [
+                "public",
+                "events_9069baf0_b3e6_42f7_980f_188416eab3fx3"
+            ],
+            "operation": "insert",
+            "lsn": 27294160,
+            "op_position": 0,
+            "txids": [
+                753
+            ]
+        }
+    },
+    {
+        "headers": {
+            "control": "up-to-date",
+            "global_last_seen_lsn": 27294160
+        }
+    }
+]
+
 
 Also see: https://github.com/electric-sql/electric/blob/main/packages/typescript-client/src/client.ts
 
 */
 
+const LiveStoreEventGlobalFromStringRecord = Schema.Struct({
+  seqNum: Schema.NumberFromString,
+  parentSeqNum: Schema.NumberFromString,
+  name: Schema.String,
+  args: Schema.parseJson(Schema.Any),
+  clientId: Schema.String,
+  sessionId: Schema.String,
+}).pipe(
+  Schema.transform(LiveStoreEvent.AnyEncodedGlobal, {
+    decode: (_) => _,
+    encode: (_) => _,
+  }),
+)
+
 const ResponseItem = Schema.Struct({
-  /** Postgres path (e.g. "public.events/1") */
+  /** Postgres path (e.g. `"public"."events_9069baf0_b3e6_42f7_980f_188416eab3fx3"/"0"`) */
   key: Schema.optional(Schema.String),
-  value: Schema.optional(MutationEvent.AnyEncodedGlobal),
-  headers: Schema.Record({ key: Schema.String, value: Schema.Any }),
-  offset: Schema.optional(Schema.String),
+  value: Schema.optional(LiveStoreEventGlobalFromStringRecord),
+  headers: Schema.Union(
+    Schema.Struct({
+      operation: Schema.Union(Schema.Literal('insert'), Schema.Literal('update'), Schema.Literal('delete')),
+      relation: Schema.Array(Schema.String),
+    }),
+    Schema.Struct({
+      control: Schema.String,
+    }),
+  ),
 })
 
 const ResponseHeaders = Schema.Struct({
-  'x-electric-shape-id': Schema.String,
-  // 'x-electric-schema': Schema.parseJson(Schema.Any),
+  'electric-handle': Schema.String,
+  // 'electric-schema': Schema.parseJson(Schema.Any),
   /** e.g. 26799576_0 */
-  'x-electric-chunk-last-offset': Schema.String,
+  'electric-offset': Schema.String,
 })
 
 export const syncBackend = {} as any
 
-export const ApiPushEventPayload = Schema.TaggedStruct('sync-electric.PushEvent', {
-  roomId: Schema.String,
-  batch: Schema.Array(MutationEvent.AnyEncodedGlobal),
-})
-
-export const ApiInitRoomPayload = Schema.TaggedStruct('sync-electric.InitRoom', {
-  roomId: Schema.String,
-})
-
-export const ApiPayload = Schema.Union(ApiPushEventPayload, ApiInitRoomPayload)
-
 export const syncBackendOptions = <TOptions extends SyncBackendOptions>(options: TOptions) => options
 
-export interface SyncBackendOptions extends SyncBackendOptionsBase {
-  type: 'electric'
-  /**
-   * The host of the Electric server
-   *
-   * @example "https://localhost:3000"
-   */
+/**
+ * This function should be called in a trusted environment (e.g. a proxy server) as it
+ * requires access to senstive information (e.g. `apiSecret` / `sourceSecret`).
+ */
+export const makeElectricUrl = ({
+  electricHost,
+  searchParams: providedSearchParams,
+  sourceId,
+  sourceSecret,
+  apiSecret,
+}: {
   electricHost: string
-  roomId: string
   /**
-   * The POST endpoint to push events to
-   *
-   * @example "/api/push-event"
-   * @example "https://api.myapp.com/push-event"
+   * Needed to extract information from the search params which the `@livestore/sync-electric`
+   * client implementation automatically adds:
+   * - `handle`: the ElectricSQL handle
+   * - `storeId`: the Livestore storeId
    */
-  pushEventEndpoint: string
+  searchParams: URLSearchParams
+  /** Needed for Electric Cloud */
+  sourceId?: string
+  /** Needed for Electric Cloud */
+  sourceSecret?: string
+  /** For self-hosted ElectricSQL */
+  apiSecret?: string
+}) => {
+  const endpointUrl = `${electricHost}/v1/shape`
+  const argsResult = Schema.decodeUnknownEither(Schema.Struct({ args: Schema.parseJson(ApiSchema.PullPayload) }))(
+    Object.fromEntries(providedSearchParams.entries()),
+  )
+
+  if (argsResult._tag === 'Left') {
+    return shouldNeverHappen(
+      'Invalid search params provided to makeElectricUrl',
+      providedSearchParams,
+      Object.fromEntries(providedSearchParams.entries()),
+    )
+  }
+
+  const args = argsResult.right.args
+  const tableName = toTableName(args.storeId)
+  const searchParams = new URLSearchParams()
+  searchParams.set('table', tableName)
+  if (sourceId !== undefined) {
+    searchParams.set('source_id', sourceId)
+  }
+  if (sourceSecret !== undefined) {
+    searchParams.set('source_secret', sourceSecret)
+  }
+  if (apiSecret !== undefined) {
+    searchParams.set('api_secret', apiSecret)
+  }
+  if (args.handle._tag === 'None') {
+    searchParams.set('offset', '-1')
+  } else {
+    searchParams.set('offset', args.handle.value.offset)
+    searchParams.set('handle', args.handle.value.handle)
+    searchParams.set('live', 'true')
+  }
+
+  const payload = args.payload
+
+  const url = `${endpointUrl}?${searchParams.toString()}`
+
+  return { url, storeId: args.storeId, needsInit: args.handle._tag === 'None', payload }
 }
 
-interface LiveStoreGlobalElectric {
-  syncBackend: SyncBackendOptions
+export interface SyncBackendOptions {
+  /**
+   * The endpoint to pull/push events. Pull is a `GET` request, push is a `POST` request.
+   * Usually this endpoint is part of your API layer to proxy requests to the Electric server
+   * e.g. to implement auth, rate limiting, etc.
+   *
+   * @example "/api/electric"
+   * @example { push: "/api/push-event", pull: "/api/pull-event" }
+   */
+  endpoint:
+    | string
+    | {
+        push: string
+        pull: string
+      }
 }
 
-declare global {
-  interface LiveStoreGlobal extends LiveStoreGlobalElectric {}
-}
+export const SyncMetadata = Schema.Struct({
+  offset: Schema.String,
+  // TODO move this into some kind of "global" sync metadata as it's the same for each event
+  handle: Schema.String,
+})
 
 type SyncMetadata = {
   offset: string
   // TODO move this into some kind of "global" sync metadata as it's the same for each event
-  shapeId: string
+  handle: string
 }
 
-export const makeSyncBackend = ({
-  electricHost,
-  roomId,
-  pushEventEndpoint,
-}: SyncBackendOptions): Effect.Effect<SyncBackend<SyncMetadata>, never, Scope.Scope> =>
-  Effect.gen(function* () {
-    const endpointUrl = `${electricHost}/v1/shape/events_${roomId}`
-
-    const isConnected = yield* SubscriptionRef.make(true)
-
-    const initRoom = HttpClientRequest.schemaBodyJson(ApiInitRoomPayload)(
-      HttpClientRequest.post(pushEventEndpoint),
-      ApiInitRoomPayload.make({ roomId }),
-    ).pipe(Effect.andThen(HttpClient.execute))
-
-    // TODO check whether we still need this
-    const pendingPushDeferredMap = new Map<EventId.GlobalEventId, Deferred.Deferred<SyncMetadata>>()
-
-    const pull = (args: Option.Option<SyncMetadata>) =>
-      Effect.gen(function* () {
-        const url =
-          args._tag === 'None'
-            ? `${endpointUrl}?offset=-1`
-            : `${endpointUrl}?offset=${args.value.offset}&shape_id=${args.value.shapeId}&live=true`
-
-        const resp = yield* HttpClient.get(url).pipe(
-          Effect.tapErrorTag('ResponseError', (error) =>
-            // TODO handle 409 error when the shapeId you request no longer exists for whatever reason.
-            // The correct behavior here is to refetch the shape from scratch and to reset the local state.
-            error.response.status === 400 ? initRoom : Effect.fail(error),
-          ),
-          Effect.retry({ times: 1 }),
-        )
-
-        const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp)
-        const nextCursor = {
-          offset: headers['x-electric-chunk-last-offset'],
-          shapeId: headers['x-electric-shape-id'],
-        }
-
-        // Electric completes the long-poll request after ~20 seconds with a 204 status
-        // In this case we just retry where we left off
-        if (resp.status === 204) {
-          return Option.some([Chunk.empty(), Option.some(nextCursor)] as const)
-        }
-
-        const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp)
-
-        const items = body
-          .filter((item) => item.value !== undefined)
-          .map((item) => ({
-            metadata: Option.some({ offset: item.offset!, shapeId: nextCursor.shapeId }),
-            mutationEventEncoded: {
-              mutation: item.value!.mutation,
-              args: JSON.parse(item.value!.args),
-              id: item.value!.id,
-              parentId: item.value!.parentId,
-            },
-          }))
-
-        // // TODO implement proper `remaining` handling
-        // remaining: 0,
-
-        // if (listenForNew === false && items.length === 0) {
-        //   return Option.none()
-        // }
-
-        const [newItems, pendingPushItems] = Chunk.fromIterable(items).pipe(
-          Chunk.partition((item) => pendingPushDeferredMap.has(item.mutationEventEncoded.id)),
-        )
-
-        for (const item of pendingPushItems) {
-          const deferred = pendingPushDeferredMap.get(item.mutationEventEncoded.id)!
-          yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata))
-        }
-
-        return Option.some([newItems, Option.some(nextCursor)] as const)
-      }).pipe(
-        Effect.scoped,
-        Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })),
-      )
-
-    return {
-      pull: (args) =>
-        Stream.unfoldChunkEffect(
-          args.pipe(
-            Option.map((_) => _.metadata),
-            Option.flatten,
-          ),
-          (metadataOption) => pull(metadataOption),
-        ).pipe(
-          Stream.chunks,
-          Stream.map((chunk) => ({ batch: [...chunk], remaining: 0 })),
-        ),
-
-      push: (batch) =>
+export const makeSyncBackend =
+  ({ endpoint }: SyncBackendOptions): SyncBackendConstructor<SyncMetadata> =>
+  ({ storeId, payload }) =>
+    Effect.gen(function* () {
+      const isConnected = yield* SubscriptionRef.make(true)
+      const pullEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.pull
+      const pushEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.push
+
+      const pull = (
+        handle: Option.Option<SyncMetadata>,
+      ): Effect.Effect<
+        Option.Option<
+          readonly [
+            Chunk.Chunk<{
+              metadata: Option.Option<SyncMetadata>
+              eventEncoded: LiveStoreEvent.AnyEncodedGlobal
+            }>,
+            Option.Option<SyncMetadata>,
+          ]
+        >,
+        InvalidPullError | IsOfflineError,
+        HttpClient.HttpClient
+      > =>
         Effect.gen(function* () {
-          const deferreds: Deferred.Deferred<SyncMetadata>[] = []
-          for (const mutationEventEncoded of batch) {
-            const deferred = yield* Deferred.make<SyncMetadata>()
-            pendingPushDeferredMap.set(mutationEventEncoded.id, deferred)
-            deferreds.push(deferred)
+          const argsJson = yield* Schema.encode(Schema.parseJson(ApiSchema.PullPayload))(
+            ApiSchema.PullPayload.make({ storeId, handle, payload }),
+          )
+          const url = `${pullEndpoint}?args=${argsJson}`
+
+          const resp = yield* HttpClient.get(url)
+
+          if (resp.status === 401) {
+            const body = yield* resp.text.pipe(Effect.catchAll(() => Effect.succeed('-')))
+            return yield* InvalidPullError.make({
+              message: `Unauthorized (401): Couldn't connect to ElectricSQL: ${body}`,
+            })
+          } else if (resp.status === 409) {
+            // https://electric-sql.com/openapi.html#/paths/~1v1~1shape/get
+            // {
+            // "message": "The shape associated with this shape_handle and offset was not found. Resync to fetch the latest shape",
+            // "shape_handle": "2494_84241",
+            // "offset": "-1"
+            // }
+
+            // TODO: implementation plan:
+            // start pulling events from scratch with the new handle and ignore the "old events"
+            // until we found a new event, then, continue with the new handle
+            return notYetImplemented(`Electric shape not found`)
+          } else if (resp.status < 200 || resp.status >= 300) {
+            return yield* InvalidPullError.make({
+              message: `Unexpected status code: ${resp.status}`,
+            })
           }
 
-          const resp = yield* HttpClientRequest.schemaBodyJson(ApiPushEventPayload)(
-            HttpClientRequest.post(pushEventEndpoint),
-            ApiPushEventPayload.make({ roomId, batch }),
-          ).pipe(
-            Effect.andThen(HttpClient.execute),
-            Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))),
-            Effect.scoped,
-            Effect.mapError((cause) =>
-              InvalidPushError.make({ reason: { _tag: 'Unexpected', message: cause.toString() } }),
-            ),
-          )
+          const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp)
+          const nextHandle = {
+            offset: headers['electric-offset'],
+            handle: headers['electric-handle'],
+          }
 
-          if (!resp.success) {
-            yield* InvalidPushError.make({ reason: { _tag: 'Unexpected', message: 'Push failed' } })
+          // Electric completes the long-poll request after ~20 seconds with a 204 status
+          // In this case we just retry where we left off
+          if (resp.status === 204) {
+            return Option.some([Chunk.empty(), Option.some(nextHandle)] as const)
           }
 
-          const metadata = yield* Effect.all(deferreds, { concurrency: 'unbounded' }).pipe(
-            Effect.map((_) => _.map(Option.some)),
-          )
+          const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem), {
+            onExcessProperty: 'preserve',
+          })(resp)
+
+          const items = body
+            .filter((item) => item.value !== undefined && (item.headers as any).operation === 'insert')
+            .map((item) => ({
+              metadata: Option.some({ offset: nextHandle.offset!, handle: nextHandle.handle }),
+              eventEncoded: item.value! as LiveStoreEvent.AnyEncodedGlobal,
+            }))
+
+          // // TODO implement proper `remaining` handling
+          // remaining: 0,
+
+          // if (listenForNew === false && items.length === 0) {
+          //   return Option.none()
+          // }
+
+          return Option.some([Chunk.fromIterable(items), Option.some(nextHandle)] as const)
+        }).pipe(
+          Effect.scoped,
+          Effect.mapError((cause) =>
+            cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ message: cause.toString() }),
+          ),
+        )
 
-          for (const mutationEventEncoded of batch) {
-            pendingPushDeferredMap.delete(mutationEventEncoded.id)
-          }
+      const pullEndpointHasSameOrigin =
+        pullEndpoint.startsWith('/') ||
+        (globalThis.location !== undefined && globalThis.location.origin === new URL(pullEndpoint).origin)
+
+      return {
+        // If the pull endpoint has the same origin as the current page, we can assume that we already have a connection
+        // otherwise we send a HEAD request to speed up the connection process
+        connect: pullEndpointHasSameOrigin
+          ? Effect.void
+          : HttpClient.head(pullEndpoint).pipe(UnexpectedError.mapToUnexpectedError),
+        pull: (args) =>
+          Stream.unfoldChunkEffect(
+            args.pipe(
+              Option.map((_) => _.metadata),
+              Option.flatten,
+            ),
+            (metadataOption) => pull(metadataOption),
+          ).pipe(
+            Stream.chunks,
+            Stream.map((chunk) => ({ batch: [...chunk], remaining: 0 })),
+          ),
 
-          return { metadata }
-        }),
-      isConnected,
-    } satisfies SyncBackend<SyncMetadata>
-  })
+        push: (batch) =>
+          Effect.gen(function* () {
+            const resp = yield* HttpClientRequest.schemaBodyJson(ApiSchema.PushPayload)(
+              HttpClientRequest.post(pushEndpoint),
+              ApiSchema.PushPayload.make({ storeId, batch }),
+            ).pipe(
+              Effect.andThen(HttpClient.execute),
+              Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))),
+              Effect.scoped,
+              Effect.mapError((cause) =>
+                InvalidPushError.make({ reason: { _tag: 'Unexpected', message: cause.toString() } }),
+              ),
+            )
+
+            if (!resp.success) {
+              yield* InvalidPushError.make({ reason: { _tag: 'Unexpected', message: 'Push failed' } })
+            }
+          }),
+        isConnected,
+        metadata: {
+          name: '@livestore/sync-electric',
+          description: 'LiveStore sync backend implementation using ElectricSQL',
+          protocol: 'http',
+          endpoint,
+        },
+      } satisfies SyncBackend<SyncMetadata>
+    })
+
+/**
+ * Needs to be bumped when the storage format changes (e.g. eventlogTable schema changes)
+ *
+ * Changing this version number will lead to a "soft reset".
+ */
+export const PERSISTENCE_FORMAT_VERSION = 6
+
+export const toTableName = (storeId: string) => {
+  const escapedStoreId = storeId.replaceAll(/[^a-zA-Z0-9_]/g, '_')
+  return `eventlog_${PERSISTENCE_FORMAT_VERSION}_${escapedStoreId}`
+}

package/tsconfig.json

@@ -1,10 +0,0 @@
-{
-  "extends": "../../../tsconfig.base.json",
-  "compilerOptions": {
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "tsBuildInfoFile": "./dist/.tsbuildinfo"
-  },
-  "include": ["./src"],
-  "references": [{ "path": "../common" }, { "path": "../utils" }]
-}