@livestore/sync-electric 0.4.0-dev.0 → 0.4.0-dev.10

package/src/index.ts

@@ -1,14 +1,21 @@
-import type { IsOfflineError, SyncBackend, SyncBackendConstructor } from '@livestore/common'
-import { InvalidPullError, InvalidPushError, UnexpectedError } from '@livestore/common'
+import {
+  InvalidPullError,
+  InvalidPushError,
+  type IsOfflineError,
+  SyncBackend,
+  UnexpectedError,
+} from '@livestore/common'
 import { LiveStoreEvent } from '@livestore/common/schema'
-import { notYetImplemented, shouldNeverHappen } from '@livestore/utils'
+import { notYetImplemented } from '@livestore/utils'
 import {
-  Chunk,
+  type Duration,
   Effect,
   HttpClient,
   HttpClientRequest,
   HttpClientResponse,
   Option,
+  ReadonlyArray,
+  Schedule,
   Schema,
   Stream,
   SubscriptionRef,
@@ -16,7 +23,13 @@ import {
 
 import * as ApiSchema from './api-schema.ts'
 
+export class InvalidOperationError extends Schema.TaggedError<InvalidOperationError>()('InvalidOperationError', {
+  operation: Schema.Union(Schema.Literal('delete'), Schema.Literal('update')),
+  message: Schema.String,
+}) {}
+
 export * as ApiSchema from './api-schema.ts'
+export * from './make-electric-url.ts'
 
 /*
 Example data:
@@ -65,27 +78,36 @@ const LiveStoreEventGlobalFromStringRecord = Schema.Struct({
   args: Schema.parseJson(Schema.Any),
   clientId: Schema.String,
   sessionId: Schema.String,
-}).pipe(
-  Schema.transform(LiveStoreEvent.AnyEncodedGlobal, {
-    decode: (_) => _,
-    encode: (_) => _,
-  }),
-)
-
-const ResponseItem = Schema.Struct({
+})
+  .pipe(
+    Schema.transform(LiveStoreEvent.AnyEncodedGlobal, {
+      decode: (_) => _,
+      encode: (_) => _,
+    }),
+  )
+  .annotations({ title: '@livestore/sync-electric:LiveStoreEventGlobalFromStringRecord' })
+
+const ResponseItemInsert = Schema.Struct({
   /** Postgres path (e.g. `"public"."events_9069baf0_b3e6_42f7_980f_188416eab3fx3"/"0"`) */
   key: Schema.optional(Schema.String),
-  value: Schema.optional(LiveStoreEventGlobalFromStringRecord),
-  headers: Schema.Union(
-    Schema.Struct({
-      operation: Schema.Union(Schema.Literal('insert'), Schema.Literal('update'), Schema.Literal('delete')),
-      relation: Schema.Array(Schema.String),
-    }),
-    Schema.Struct({
-      control: Schema.String,
-    }),
-  ),
-})
+  value: LiveStoreEventGlobalFromStringRecord,
+  headers: Schema.Struct({ operation: Schema.Literal('insert'), relation: Schema.Array(Schema.String) }),
+}).annotations({ title: '@livestore/sync-electric:ResponseItemInsert' })
+
+const ResponseItemInvalid = Schema.Struct({
+  /** Postgres path (e.g. `"public"."events_9069baf0_b3e6_42f7_980f_188416eab3fx3"/"0"`) */
+  key: Schema.optional(Schema.String),
+  value: Schema.Any,
+  headers: Schema.Struct({ operation: Schema.Literal('update', 'delete'), relation: Schema.Array(Schema.String) }),
+}).annotations({ title: '@livestore/sync-electric:ResponseItemInvalid' })
+
+const ResponseItemControl = Schema.Struct({
+  key: Schema.optional(Schema.String),
+  value: Schema.optional(Schema.Any),
+  headers: Schema.Struct({ control: Schema.String }),
+}).annotations({ title: '@livestore/sync-electric:ResponseItemControl' })
+
+const ResponseItem = Schema.Union(ResponseItemInsert, ResponseItemInvalid, ResponseItemControl)
 
 const ResponseHeaders = Schema.Struct({
   'electric-handle': Schema.String,
@@ -94,77 +116,8 @@ const ResponseHeaders = Schema.Struct({
   'electric-offset': Schema.String,
 })
 
-export const syncBackend = {} as any
-
 export const syncBackendOptions = <TOptions extends SyncBackendOptions>(options: TOptions) => options
 
-/**
- * This function should be called in a trusted environment (e.g. a proxy server) as it
- * requires access to senstive information (e.g. `apiSecret` / `sourceSecret`).
- */
-export const makeElectricUrl = ({
-  electricHost,
-  searchParams: providedSearchParams,
-  sourceId,
-  sourceSecret,
-  apiSecret,
-}: {
-  electricHost: string
-  /**
-   * Needed to extract information from the search params which the `@livestore/sync-electric`
-   * client implementation automatically adds:
-   * - `handle`: the ElectricSQL handle
-   * - `storeId`: the Livestore storeId
-   */
-  searchParams: URLSearchParams
-  /** Needed for Electric Cloud */
-  sourceId?: string
-  /** Needed for Electric Cloud */
-  sourceSecret?: string
-  /** For self-hosted ElectricSQL */
-  apiSecret?: string
-}) => {
-  const endpointUrl = `${electricHost}/v1/shape`
-  const argsResult = Schema.decodeUnknownEither(Schema.Struct({ args: Schema.parseJson(ApiSchema.PullPayload) }))(
-    Object.fromEntries(providedSearchParams.entries()),
-  )
-
-  if (argsResult._tag === 'Left') {
-    return shouldNeverHappen(
-      'Invalid search params provided to makeElectricUrl',
-      providedSearchParams,
-      Object.fromEntries(providedSearchParams.entries()),
-    )
-  }
-
-  const args = argsResult.right.args
-  const tableName = toTableName(args.storeId)
-  const searchParams = new URLSearchParams()
-  searchParams.set('table', tableName)
-  if (sourceId !== undefined) {
-    searchParams.set('source_id', sourceId)
-  }
-  if (sourceSecret !== undefined) {
-    searchParams.set('source_secret', sourceSecret)
-  }
-  if (apiSecret !== undefined) {
-    searchParams.set('api_secret', apiSecret)
-  }
-  if (args.handle._tag === 'None') {
-    searchParams.set('offset', '-1')
-  } else {
-    searchParams.set('offset', args.handle.value.offset)
-    searchParams.set('handle', args.handle.value.handle)
-    searchParams.set('live', 'true')
-  }
-
-  const payload = args.payload
-
-  const url = `${endpointUrl}?${searchParams.toString()}`
-
-  return { url, storeId: args.storeId, needsInit: args.handle._tag === 'None', payload }
-}
-
 export interface SyncBackendOptions {
   /**
    * The endpoint to pull/push events. Pull is a `GET` request, push is a `POST` request.
@@ -179,7 +132,25 @@ export interface SyncBackendOptions {
     | {
         push: string
         pull: string
+        ping: string
       }
+
+  ping?: {
+    /**
+     * @default true
+     */
+    enabled?: boolean
+    /**
+     * How long to wait for a ping response before timing out
+     * @default 10 seconds
+     */
+    requestTimeout?: Duration.DurationInput
+    /**
+     * How often to send ping requests
+     * @default 10 seconds
+     */
+    requestInterval?: Duration.DurationInput
+  }
 }
 
 export const SyncMetadata = Schema.Struct({
@@ -188,48 +159,53 @@ export const SyncMetadata = Schema.Struct({
   handle: Schema.String,
 })
 
-type SyncMetadata = {
-  offset: string
-  // TODO move this into some kind of "global" sync metadata as it's the same for each event
-  handle: string
-}
+export type SyncMetadata = typeof SyncMetadata.Type
 
 export const makeSyncBackend =
-  ({ endpoint }: SyncBackendOptions): SyncBackendConstructor<SyncMetadata> =>
+  ({ endpoint, ...options }: SyncBackendOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
   ({ storeId, payload }) =>
     Effect.gen(function* () {
-      const isConnected = yield* SubscriptionRef.make(true)
+      const isConnected = yield* SubscriptionRef.make(false)
       const pullEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.pull
       const pushEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.push
+      const pingEndpoint = typeof endpoint === 'string' ? endpoint : endpoint.ping
+
+      const httpClient = yield* HttpClient.HttpClient
 
-      const pull = (
+      const runPull = (
         handle: Option.Option<SyncMetadata>,
+        { live }: { live: boolean },
       ): Effect.Effect<
         Option.Option<
           readonly [
-            Chunk.Chunk<{
+            /** The batch of events */
+            ReadonlyArray<{
               metadata: Option.Option<SyncMetadata>
               eventEncoded: LiveStoreEvent.AnyEncodedGlobal
             }>,
+            /** The next handle to use for the next pull */
             Option.Option<SyncMetadata>,
           ]
         >,
-        InvalidPullError | IsOfflineError,
-        HttpClient.HttpClient
+        InvalidPullError | IsOfflineError
       > =>
         Effect.gen(function* () {
-          const argsJson = yield* Schema.encode(Schema.parseJson(ApiSchema.PullPayload))(
-            ApiSchema.PullPayload.make({ storeId, handle, payload }),
+          const argsJson = yield* Schema.encode(ApiSchema.ArgsSchema)(
+            ApiSchema.PullPayload.make({ storeId, handle, payload, live }),
           )
           const url = `${pullEndpoint}?args=${argsJson}`
 
-          const resp = yield* HttpClient.get(url)
+          const resp = yield* httpClient.get(url)
 
           if (resp.status === 401) {
             const body = yield* resp.text.pipe(Effect.catchAll(() => Effect.succeed('-')))
             return yield* InvalidPullError.make({
-              message: `Unauthorized (401): Couldn't connect to ElectricSQL: ${body}`,
+              cause: new Error(`Unauthorized (401): Couldn't connect to ElectricSQL: ${body}`),
             })
+          } else if (resp.status === 400) {
+            // Electric returns 400 when table doesn't exist
+            // Return empty result for non-existent tables
+            return Option.some([[], Option.none()] as const)
           } else if (resp.status === 409) {
             // https://electric-sql.com/openapi.html#/paths/~1v1~1shape/get
             // {
@@ -243,8 +219,9 @@ export const makeSyncBackend =
             // until we found a new event, then, continue with the new handle
             return notYetImplemented(`Electric shape not found`)
           } else if (resp.status < 200 || resp.status >= 300) {
+            const body = yield* resp.text
             return yield* InvalidPullError.make({
-              message: `Unexpected status code: ${resp.status}`,
+              cause: new Error(`Unexpected status code: ${resp.status}: ${body}`),
             })
           }
 
@@ -257,56 +234,105 @@ export const makeSyncBackend =
           // Electric completes the long-poll request after ~20 seconds with a 204 status
           // In this case we just retry where we left off
           if (resp.status === 204) {
-            return Option.some([Chunk.empty(), Option.some(nextHandle)] as const)
+            return Option.some([[], Option.some(nextHandle)] as const)
           }
 
-          const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem), {
+          const allItems = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem), {
             onExcessProperty: 'preserve',
           })(resp)
 
-          const items = body
-            .filter((item) => item.value !== undefined && (item.headers as any).operation === 'insert')
-            .map((item) => ({
-              metadata: Option.some({ offset: nextHandle.offset!, handle: nextHandle.handle }),
-              eventEncoded: item.value! as LiveStoreEvent.AnyEncodedGlobal,
-            }))
+          // Check for delete/update operations and throw descriptive error
+          const invalidOperations = ReadonlyArray.filterMap(allItems, (item) =>
+            Schema.is(ResponseItemInvalid)(item) ? Option.some(item.headers.operation) : Option.none(),
+          )
+
+          if (invalidOperations.length > 0) {
+            const operation = invalidOperations[0]!
+            return yield* new InvalidOperationError({
+              operation,
+              message: `ElectricSQL '${operation}' event received. This results from directly mutating the event log. Append a series of events that produce the desired state instead of mutating the event log.`,
+            })
+          }
 
-          // // TODO implement proper `remaining` handling
-          // remaining: 0,
+          const items = allItems.filter(Schema.is(ResponseItemInsert)).map((item) => ({
+            metadata: Option.some({ offset: nextHandle.offset, handle: nextHandle.handle }),
+            eventEncoded: item.value as LiveStoreEvent.AnyEncodedGlobal,
+          }))
 
-          // if (listenForNew === false && items.length === 0) {
-          //   return Option.none()
-          // }
+          yield* Effect.annotateCurrentSpan({ itemsCount: items.length, nextHandle })
 
-          return Option.some([Chunk.fromIterable(items), Option.some(nextHandle)] as const)
+          return Option.some([items, Option.some(nextHandle)] as const)
         }).pipe(
           Effect.scoped,
-          Effect.mapError((cause) =>
-            cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ message: cause.toString() }),
-          ),
+          Effect.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
+          Effect.withSpan('electric-provider:runPull', { attributes: { handle, live } }),
         )
 
       const pullEndpointHasSameOrigin =
         pullEndpoint.startsWith('/') ||
         (globalThis.location !== undefined && globalThis.location.origin === new URL(pullEndpoint).origin)
 
-      return {
-        // If the pull endpoint has the same origin as the current page, we can assume that we already have a connection
-        // otherwise we send a HEAD request to speed up the connection process
-        connect: pullEndpointHasSameOrigin
-          ? Effect.void
-          : HttpClient.head(pullEndpoint).pipe(UnexpectedError.mapToUnexpectedError),
-        pull: (args) =>
-          Stream.unfoldChunkEffect(
-            args.pipe(
-              Option.map((_) => _.metadata),
-              Option.flatten,
-            ),
-            (metadataOption) => pull(metadataOption),
+      const pingTimeout = options.ping?.requestTimeout ?? 10_000
+
+      const ping: SyncBackend.SyncBackend<SyncMetadata>['ping'] = Effect.gen(function* () {
+        yield* httpClient.pipe(HttpClient.filterStatusOk).head(pingEndpoint)
+
+        yield* SubscriptionRef.set(isConnected, true)
+      }).pipe(
+        UnexpectedError.mapToUnexpectedError,
+        Effect.timeout(pingTimeout),
+        Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
+        Effect.withSpan('electric-provider:ping'),
+      )
+
+      const pingInterval = options.ping?.requestInterval ?? 10_000
+
+      if (options.ping?.enabled !== false) {
+        // Automatically ping the server to keep the connection alive
+        yield* ping.pipe(Effect.repeat(Schedule.spaced(pingInterval)), Effect.tapCauseLogPretty, Effect.forkScoped)
+      }
+
+      // If the pull endpoint has the same origin as the current page, we can assume that we already have a connection
+      // otherwise we send a HEAD request to speed up the connection process
+      const connect: SyncBackend.SyncBackend<SyncMetadata>['connect'] = pullEndpointHasSameOrigin
+        ? Effect.void
+        : ping.pipe(UnexpectedError.mapToUnexpectedError)
+
+      return SyncBackend.of({
+        connect,
+        pull: (cursor, options) => {
+          let hasEmittedAtLeastOnce = false
+
+          return Stream.unfoldEffect(cursor.pipe(Option.flatMap((_) => _.metadata)), (metadataOption) =>
+            Effect.gen(function* () {
+              const result = yield* runPull(metadataOption, { live: options?.live ?? false })
+              if (Option.isNone(result)) return Option.none()
+
+              const [batch, nextMetadataOption] = result.value
+
+              // Continue pagination if we have data
+              if (batch.length > 0) {
+                hasEmittedAtLeastOnce = true
+                return Option.some([{ batch, hasMore: true }, nextMetadataOption])
+              }
+
+              // Make sure we emit at least once even if there's no data or we're live-pulling
+              if (hasEmittedAtLeastOnce === false || options?.live) {
+                hasEmittedAtLeastOnce = true
+                return Option.some([{ batch, hasMore: false }, nextMetadataOption])
+              }
+
+              // Stop on empty batch (when not live)
+              return Option.none()
+            }),
           ).pipe(
-            Stream.chunks,
-            Stream.map((chunk) => ({ batch: [...chunk], remaining: 0 })),
-          ),
+            Stream.map(({ batch, hasMore }) => ({
+              batch,
+              pageInfo: hasMore ? SyncBackend.pageInfoMoreUnknown : SyncBackend.pageInfoNoMore,
+            })),
+            Stream.withSpan('electric-provider:pull'),
+          )
+        },
 
         push: (batch) =>
           Effect.gen(function* () {
@@ -314,18 +340,17 @@ export const makeSyncBackend =
               HttpClientRequest.post(pushEndpoint),
               ApiSchema.PushPayload.make({ storeId, batch }),
             ).pipe(
-              Effect.andThen(HttpClient.execute),
+              Effect.andThen(httpClient.pipe(HttpClient.filterStatusOk).execute),
               Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))),
               Effect.scoped,
-              Effect.mapError((cause) =>
-                InvalidPushError.make({ reason: { _tag: 'Unexpected', message: cause.toString() } }),
-              ),
+              Effect.mapError((cause) => InvalidPushError.make({ cause: UnexpectedError.make({ cause }) })),
             )
 
             if (!resp.success) {
-              yield* InvalidPushError.make({ reason: { _tag: 'Unexpected', message: 'Push failed' } })
+              return yield* InvalidPushError.make({ cause: new UnexpectedError({ cause: new Error('Push failed') }) })
             }
-          }),
+          }).pipe(Effect.withSpan('electric-provider:push')),
+        ping,
         isConnected,
         metadata: {
           name: '@livestore/sync-electric',
@@ -333,17 +358,11 @@ export const makeSyncBackend =
           protocol: 'http',
           endpoint,
         },
-      } satisfies SyncBackend<SyncMetadata>
+        supports: {
+          // Given Electric is heavily optimized for immutable caching, we can't know the remaining count
+          // until we've reached the end of the stream
+          pullPageInfoKnown: false,
+          pullLive: true,
+        },
+      })
     })
-
-/**
- * Needs to be bumped when the storage format changes (e.g. eventlogTable schema changes)
- *
- * Changing this version number will lead to a "soft reset".
- */
-export const PERSISTENCE_FORMAT_VERSION = 6
-
-export const toTableName = (storeId: string) => {
-  const escapedStoreId = storeId.replaceAll(/[^a-zA-Z0-9_]/g, '_')
-  return `eventlog_${PERSISTENCE_FORMAT_VERSION}_${escapedStoreId}`
-}

package/src/make-electric-url.ts

@@ -0,0 +1,110 @@
+import { shouldNeverHappen } from '@livestore/utils'
+import { Hash, Schema } from '@livestore/utils/effect'
+import * as ApiSchema from './api-schema.ts'
+
+/**
+ * This function should be called in a trusted environment (e.g. a proxy server) as it
+ * requires access to senstive information (e.g. `apiSecret` / `sourceSecret`).
+ */
+export const makeElectricUrl = ({
+  electricHost,
+  searchParams: providedSearchParams,
+  sourceId,
+  sourceSecret,
+  apiSecret,
+}: {
+  electricHost: string
+  /**
+   * Needed to extract information from the search params which the `@livestore/sync-electric`
+   * client implementation automatically adds:
+   * - `handle`: the ElectricSQL handle
+   * - `storeId`: the Livestore storeId
+   */
+  searchParams: URLSearchParams
+  /** Needed for Electric Cloud */
+  sourceId?: string
+  /** Needed for Electric Cloud */
+  sourceSecret?: string
+  /** For self-hosted ElectricSQL */
+  apiSecret?: string
+}): {
+  /**
+   * The URL to the ElectricSQL API endpoint with needed search params.
+   */
+  url: string
+  /** The Livestore storeId */
+  storeId: string
+  /**
+   * Whether the Postgres table needs to be created.
+   */
+  needsInit: boolean
+  /** Sync payload provided by the client */
+  payload: Schema.JsonValue | undefined
+} => {
+  const endpointUrl = `${electricHost}/v1/shape`
+  const UrlParamsSchema = Schema.Struct({ args: ApiSchema.ArgsSchema })
+  const argsResult = Schema.decodeUnknownEither(UrlParamsSchema)(Object.fromEntries(providedSearchParams.entries()))
+
+  if (argsResult._tag === 'Left') {
+    return shouldNeverHappen(
+      'Invalid search params provided to makeElectricUrl',
+      providedSearchParams,
+      Object.fromEntries(providedSearchParams.entries()),
+    )
+  }
+
+  const args = argsResult.right.args
+  const tableName = toTableName(args.storeId)
+  // TODO refactor with Effect URLSearchParams schema
+  // https://electric-sql.com/openapi.html
+  const searchParams = new URLSearchParams()
+  // Electric requires table names with capital letters to be quoted
+  // Since our table names include the storeId which may have capitals, we always quote
+  searchParams.set('table', `"${tableName}"`)
+  if (sourceId !== undefined) {
+    searchParams.set('source_id', sourceId)
+  }
+  if (sourceSecret !== undefined) {
+    searchParams.set('source_secret', sourceSecret)
+  }
+  if (apiSecret !== undefined) {
+    searchParams.set('api_secret', apiSecret)
+  }
+  if (args.handle._tag === 'None') {
+    searchParams.set('offset', '-1')
+  } else {
+    searchParams.set('offset', args.handle.value.offset)
+    searchParams.set('handle', args.handle.value.handle)
+    searchParams.set('live', args.live ? 'true' : 'false')
+  }
+
+  const payload = args.payload
+
+  const url = `${endpointUrl}?${searchParams.toString()}`
+
+  return { url, storeId: args.storeId, needsInit: args.handle._tag === 'None', payload }
+}
+
+export const toTableName = (storeId: string) => {
+  const escapedStoreId = storeId.replaceAll(/[^a-zA-Z0-9_]/g, '_')
+  const tableName = `eventlog_${PERSISTENCE_FORMAT_VERSION}_${escapedStoreId}`
+
+  if (tableName.length > 63) {
+    const hashedStoreId = Hash.string(storeId)
+
+    console.warn(
+      `Table name is too long: "${tableName}". Postgres table names are limited to 63 characters. Using hashed storeId instead: "${hashedStoreId}".`,
+    )
+
+    return `eventlog_${PERSISTENCE_FORMAT_VERSION}_hash_${hashedStoreId}`
+  }
+
+  return tableName
+}
+
+/**
+ * Needs to be bumped when the storage format changes (e.g. eventlogTable schema changes)
+ *
+ * Changing this version number will lead to a "soft reset".
+ */
+export const PERSISTENCE_FORMAT_VERSION = 6