@livestore/sync-electric 0.0.58-dev.4 → 0.0.58-dev.6

package/dist/index.d.ts

@@ -1,5 +1,59 @@
-import type { SyncImpl } from '@livestore/common';
+import type { SyncBackend, SyncBackendOptionsBase } from '@livestore/common';
 import type { Scope } from '@livestore/utils/effect';
-import { Effect } from '@livestore/utils/effect';
-export declare const makeSync: (electricHost: string, roomId: string) => Effect.Effect<SyncImpl, never, Scope.Scope>;
+import { Effect, Schema } from '@livestore/utils/effect';
+export declare const syncBackend: any;
+export declare const ApiPushEventPayload: Schema.TaggedStruct<"sync-electric.PushEvent", {
+    roomId: typeof Schema.String;
+    mutationEventEncoded: Schema.Struct<{
+        mutation: typeof Schema.String;
+        args: typeof Schema.Any;
+        id: typeof Schema.String;
+    }>;
+    persisted: typeof Schema.Boolean;
+}>;
+export declare const ApiInitRoomPayload: Schema.TaggedStruct<"sync-electric.InitRoom", {
+    roomId: typeof Schema.String;
+}>;
+export declare const ApiPayload: Schema.Union<[Schema.TaggedStruct<"sync-electric.PushEvent", {
+    roomId: typeof Schema.String;
+    mutationEventEncoded: Schema.Struct<{
+        mutation: typeof Schema.String;
+        args: typeof Schema.Any;
+        id: typeof Schema.String;
+    }>;
+    persisted: typeof Schema.Boolean;
+}>, Schema.TaggedStruct<"sync-electric.InitRoom", {
+    roomId: typeof Schema.String;
+}>]>;
+export declare const syncBackendOptions: <TOptions extends SyncBackendOptions>(options: TOptions) => TOptions;
+export interface SyncBackendOptions extends SyncBackendOptionsBase {
+    type: 'electric';
+    /**
+     * The host of the Electric server
+     *
+     * @example "https://localhost:3000"
+     */
+    electricHost: string;
+    roomId: string;
+    /**
+     * The POST endpoint to push events to
+     *
+     * @example "/api/push-event"
+     * @example "https://api.myapp.com/push-event"
+     */
+    pushEventEndpoint: string;
+}
+interface LiveStoreGlobalElectric {
+    syncBackend: SyncBackendOptions;
+}
+declare global {
+    interface LiveStoreGlobal extends LiveStoreGlobalElectric {
+    }
+}
+type SyncMetadata = {
+    offset: string;
+    shapeId: string;
+};
+export declare const makeSyncBackend: ({ electricHost, roomId, pushEventEndpoint, }: SyncBackendOptions) => Effect.Effect<SyncBackend<SyncMetadata>, never, Scope.Scope>;
+export {};
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAKjD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAA;AACpD,OAAO,EAGL,MAAM,EAUP,MAAM,yBAAyB,CAAA;AA0BhC,eAAO,MAAM,QAAQ,iBAML,MAAM,UACZ,MAAM,KACb,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAoC5C,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAA;AAG5E,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAA;AACpD,OAAO,EAGL,MAAM,EAKN,MAAM,EAGP,MAAM,yBAAyB,CAAA;AA4BhC,eAAO,MAAM,WAAW,EAAS,GAAG,CAAA;AAEpC,eAAO,MAAM,mBAAmB;;;;;;;;EAI9B,CAAA;AAEF,eAAO,MAAM,kBAAkB;;EAE7B,CAAA;AAEF,eAAO,MAAM,UAAU;;;;;;;;;;IAAwD,CAAA;AAE/E,eAAO,MAAM,kBAAkB,GAAI,QAAQ,SAAS,kBAAkB,WAAW,QAAQ,aAAY,CAAA;AAErG,MAAM,WAAW,kBAAmB,SAAQ,sBAAsB;IAChE,IAAI,EAAE,UAAU,CAAA;IAChB;;;;OAIG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;;;;OAKG;IACH,iBAAiB,EAAE,MAAM,CAAA;CAC1B;AAED,UAAU,uBAAuB;IAC/B,WAAW,EAAE,kBAAkB,CAAA;CAChC;AAED,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,eAAgB,SAAQ,uBAAuB;KAAG;CAC7D;AAED,KAAK,YAAY,GAAG;IAClB,MAAM,EAAE,MAAM,CAAA;IAEd,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,eAAO,MAAM,eAAe,iDAIzB,kBAAkB,KAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAiH/E,CAAA"}

package/dist/index.js

@@ -1,7 +1,6 @@
-import { InvalidPullError } from '@livestore/common';
+import { InvalidPullError, InvalidPushError } from '@livestore/common';
 import { mutationEventSchemaEncodedAny } from '@livestore/common/schema';
-import { isNotUndefined } from '@livestore/utils';
-import { Chunk, Effect, HttpClient, HttpClientRequest, HttpClientResponse, Option, Schema, Stream, SubscriptionRef, } from '@livestore/utils/effect';
+import { Chunk, Deferred, Effect, HttpClient, HttpClientRequest, HttpClientResponse, Option, Schema, Stream, SubscriptionRef, } from '@livestore/utils/effect';
 /*
 Example data:
 
@@ -9,6 +8,8 @@ Example data:
 ,{"key":"\"public\".\"events\"/\"1725703554783\"","value":{"id":"1725703554783","mutation":"test","args_json":"{\"test\":\"test\"}","schema_hash":"1","created_at":"2024-09-07T10:05:54.783Z"},"headers":{"operation":"insert","relation":["public","events"]},"offset":"0_0"}
 ,{"headers":{"control":"up-to-date"}}]
 
+Also see: https://github.com/electric-sql/electric/blob/main/packages/typescript-client/src/client.ts
+
 */
 const ResponseItem = Schema.Struct({
     /** Postgres path (e.g. "public.events/1") */
@@ -19,36 +20,81 @@ const ResponseItem = Schema.Struct({
 });
 const ResponseHeaders = Schema.Struct({
     'x-electric-shape-id': Schema.String,
-    'x-electric-schema': Schema.parseJson(Schema.Any),
+    // 'x-electric-schema': Schema.parseJson(Schema.Any),
     /** e.g. 26799576_0 */
     'x-electric-chunk-last-offset': Schema.String,
 });
-export const makeSync = (
-/**
- * The host of the Electric server
- *
- * @example "https://my-electric.com"
- */
-electricHost, roomId) => {
-    return Effect.gen(function* () {
-        const endpointUrl = `${electricHost}/v1/shape/${roomId}`;
-        // ?offset=-1
-        const isConnected = yield* SubscriptionRef.make(true);
-        return {
-            pull: (cursor) => Stream.unfoldChunkEffect(cursor, (cursor) => Effect.gen(function* () {
-                const resp = yield* HttpClientRequest.get(`${endpointUrl}?offset=${cursor ?? '-1'}`).pipe(HttpClient.fetchOk);
-                const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp);
-                const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp);
-                const items = body.map((item) => item.value).filter(isNotUndefined);
-                // if (items.length === 0) {
-                //   return Option.none()
-                // }
-                return Option.some([Chunk.fromIterable(items), headers['x-electric-chunk-last-offset']]);
-            }).pipe(Effect.scoped, Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })))),
-            pushes: Stream.never,
-            push: (mutationEventEncoded, persisted) => Effect.void,
-            isConnected,
+export const syncBackend = {};
+export const ApiPushEventPayload = Schema.TaggedStruct('sync-electric.PushEvent', {
+    roomId: Schema.String,
+    mutationEventEncoded: mutationEventSchemaEncodedAny,
+    persisted: Schema.Boolean,
+});
+export const ApiInitRoomPayload = Schema.TaggedStruct('sync-electric.InitRoom', {
+    roomId: Schema.String,
+});
+export const ApiPayload = Schema.Union(ApiPushEventPayload, ApiInitRoomPayload);
+export const syncBackendOptions = (options) => options;
+export const makeSyncBackend = ({ electricHost, roomId, pushEventEndpoint, }) => Effect.gen(function* () {
+    const endpointUrl = `${electricHost}/v1/shape/events_${roomId}`;
+    const isConnected = yield* SubscriptionRef.make(true);
+    const initRoom = HttpClientRequest.schemaBody(ApiInitRoomPayload)(HttpClientRequest.post(pushEventEndpoint), ApiInitRoomPayload.make({ roomId })).pipe(Effect.andThen(HttpClient.fetchOk));
+    const pendingPushDeferredMap = new Map();
+    const pull = (args, { listenForNew }) => Effect.gen(function* () {
+        const liveParam = listenForNew ? '&live=true' : '';
+        const url = args._tag === 'None'
+            ? `${endpointUrl}?offset=-1`
+            : `${endpointUrl}?offset=${args.value.offset}&shape_id=${args.value.shapeId}${liveParam}`;
+        const resp = yield* HttpClientRequest.get(url).pipe(HttpClient.fetchOk, Effect.tapErrorTag('ResponseError', (error) => 
+        // TODO handle 409 error when the shapeId you request no longer exists for whatever reason.
+        // The correct behavior here is to refetch the shape from scratch and to reset the local state.
+        error.response.status === 400 ? initRoom : Effect.fail(error)), Effect.retry({ times: 1 }));
+        const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp);
+        const nextCursor = {
+            offset: headers['x-electric-chunk-last-offset'],
+            shapeId: headers['x-electric-shape-id'],
         };
-    });
-};
+        // Electric completes the long-poll request after ~20 seconds with a 204 status
+        // In this case we just retry where we left off
+        if (resp.status === 204) {
+            return Option.some([Chunk.empty(), Option.some(nextCursor)]);
+        }
+        const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp);
+        const items = body
+            .filter((item) => item.value !== undefined)
+            .map((item) => ({
+            metadata: Option.some({ offset: item.offset, shapeId: nextCursor.shapeId }),
+            mutationEventEncoded: {
+                mutation: item.value.mutation,
+                args: JSON.parse(item.value.args),
+                id: item.value.id,
+            },
+            persisted: true,
+        }));
+        if (listenForNew === false && items.length === 0) {
+            return Option.none();
+        }
+        const [newItems, pendingPushItems] = Chunk.fromIterable(items).pipe(Chunk.partition((item) => pendingPushDeferredMap.has(item.mutationEventEncoded.id)));
+        for (const item of pendingPushItems) {
+            const deferred = pendingPushDeferredMap.get(item.mutationEventEncoded.id);
+            yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata));
+        }
+        return Option.some([newItems, Option.some(nextCursor)]);
+    }).pipe(Effect.scoped, Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })));
+    return {
+        pull: (args, { listenForNew }) => Stream.unfoldChunkEffect(args.pipe(Option.map((_) => _.metadata), Option.flatten), (metadataOption) => pull(metadataOption, { listenForNew })),
+        push: (mutationEventEncoded, persisted) => Effect.gen(function* () {
+            const deferred = yield* Deferred.make();
+            pendingPushDeferredMap.set(mutationEventEncoded.id, deferred);
+            const resp = yield* HttpClientRequest.schemaBody(ApiPushEventPayload)(HttpClientRequest.post(pushEventEndpoint), ApiPushEventPayload.make({ roomId, mutationEventEncoded, persisted })).pipe(Effect.andThen(HttpClient.fetchOk), Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))), Effect.scoped, Effect.mapError((cause) => InvalidPushError.make({ message: cause.toString() })));
+            if (!resp.success) {
+                yield* InvalidPushError.make({ message: 'Push failed' });
+            }
+            const metadata = yield* Deferred.await(deferred);
+            pendingPushDeferredMap.delete(mutationEventEncoded.id);
+            return { metadata: Option.some(metadata) };
+        }),
+        isConnected,
+    };
+});
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAoB,MAAM,mBAAmB,CAAA;AACtE,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAA;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AAGjD,OAAO,EACL,KAAK,EAEL,MAAM,EACN,UAAU,EACV,iBAAiB,EACjB,kBAAkB,EAClB,MAAM,EAGN,MAAM,EACN,MAAM,EACN,eAAe,GAChB,MAAM,yBAAyB,CAAA;AAEhC;;;;;;;EAOE;AAEF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,6CAA6C;IAC7C,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACrD,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;IACjE,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;CACvC,CAAC,CAAA;AAEF,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC,MAAM;IACpC,mBAAmB,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC;IACjD,sBAAsB;IACtB,8BAA8B,EAAE,MAAM,CAAC,MAAM;CAC9C,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG;AACtB;;;;GAIG;AACH,YAAoB,EACpB,MAAc,EAC+B,EAAE;IAC/C,OAAO,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACzB,MAAM,WAAW,GAAG,GAAG,YAAY,aAAa,MAAM,EAAE,CAAA;QACxD,aAAa;QAEb,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAErD,OAAO;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CACf,MAAM,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1C,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAClB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,WAAW,WAAW,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CACvF,UAAU,CAAC,OAAO,CACnB,CAAA;gBAED,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAA;gBAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,kBAAkB,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;gBAEvF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;gBAEnE,4BAA4B;gBAC5B,yBAAyB;gBACzB,IAAI;gBAEJ,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,8BAA8B,CAAC,CAAU,CAAC,CAAA;YACnG,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CACjF,CACF;YAEH,MAAM,EAAE,MAAM,CAAC,KAAK;YACpB,IAAI,EAAE,CAAC,oBAAoB,EAAE,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI;YACtD,WAAW;SACO,CAAA;IACtB,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACtE,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAA;AAExE,OAAO,EACL,KAAK,EACL,QAAQ,EACR,MAAM,EACN,UAAU,EACV,iBAAiB,EACjB,kBAAkB,EAClB,MAAM,EACN,MAAM,EACN,MAAM,EACN,eAAe,GAChB,MAAM,yBAAyB,CAAA;AAEhC;;;;;;;;;EASE;AAEF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,6CAA6C;IAC7C,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACrD,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;IACjE,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;CACvC,CAAC,CAAA;AAEF,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC,MAAM;IACpC,qDAAqD;IACrD,sBAAsB;IACtB,8BAA8B,EAAE,MAAM,CAAC,MAAM;CAC9C,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,EAAS,CAAA;AAEpC,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC,YAAY,CAAC,yBAAyB,EAAE;IAChF,MAAM,EAAE,MAAM,CAAC,MAAM;IACrB,oBAAoB,EAAE,6BAA6B;IACnD,SAAS,EAAE,MAAM,CAAC,OAAO;CAC1B,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC,YAAY,CAAC,wBAAwB,EAAE;IAC9E,MAAM,EAAE,MAAM,CAAC,MAAM;CACtB,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,kBAAkB,CAAC,CAAA;AAE/E,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAsC,OAAiB,EAAE,EAAE,CAAC,OAAO,CAAA;AAkCrG,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAC9B,YAAY,EACZ,MAAM,EACN,iBAAiB,GACE,EAAgE,EAAE,CACrF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,WAAW,GAAG,GAAG,YAAY,oBAAoB,MAAM,EAAE,CAAA;IAE/D,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAErD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAC/D,iBAAiB,CAAC,IAAI,CAAC,iBAAiB,CAAC,EACzC,kBAAkB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CACpC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAA;IAE1C,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA2C,CAAA;IAEjF,MAAM,IAAI,GAAG,CAAC,IAAiC,EAAE,EAAE,YAAY,EAA6B,EAAE,EAAE,CAC9F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAA;QAClD,MAAM,GAAG,GACP,IAAI,CAAC,IAAI,KAAK,MAAM;YAClB,CAAC,CAAC,GAAG,WAAW,YAAY;YAC5B,CAAC,CAAC,GAAG,WAAW,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,aAAa,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,SAAS,EAAE,CAAA;QAE7F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CACjD,UAAU,CAAC,OAAO,EAClB,MAAM,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC,KAAK,EAAE,EAAE;QAC5C,2FAA2F;QAC3F,+FAA+F;QAC/F,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAC9D,EACD,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAC3B,CAAA;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAA;QAC9E,MAAM,UAAU,GAAG;YACjB,MAAM,EAAE,OAAO,CAAC,8BAA8B,CAAC;YAC/C,OAAO,EAAE,OAAO,CAAC,qBAAqB,CAAC;SACxC,CAAA;QAED,+EAA+E;QAC/E,+CAA+C;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACxB,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAU,CAAC,CAAA;QACvE,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,kBAAkB,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEvF,MAAM,KAAK,GAAG,IAAI;aACf,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC;aAC1C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;YAC5E,oBAAoB,EAAE;gBACpB,QAAQ,EAAE,IAAI,CAAC,KAAM,CAAC,QAAQ;gBAC9B,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAM,CAAC,IAAI,CAAC;gBAClC,EAAE,EAAE,IAAI,CAAC,KAAM,CAAC,EAAE;aACnB;YACD,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC,CAAA;QAEL,IAAI,YAAY,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAA;QACtB,CAAC;QAED,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CACjE,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,CACpF,CAAA;QAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAE,CAAA;YAC1E,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAA;QACrE,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAU,CAAC,CAAA;IAClE,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CACjF,CAAA;IAEH,OAAO;QACL,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAC/B,MAAM,CAAC,iBAAiB,CACtB,IAAI,CAAC,IAAI,CACP,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAC7B,MAAM,CAAC,OAAO,CACf,EACD,CAAC,cAAc,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,YAAY,EAAE,CAAC,CAC3D;QAEH,IAAI,EAAE,CAAC,oBAAoB,EAAE,SAAS,EAAE,EAAE,CACxC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAgB,CAAA;YACrD,sBAAsB,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;YAE7D,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAC,UAAU,CAAC,mBAAmB,CAAC,CACnE,iBAAiB,CAAC,IAAI,CAAC,iBAAiB,CAAC,EACzC,mBAAmB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,SAAS,EAAE,CAAC,CACtE,CAAC,IAAI,CACJ,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,EAClC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAC7F,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CACjF,CAAA;YAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,KAAK,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAA;YAC1D,CAAC;YAED,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YAEhD,sBAAsB,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAA;YAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAA;QAC5C,CAAC,CAAC;QACJ,WAAW;KACwB,CAAA;AACvC,CAAC,CAAC,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livestore/sync-electric",
-  "version": "0.0.58-dev.4",
+  "version": "0.0.58-dev.6",
   "type": "module",
   "exports": {
     ".": {
@@ -9,8 +9,8 @@
     }
   },
   "dependencies": {
-    "@livestore/utils": "0.0.58-dev.4",
-    "@livestore/common": "0.0.58-dev.4"
+    "@livestore/common": "0.0.58-dev.6",
+    "@livestore/utils": "0.0.58-dev.6"
   },
   "devDependencies": {},
   "publishConfig": {

package/src/index.ts

@@ -1,8 +1,6 @@
-import type { SyncImpl } from '@livestore/common'
+import type { SyncBackend, SyncBackendOptionsBase } from '@livestore/common'
 import { InvalidPullError, InvalidPushError } from '@livestore/common'
 import { mutationEventSchemaEncodedAny } from '@livestore/common/schema'
-import { isNotUndefined } from '@livestore/utils'
-import { cuid } from '@livestore/utils/cuid'
 import type { Scope } from '@livestore/utils/effect'
 import {
   Chunk,
@@ -12,8 +10,6 @@ import {
   HttpClientRequest,
   HttpClientResponse,
   Option,
-  Queue,
-  ReadonlyArray,
   Schema,
   Stream,
   SubscriptionRef,
@@ -26,6 +22,8 @@ Example data:
 ,{"key":"\"public\".\"events\"/\"1725703554783\"","value":{"id":"1725703554783","mutation":"test","args_json":"{\"test\":\"test\"}","schema_hash":"1","created_at":"2024-09-07T10:05:54.783Z"},"headers":{"operation":"insert","relation":["public","events"]},"offset":"0_0"}
 ,{"headers":{"control":"up-to-date"}}]
 
+Also see: https://github.com/electric-sql/electric/blob/main/packages/typescript-client/src/client.ts
+
 */
 
 const ResponseItem = Schema.Struct({
@@ -38,53 +36,174 @@ const ResponseItem = Schema.Struct({
 
 const ResponseHeaders = Schema.Struct({
   'x-electric-shape-id': Schema.String,
-  'x-electric-schema': Schema.parseJson(Schema.Any),
+  // 'x-electric-schema': Schema.parseJson(Schema.Any),
   /** e.g. 26799576_0 */
   'x-electric-chunk-last-offset': Schema.String,
 })
 
-export const makeSync = (
+export const syncBackend = {} as any
+
+export const ApiPushEventPayload = Schema.TaggedStruct('sync-electric.PushEvent', {
+  roomId: Schema.String,
+  mutationEventEncoded: mutationEventSchemaEncodedAny,
+  persisted: Schema.Boolean,
+})
+
+export const ApiInitRoomPayload = Schema.TaggedStruct('sync-electric.InitRoom', {
+  roomId: Schema.String,
+})
+
+export const ApiPayload = Schema.Union(ApiPushEventPayload, ApiInitRoomPayload)
+
+export const syncBackendOptions = <TOptions extends SyncBackendOptions>(options: TOptions) => options
+
+export interface SyncBackendOptions extends SyncBackendOptionsBase {
+  type: 'electric'
   /**
    * The host of the Electric server
    *
-   * @example "https://my-electric.com"
+   * @example "https://localhost:3000"
    */
-  electricHost: string,
-  roomId: string,
-): Effect.Effect<SyncImpl, never, Scope.Scope> => {
-  return Effect.gen(function* () {
-    const endpointUrl = `${electricHost}/v1/shape/${roomId}`
-    // ?offset=-1
+  electricHost: string
+  roomId: string
+  /**
+   * The POST endpoint to push events to
+   *
+   * @example "/api/push-event"
+   * @example "https://api.myapp.com/push-event"
+   */
+  pushEventEndpoint: string
+}
+
+interface LiveStoreGlobalElectric {
+  syncBackend: SyncBackendOptions
+}
+
+declare global {
+  interface LiveStoreGlobal extends LiveStoreGlobalElectric {}
+}
+
+type SyncMetadata = {
+  offset: string
+  // TODO move this into "global" sync metadata as it's the same for each event
+  shapeId: string
+}
+
+export const makeSyncBackend = ({
+  electricHost,
+  roomId,
+  pushEventEndpoint,
+}: SyncBackendOptions): Effect.Effect<SyncBackend<SyncMetadata>, never, Scope.Scope> =>
+  Effect.gen(function* () {
+    const endpointUrl = `${electricHost}/v1/shape/events_${roomId}`
 
     const isConnected = yield* SubscriptionRef.make(true)
 
-    return {
-      pull: (cursor) =>
-        Stream.unfoldChunkEffect(cursor, (cursor) =>
-          Effect.gen(function* () {
-            const resp = yield* HttpClientRequest.get(`${endpointUrl}?offset=${cursor ?? '-1'}`).pipe(
-              HttpClient.fetchOk,
-            )
+    const initRoom = HttpClientRequest.schemaBody(ApiInitRoomPayload)(
+      HttpClientRequest.post(pushEventEndpoint),
+      ApiInitRoomPayload.make({ roomId }),
+    ).pipe(Effect.andThen(HttpClient.fetchOk))
 
-            const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp)
-            const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp)
+    const pendingPushDeferredMap = new Map<string, Deferred.Deferred<SyncMetadata>>()
 
-            const items = body.map((item) => item.value).filter(isNotUndefined)
+    const pull = (args: Option.Option<SyncMetadata>, { listenForNew }: { listenForNew: boolean }) =>
+      Effect.gen(function* () {
+        const liveParam = listenForNew ? '&live=true' : ''
+        const url =
+          args._tag === 'None'
+            ? `${endpointUrl}?offset=-1`
+            : `${endpointUrl}?offset=${args.value.offset}&shape_id=${args.value.shapeId}${liveParam}`
 
-            // if (items.length === 0) {
-            //   return Option.none()
-            // }
+        const resp = yield* HttpClientRequest.get(url).pipe(
+          HttpClient.fetchOk,
+          Effect.tapErrorTag('ResponseError', (error) =>
+            // TODO handle 409 error when the shapeId you request no longer exists for whatever reason.
+            // The correct behavior here is to refetch the shape from scratch and to reset the local state.
+            error.response.status === 400 ? initRoom : Effect.fail(error),
+          ),
+          Effect.retry({ times: 1 }),
+        )
 
-            return Option.some([Chunk.fromIterable(items), headers['x-electric-chunk-last-offset']] as const)
-          }).pipe(
-            Effect.scoped,
-            Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })),
+        const headers = yield* HttpClientResponse.schemaHeaders(ResponseHeaders)(resp)
+        const nextCursor = {
+          offset: headers['x-electric-chunk-last-offset'],
+          shapeId: headers['x-electric-shape-id'],
+        }
+
+        // Electric completes the long-poll request after ~20 seconds with a 204 status
+        // In this case we just retry where we left off
+        if (resp.status === 204) {
+          return Option.some([Chunk.empty(), Option.some(nextCursor)] as const)
+        }
+
+        const body = yield* HttpClientResponse.schemaBodyJson(Schema.Array(ResponseItem))(resp)
+
+        const items = body
+          .filter((item) => item.value !== undefined)
+          .map((item) => ({
+            metadata: Option.some({ offset: item.offset!, shapeId: nextCursor.shapeId }),
+            mutationEventEncoded: {
+              mutation: item.value!.mutation,
+              args: JSON.parse(item.value!.args),
+              id: item.value!.id,
+            },
+            persisted: true,
+          }))
+
+        if (listenForNew === false && items.length === 0) {
+          return Option.none()
+        }
+
+        const [newItems, pendingPushItems] = Chunk.fromIterable(items).pipe(
+          Chunk.partition((item) => pendingPushDeferredMap.has(item.mutationEventEncoded.id)),
+        )
+
+        for (const item of pendingPushItems) {
+          const deferred = pendingPushDeferredMap.get(item.mutationEventEncoded.id)!
+          yield* Deferred.succeed(deferred, Option.getOrThrow(item.metadata))
+        }
+
+        return Option.some([newItems, Option.some(nextCursor)] as const)
+      }).pipe(
+        Effect.scoped,
+        Effect.mapError((cause) => InvalidPullError.make({ message: cause.toString() })),
+      )
+
+    return {
+      pull: (args, { listenForNew }) =>
+        Stream.unfoldChunkEffect(
+          args.pipe(
+            Option.map((_) => _.metadata),
+            Option.flatten,
           ),
+          (metadataOption) => pull(metadataOption, { listenForNew }),
         ),
 
-      pushes: Stream.never,
-      push: (mutationEventEncoded, persisted) => Effect.void,
+      push: (mutationEventEncoded, persisted) =>
+        Effect.gen(function* () {
+          const deferred = yield* Deferred.make<SyncMetadata>()
+          pendingPushDeferredMap.set(mutationEventEncoded.id, deferred)
+
+          const resp = yield* HttpClientRequest.schemaBody(ApiPushEventPayload)(
+            HttpClientRequest.post(pushEventEndpoint),
+            ApiPushEventPayload.make({ roomId, mutationEventEncoded, persisted }),
+          ).pipe(
+            Effect.andThen(HttpClient.fetchOk),
+            Effect.andThen(HttpClientResponse.schemaBodyJson(Schema.Struct({ success: Schema.Boolean }))),
+            Effect.scoped,
+            Effect.mapError((cause) => InvalidPushError.make({ message: cause.toString() })),
+          )
+
+          if (!resp.success) {
+            yield* InvalidPushError.make({ message: 'Push failed' })
+          }
+
+          const metadata = yield* Deferred.await(deferred)
+
+          pendingPushDeferredMap.delete(mutationEventEncoded.id)
+
+          return { metadata: Option.some(metadata) }
+        }),
       isConnected,
-    } satisfies SyncImpl
+    } satisfies SyncBackend<SyncMetadata>
   })
-}