@livestore/sync-cf 0.3.0-dev.2 → 0.3.0-dev.22

package/src/common/ws-message-types.ts

@@ -17,35 +17,27 @@ export const SyncMetadata = Schema.Struct({
 export type SyncMetadata = typeof SyncMetadata.Type
 
 export const PullRes = Schema.TaggedStruct('WSMessage.PullRes', {
-  requestId: Schema.String,
-  events: Schema.Array(
+  batch: Schema.Array(
     Schema.Struct({
-      mutationEventEncoded: MutationEvent.EncodedAny,
+      mutationEventEncoded: MutationEvent.AnyEncodedGlobal,
       metadata: Schema.Option(SyncMetadata),
     }),
   ),
+  requestId: Schema.Struct({ context: Schema.Literal('pull', 'push'), requestId: Schema.String }),
   remaining: Schema.Number,
 })
 
 export type PullRes = typeof PullRes.Type
 
-export const PushBroadcast = Schema.TaggedStruct('WSMessage.PushBroadcast', {
-  mutationEventEncoded: MutationEvent.EncodedAny,
-  metadata: Schema.Option(SyncMetadata),
-})
-
-export type PushBroadcast = typeof PushBroadcast.Type
-
 export const PushReq = Schema.TaggedStruct('WSMessage.PushReq', {
   requestId: Schema.String,
-  batch: Schema.Array(MutationEvent.EncodedAny),
+  batch: Schema.Array(MutationEvent.AnyEncodedGlobal),
 })
 
 export type PushReq = typeof PushReq.Type
 
 export const PushAck = Schema.TaggedStruct('WSMessage.PushAck', {
   requestId: Schema.String,
-  mutationId: Schema.Number,
 })
 
 export type PushAck = typeof PushAck.Type
@@ -55,6 +47,8 @@ export const Error = Schema.TaggedStruct('WSMessage.Error', {
   message: Schema.String,
 })
 
+export type Error = typeof Error.Type
+
 export const Ping = Schema.TaggedStruct('WSMessage.Ping', {
   requestId: Schema.Literal('ping'),
 })
@@ -99,7 +93,6 @@ export type AdminInfoRes = typeof AdminInfoRes.Type
 export const Message = Schema.Union(
   PullReq,
   PullRes,
-  PushBroadcast,
   PushReq,
   PushAck,
   Error,
@@ -113,15 +106,7 @@ export const Message = Schema.Union(
 export type Message = typeof Message.Type
 export type MessageEncoded = typeof Message.Encoded
 
-export const BackendToClientMessage = Schema.Union(
-  PullRes,
-  PushBroadcast,
-  PushAck,
-  AdminResetRoomRes,
-  AdminInfoRes,
-  Error,
-  Pong,
-)
+export const BackendToClientMessage = Schema.Union(PullRes, PushAck, AdminResetRoomRes, AdminInfoRes, Error, Pong)
 export type BackendToClientMessage = typeof BackendToClientMessage.Type
 
 export const ClientToBackendMessage = Schema.Union(PullReq, PushReq, AdminResetRoomReq, AdminInfoReq, Ping)

package/src/sync-impl/ws-impl.ts

@@ -1,8 +1,9 @@
 /// <reference lib="dom" />
 
-import type { SyncBackend, SyncBackendOptionsBase } from '@livestore/common'
+import type { SyncBackend } from '@livestore/common'
 import { InvalidPullError, InvalidPushError } from '@livestore/common'
-import { pick } from '@livestore/utils'
+import { EventId } from '@livestore/common/schema'
+import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
 import type { Scope } from '@livestore/utils/effect'
 import {
   Deferred,
@@ -18,57 +19,92 @@ import {
 } from '@livestore/utils/effect'
 import { nanoid } from '@livestore/utils/nanoid'
 
-import { WSMessage } from '../common/index.js'
+import { WSMessage } from '../common/mod.js'
 import type { SyncMetadata } from '../common/ws-message-types.js'
 
-export interface WsSyncOptions extends SyncBackendOptionsBase {
-  type: 'cf'
+export interface WsSyncOptions {
   url: string
-  roomId: string
-}
-
-interface LiveStoreGlobalCf {
-  syncBackend: WsSyncOptions
-}
-
-declare global {
-  interface LiveStoreGlobal extends LiveStoreGlobalCf {}
+  storeId: string
 }
 
 export const makeWsSync = (options: WsSyncOptions): Effect.Effect<SyncBackend<SyncMetadata>, never, Scope.Scope> =>
   Effect.gen(function* () {
-    const wsUrl = `${options.url}/websocket?room=${options.roomId}`
+    // TODO also allow for auth scenarios
+    const wsUrl = `${options.url}/websocket?storeId=${options.storeId}`
 
     const { isConnected, incomingMessages, send } = yield* connect(wsUrl)
 
+    /**
+     * We need to account for the scenario where push-caused PullRes message arrive before the pull-caused PullRes message.
+     * i.e. a scenario where the WS connection is created but before the server processed the initial pull, a push from
+     * another client triggers a PullRes message sent to this client which we need to stash until our pull-caused
+     * PullRes message arrives at which point we can combine the stashed events with the pull-caused events and continue.
+     */
+    const stashedPullBatch: WSMessage.PullRes['batch'][number][] = []
+
+    // We currently only support one pull stream for a sync backend.
+    let pullStarted = false
+
     const api = {
       isConnected,
       pull: (args) =>
         Effect.gen(function* () {
+          if (pullStarted) {
+            return shouldNeverHappen(`Pull already started for this sync backend.`)
+          }
+
+          pullStarted = true
+
+          let pullResponseReceived = false
+
           const requestId = nanoid()
           const cursor = Option.getOrUndefined(args)?.cursor.global
 
           yield* send(WSMessage.PullReq.make({ cursor, requestId }))
 
           return Stream.fromPubSub(incomingMessages).pipe(
-            Stream.filter((_) => (_._tag === 'WSMessage.PullRes' ? _.requestId === requestId : true)),
             Stream.tap((_) =>
               _._tag === 'WSMessage.Error' && _.requestId === requestId
                 ? new InvalidPullError({ message: _.message })
                 : Effect.void,
             ),
-            Stream.filter(Schema.is(Schema.Union(WSMessage.PushBroadcast, WSMessage.PullRes))),
-            Stream.map((msg) =>
-              msg._tag === 'WSMessage.PushBroadcast'
-                ? { batch: [pick(msg, ['mutationEventEncoded', 'metadata'])], remaining: 0 }
-                : {
-                    batch: msg.events.map(({ mutationEventEncoded, metadata }) => ({
-                      mutationEventEncoded,
-                      metadata,
-                    })),
-                    remaining: msg.remaining,
-                  },
-            ),
+            Stream.filterMap((msg) => {
+              if (msg._tag === 'WSMessage.PullRes') {
+                if (msg.requestId.context === 'pull') {
+                  if (msg.requestId.requestId === requestId) {
+                    pullResponseReceived = true
+
+                    if (stashedPullBatch.length > 0 && msg.remaining === 0) {
+                      const pullResHead = msg.batch.at(-1)?.mutationEventEncoded.id ?? EventId.ROOT.global
+                      // Index where stashed events are greater than pullResHead
+                      const newPartialBatchIndex = stashedPullBatch.findIndex(
+                        (batchItem) => batchItem.mutationEventEncoded.id > pullResHead,
+                      )
+                      const batchWithNewStashedEvents =
+                        newPartialBatchIndex === -1 ? [] : stashedPullBatch.slice(newPartialBatchIndex)
+                      const combinedBatch = [...msg.batch, ...batchWithNewStashedEvents]
+                      return Option.some({ ...msg, batch: combinedBatch, remaining: 0 })
+                    } else {
+                      return Option.some(msg)
+                    }
+                  } else {
+                    // Ignore
+                    return Option.none()
+                  }
+                } else {
+                  if (pullResponseReceived) {
+                    return Option.some(msg)
+                  } else {
+                    stashedPullBatch.push(...msg.batch)
+                    return Option.none()
+                  }
+                }
+              }
+
+              return Option.none()
+            }),
+            // This call is mostly here to for type narrowing
+            Stream.filter(Schema.is(WSMessage.PullRes)),
           )
         }).pipe(Stream.unwrap),
 
@@ -78,15 +114,12 @@ export const makeWsSync = (options: WsSyncOptions): Effect.Effect<SyncBackend<Sy
           const requestId = nanoid()
 
           yield* Stream.fromPubSub(incomingMessages).pipe(
-            Stream.filter((_) => _._tag !== 'WSMessage.PushBroadcast' && _.requestId === requestId),
             Stream.tap((_) =>
-              _._tag === 'WSMessage.Error'
+              _._tag === 'WSMessage.Error' && _.requestId === requestId
                 ? Deferred.fail(ready, new InvalidPushError({ reason: { _tag: 'Unexpected', message: _.message } }))
                 : Effect.void,
             ),
-            Stream.filter(Schema.is(WSMessage.PushAck)),
-            // TODO bring back filterting of "own events"
-            // Stream.filter((_) => _.mutationId === mutationEventEncoded.id.global),
+            Stream.filter((_) => _._tag === 'WSMessage.PushAck' && _.requestId === requestId),
             Stream.take(1),
             Stream.tap(() => Deferred.succeed(ready, void 0)),
             Stream.runDrain,
@@ -123,21 +156,23 @@ const connect = (wsUrl: string) =>
         // Wait first until we're online
         yield* waitUntilOnline
 
-        yield* Effect.spanEvent(
-          `Sending message: ${message._tag}`,
-          message._tag === 'WSMessage.PushReq'
-            ? {
-                id: message.batch[0]!.id.global,
-                parentId: message.batch[0]!.parentId.global,
-                batchLength: message.batch.length,
-              }
-            : message._tag === 'WSMessage.PullReq'
-              ? { cursor: message.cursor ?? '-' }
-              : {},
-        )
-
         // TODO use MsgPack instead of JSON to speed up the serialization / reduce the size of the messages
         socketRef.current!.send(Schema.encodeSync(Schema.parseJson(WSMessage.Message))(message))
+
+        if (LS_DEV) {
+          yield* Effect.spanEvent(
+            `Sent message: ${message._tag}`,
+            message._tag === 'WSMessage.PushReq'
+              ? {
+                  id: message.batch[0]!.id,
+                  parentId: message.batch[0]!.parentId,
+                  batchLength: message.batch.length,
+                }
+              : message._tag === 'WSMessage.PullReq'
+                ? { cursor: message.cursor ?? '-' }
+                : {},
+          )
+        }
       })
 
     const innerConnect = Effect.gen(function* () {
@@ -146,11 +181,13 @@ const connect = (wsUrl: string) =>
       while (typeof navigator !== 'undefined' && navigator.onLine === false) {
         yield* Effect.sleep(1000)
       }
+      // TODO bring this back in a cross-platform way
       // if (navigator.onLine === false) {
       //   yield* Effect.async((cb) => self.addEventListener('online', () => cb(Effect.void)))
       // }
 
       const socket = yield* WebSocket.makeWebSocket({ url: wsUrl, reconnect: Schedule.exponential(100) })
+      // socket.binaryType = 'arraybuffer'
 
       yield* SubscriptionRef.set(isConnected, true)
       socketRef.current = socket
@@ -191,7 +228,8 @@ const connect = (wsUrl: string) =>
       // NOTE it seems that this callback doesn't work reliably on a worker but only via `window.addEventListener`
       // We might need to proxy the event from the main thread to the worker if we want this to work reliably.
       // eslint-disable-next-line unicorn/prefer-global-this
-      if (typeof self !== 'undefined') {
+      if (typeof self !== 'undefined' && typeof self.addEventListener === 'function') {
+        // TODO support an Expo equivalent for this
         // eslint-disable-next-line unicorn/prefer-global-this
         yield* Effect.eventListener(self, 'offline', () => Deferred.succeed(connectionClosed, void 0))
       }

package/dist/cf-worker/index.d.ts

@@ -1,8 +0,0 @@
-/// <reference no-default-lib="true"/>
-import type { Env } from './durable-object.js';
-export * from './durable-object.js';
-declare const _default: {
-    fetch: (request: Request, env: Env, _ctx: ExecutionContext) => Promise<Response>;
-};
-export default _default;
-//# sourceMappingURL=index.d.ts.map

package/dist/cf-worker/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cf-worker/index.ts"],"names":[],"mappings":";AAMA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAA;AAE9C,cAAc,qBAAqB,CAAA;;qBA4CV,OAAO,OAAO,GAAG,QAAQ,gBAAgB,KAAG,OAAO,CAAC,QAAQ,CAAC;;AADtF,wBAgCC"}

package/dist/cf-worker/index.js

@@ -1,67 +0,0 @@
-/// <reference no-default-lib="true"/>
-/// <reference lib="esnext" />
-export * from './durable-object.js';
-// const handleRequest = (request: Request, env: Env) =>
-//   HttpServer.router.empty.pipe(
-//     HttpServer.router.get(
-//       '/websocket',
-//       Effect.gen(function* () {
-//         // This example will refer to the same Durable Object instance,
-//         // since the name "foo" is hardcoded.
-//         const id = env.WEBSOCKET_SERVER.idFromName('foo')
-//         const durableObject = env.WEBSOCKET_SERVER.get(id)
-//         HttpServer.
-//         // Expect to receive a WebSocket Upgrade request.
-//         // If there is one, accept the request and return a WebSocket Response.
-//         const headerRes = yield* HttpServer.request
-//           .schemaHeaders(
-//             Schema.Struct({
-//               Upgrade: Schema.Literal('websocket'),
-//             }),
-//           )
-//           .pipe(Effect.either)
-//         if (headerRes._tag === 'Left') {
-//           // return new Response('Durable Object expected Upgrade: websocket', { status: 426 })
-//           return yield* HttpServer.response.text('Durable Object expected Upgrade: websocket', { status: 426 })
-//         }
-//         HttpServer.response.empty
-//         return yield* Effect.promise(() => durableObject.fetch(request))
-//       }),
-//     ),
-//     HttpServer.router.catchAll((e) => {
-//       console.log(e)
-//       return HttpServer.response.empty({ status: 400 })
-//     }),
-//     (_) => HttpServer.app.toWebHandler(_)(request),
-//     // request
-//   )
-// Worker
-export default {
-    fetch: async (request, env, _ctx) => {
-        const url = new URL(request.url);
-        const searchParams = url.searchParams;
-        const roomId = searchParams.get('room');
-        if (roomId === null) {
-            return new Response('Room ID is required', { status: 400 });
-        }
-        // This example will refer to the same Durable Object instance,
-        // since the name "foo" is hardcoded.
-        const id = env.WEBSOCKET_SERVER.idFromName(roomId);
-        const durableObject = env.WEBSOCKET_SERVER.get(id);
-        if (url.pathname.endsWith('/websocket')) {
-            const upgradeHeader = request.headers.get('Upgrade');
-            if (!upgradeHeader || upgradeHeader !== 'websocket') {
-                return new Response('Durable Object expected Upgrade: websocket', { status: 426 });
-            }
-            return durableObject.fetch(request);
-        }
-        return new Response(null, {
-            status: 400,
-            statusText: 'Bad Request',
-            headers: {
-                'Content-Type': 'text/plain',
-            },
-        });
-    },
-};
-//# sourceMappingURL=index.js.map

package/dist/cf-worker/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cf-worker/index.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,8BAA8B;AAO9B,cAAc,qBAAqB,CAAA;AAEnC,wDAAwD;AACxD,kCAAkC;AAClC,6BAA6B;AAC7B,sBAAsB;AACtB,kCAAkC;AAClC,0EAA0E;AAC1E,gDAAgD;AAChD,4DAA4D;AAC5D,6DAA6D;AAE7D,sBAAsB;AAEtB,4DAA4D;AAC5D,kFAAkF;AAClF,sDAAsD;AACtD,4BAA4B;AAC5B,8BAA8B;AAC9B,sDAAsD;AACtD,kBAAkB;AAClB,cAAc;AACd,iCAAiC;AAEjC,2CAA2C;AAC3C,kGAAkG;AAClG,kHAAkH;AAClH,YAAY;AAEZ,oCAAoC;AAEpC,2EAA2E;AAC3E,YAAY;AACZ,SAAS;AACT,0CAA0C;AAC1C,uBAAuB;AACvB,0DAA0D;AAC1D,UAAU;AACV,sDAAsD;AACtD,iBAAiB;AACjB,MAAM;AAEN,SAAS;AACT,eAAe;IACb,KAAK,EAAE,KAAK,EAAE,OAAgB,EAAE,GAAQ,EAAE,IAAsB,EAAqB,EAAE;QACrF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAChC,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,CAAA;QACrC,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;QAEvC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,OAAO,IAAI,QAAQ,CAAC,qBAAqB,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,+DAA+D;QAC/D,qCAAqC;QACrC,MAAM,EAAE,GAAG,GAAG,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;QAClD,MAAM,aAAa,GAAG,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAElD,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YACpD,IAAI,CAAC,aAAa,IAAI,aAAa,KAAK,WAAW,EAAE,CAAC;gBACpD,OAAO,IAAI,QAAQ,CAAC,4CAA4C,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;YACpF,CAAC;YAED,OAAO,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QACrC,CAAC;QAED,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;YACxB,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,aAAa;YACzB,OAAO,EAAE;gBACP,cAAc,EAAE,YAAY;aAC7B;SACF,CAAC,CAAA;IACJ,CAAC;CACF,CAAA"}

package/dist/common/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/common/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,uBAAuB,CAAA"}

package/dist/common/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/common/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,uBAAuB,CAAA"}

package/dist/sync-impl/index.d.ts

@@ -1,2 +0,0 @@
-export * from './ws-impl.js';
-//# sourceMappingURL=index.d.ts.map

package/dist/sync-impl/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sync-impl/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAA"}

package/dist/sync-impl/index.js

@@ -1,2 +0,0 @@
-export * from './ws-impl.js';
-//# sourceMappingURL=index.js.map

package/dist/sync-impl/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sync-impl/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAA"}

package/src/cf-worker/index.ts

@@ -1,84 +0,0 @@
-/// <reference no-default-lib="true"/>
-/// <reference lib="esnext" />
-
-// import { EncodedAny } from '@livestore/common/schema'
-// import { Effect, HttpServer, Schema } from '@livestore/utils/effect'
-
-import type { Env } from './durable-object.js'
-
-export * from './durable-object.js'
-
-// const handleRequest = (request: Request, env: Env) =>
-//   HttpServer.router.empty.pipe(
-//     HttpServer.router.get(
-//       '/websocket',
-//       Effect.gen(function* () {
-//         // This example will refer to the same Durable Object instance,
-//         // since the name "foo" is hardcoded.
-//         const id = env.WEBSOCKET_SERVER.idFromName('foo')
-//         const durableObject = env.WEBSOCKET_SERVER.get(id)
-
-//         HttpServer.
-
-//         // Expect to receive a WebSocket Upgrade request.
-//         // If there is one, accept the request and return a WebSocket Response.
-//         const headerRes = yield* HttpServer.request
-//           .schemaHeaders(
-//             Schema.Struct({
-//               Upgrade: Schema.Literal('websocket'),
-//             }),
-//           )
-//           .pipe(Effect.either)
-
-//         if (headerRes._tag === 'Left') {
-//           // return new Response('Durable Object expected Upgrade: websocket', { status: 426 })
-//           return yield* HttpServer.response.text('Durable Object expected Upgrade: websocket', { status: 426 })
-//         }
-
-//         HttpServer.response.empty
-
-//         return yield* Effect.promise(() => durableObject.fetch(request))
-//       }),
-//     ),
-//     HttpServer.router.catchAll((e) => {
-//       console.log(e)
-//       return HttpServer.response.empty({ status: 400 })
-//     }),
-//     (_) => HttpServer.app.toWebHandler(_)(request),
-//     // request
-//   )
-
-// Worker
-export default {
-  fetch: async (request: Request, env: Env, _ctx: ExecutionContext): Promise<Response> => {
-    const url = new URL(request.url)
-    const searchParams = url.searchParams
-    const roomId = searchParams.get('room')
-
-    if (roomId === null) {
-      return new Response('Room ID is required', { status: 400 })
-    }
-
-    // This example will refer to the same Durable Object instance,
-    // since the name "foo" is hardcoded.
-    const id = env.WEBSOCKET_SERVER.idFromName(roomId)
-    const durableObject = env.WEBSOCKET_SERVER.get(id)
-
-    if (url.pathname.endsWith('/websocket')) {
-      const upgradeHeader = request.headers.get('Upgrade')
-      if (!upgradeHeader || upgradeHeader !== 'websocket') {
-        return new Response('Durable Object expected Upgrade: websocket', { status: 426 })
-      }
-
-      return durableObject.fetch(request)
-    }
-
-    return new Response(null, {
-      status: 400,
-      statusText: 'Bad Request',
-      headers: {
-        'Content-Type': 'text/plain',
-      },
-    })
-  },
-}