@livestore/sync-cf 0.4.0-dev.7 → 0.4.0-dev.8

package/dist/cf-worker/do/pull.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pull.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,gBAAgB,EAAgC,MAAM,mBAAmB,CAAA;AAC1G,OAAO,EAAuC,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAClG,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAWlC,eAAO,MAAM,oBAAoB,GAC/B,KAAK,WAAW,CAAC,WAAW,EAC5B,SAAS,MAAM,CAAC,SAAS,GAAG,SAAS,KACpC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,CA6C/D,CAAA"}
+{"version":3,"file":"pull.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,gBAAgB,EAAgC,MAAM,mBAAmB,CAAA;AAC1G,OAAO,EAAyB,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACpF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAWlC,eAAO,MAAM,oBAAoB,GAC/B,KAAK,WAAW,CAAC,WAAW,EAC5B,SAAS,MAAM,CAAC,SAAS,GAAG,SAAS,KACpC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,CA8C/D,CAAA"}

package/dist/cf-worker/do/pull.js

@@ -1,5 +1,5 @@
 import { BackendIdMismatchError, InvalidPullError, SyncBackend, UnexpectedError } from '@livestore/common';
-import { Effect, Option, pipe, ReadonlyArray, Stream } from '@livestore/utils/effect';
+import { Chunk, Effect, Option, Stream } from '@livestore/utils/effect';
 import { SyncMessage } from "../../common/mod.js";
 import { PULL_CHUNK_SIZE } from "../shared.js";
 import { DoCtx } from "./layer.js";
@@ -20,17 +20,19 @@ export const makeEndingPullStream = (req, payload) => Effect.gen(function* () {
     if (req.cursor._tag === 'Some' && req.cursor.value.backendId !== backendId) {
         return yield* new BackendIdMismatchError({ expected: backendId, received: req.cursor.value.backendId });
     }
-    // TODO use streaming for db results
-    const remainingEvents = yield* storage.getEvents(Option.getOrUndefined(req.cursor)?.eventSequenceNumber);
-    const batches = pipe(remainingEvents, ReadonlyArray.chunksOf(PULL_CHUNK_SIZE), ReadonlyArray.map((batch, i) => {
-        const remaining = Math.max(0, remainingEvents.length - (i + 1) * PULL_CHUNK_SIZE);
-        return SyncMessage.PullResponse.make({
-            batch,
-            pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
-            backendId,
-        });
-    }));
-    return Stream.fromIterable(batches).pipe(Stream.tap(Effect.fn(function* (res) {
+    const { stream: storedEvents, total } = yield* storage.getEvents(Option.getOrUndefined(req.cursor)?.eventSequenceNumber);
+    return storedEvents.pipe(Stream.grouped(PULL_CHUNK_SIZE), Stream.mapAccum(total, (remaining, chunk) => {
+        const asArray = Chunk.toReadonlyArray(chunk);
+        const nextRemaining = Math.max(0, remaining - asArray.length);
+        return [
+            nextRemaining,
+            SyncMessage.PullResponse.make({
+                batch: asArray,
+                pageInfo: nextRemaining > 0 ? SyncBackend.pageInfoMoreKnown(nextRemaining) : SyncBackend.pageInfoNoMore,
+                backendId,
+            }),
+        ];
+    }), Stream.tap(Effect.fn(function* (res) {
         if (doOptions?.onPullRes) {
             yield* Effect.tryAll(() => doOptions.onPullRes(res)).pipe(UnexpectedError.mapToUnexpectedError);
         }

package/dist/cf-worker/do/pull.js.map

@@ -1 +1 @@
-{"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/cf-worker/do/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAC1G,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAe,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAClG,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,4BAA4B;AAC5B,6EAA6E;AAC7E,EAAE;AACF,aAAa;AACb,yDAAyD;AACzD,6GAA6G;AAC7G,UAAU;AACV,yDAAyD;AACzD,wDAAwD;AACxD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,GAA4B,EAC5B,OAAqC,EAC6B,EAAE,CACpE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAE/D,IAAI,SAAS,EAAE,MAAM,EAAE,CAAC;QACtB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAU,CAAC,MAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAC5E,eAAe,CAAC,oBAAoB,CACrC,CAAA;IACH,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC3E,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA;IACzG,CAAC;IAED,oCAAoC;IACpC,MAAM,eAAe,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,mBAAmB,CAAC,CAAA;IAExG,MAAM,OAAO,GAAG,IAAI,CAClB,eAAe,EACf,aAAa,CAAC,QAAQ,CAAC,eAAe,CAAC,EACvC,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,CAAA;QAEjF,OAAO,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;YACnC,KAAK;YACL,QAAQ,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC/F,SAAS;SACV,CAAC,CAAA;IACJ,CAAC,CAAC,CACH,CAAA;IAED,OAAO,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,CACtC,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,GAAG;QACtB,IAAI,SAAS,EAAE,SAAS,EAAE,CAAC;YACzB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,SAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;QAClG,CAAC;IACH,CAAC,CAAC,CACH,EACD,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAC7D,CAAA;AACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,EAC5D,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAC5C,CAAA"}
+{"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/cf-worker/do/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAC1G,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAe,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACpF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,4BAA4B;AAC5B,6EAA6E;AAC7E,EAAE;AACF,aAAa;AACb,yDAAyD;AACzD,6GAA6G;AAC7G,UAAU;AACV,yDAAyD;AACzD,wDAAwD;AACxD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,GAA4B,EAC5B,OAAqC,EAC6B,EAAE,CACpE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAE/D,IAAI,SAAS,EAAE,MAAM,EAAE,CAAC;QACtB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAU,CAAC,MAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAC5E,eAAe,CAAC,oBAAoB,CACrC,CAAA;IACH,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC3E,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA;IACzG,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,CAC9D,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,mBAAmB,CACvD,CAAA;IAED,OAAO,YAAY,CAAC,IAAI,CACtB,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAC/B,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;QAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;QAE7D,OAAO;YACL,aAAa;YACb,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;gBAC5B,KAAK,EAAE,OAAO;gBACd,QAAQ,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;gBACvG,SAAS;aACV,CAAC;SACM,CAAA;IACZ,CAAC,CAAC,EACF,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,GAAG;QACtB,IAAI,SAAS,EAAE,SAAS,EAAE,CAAC;YACzB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,SAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;QAClG,CAAC;IACH,CAAC,CAAC,CACH,EACD,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAC7D,CAAA;AACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,EAC5D,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAC5C,CAAA"}

package/dist/cf-worker/do/push.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"push.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/push.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB,EAIjB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,KAAK,OAAO,EAAsB,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAE,MAAM,EAA2B,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,6BAA6B,EAAE,KAAK,OAAO,EAA6B,MAAM,cAAc,CAAA;AACpH,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,eAAO,MAAM,QAAQ,GAClB,0CAME;IACD,OAAO,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IACrC,OAAO,EAAE,6BAA6B,GAAG,SAAS,CAAA;IAClD,OAAO,EAAE,OAAO,CAAA;IAChB,GAAG,EAAE,OAAO,CAAC,kBAAkB,CAAA;IAC/B,GAAG,EAAE,GAAG,CAAA;CACT,MACA,aAAa,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,CAAC,+CAuHhD,CAAA"}
+{"version":3,"file":"push.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/push.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB,EAIjB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,KAAK,OAAO,EAAsB,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAE,MAAM,EAA2B,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,6BAA6B,EAAE,KAAK,OAAO,EAA6B,MAAM,cAAc,CAAA;AACpH,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,eAAO,MAAM,QAAQ,GAClB,0CAME;IACD,OAAO,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IACrC,OAAO,EAAE,6BAA6B,GAAG,SAAS,CAAA;IAClD,OAAO,EAAE,OAAO,CAAA;IAChB,GAAG,EAAE,OAAO,CAAC,kBAAkB,CAAA;IAC/B,GAAG,EAAE,GAAG,CAAA;CACT,MACA,aAAa,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,CAAC,+CAwHhD,CAAA"}

package/dist/cf-worker/do/push.js

@@ -87,7 +87,7 @@ export const makePush = ({ payload, options, storeId, ctx, env, }) => (pushReque
     if (options?.onPushRes) {
         yield* Effect.tryAll(() => options.onPushRes(message)).pipe(UnexpectedError.mapToUnexpectedError);
     }
-})), Effect.mapError((cause) => InvalidPushError.make({ cause })));
+})), Effect.mapError((cause) => InvalidPushError.make({ cause })), Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }));
 /**
  * @see https://developers.cloudflare.com/durable-objects/api/state/#blockconcurrencywhile
  */

package/dist/cf-worker/do/push.js.map

@@ -1 +1 @@
-{"version":3,"file":"push.js","sourceRoot":"","sources":["../../../src/cf-worker/do/push.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,eAAe,GAChB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAgB,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAmB,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAA8D,yBAAyB,EAAE,MAAM,cAAc,CAAA;AACpH,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,MAAM,CAAC,MAAM,QAAQ,GACnB,CAAC,EACC,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,EACH,GAAG,GAOJ,EAAE,EAAE,CACL,CAAC,WAAkD,EAAE,EAAE,CACrD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,2FAA2F;IAC3F,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAEhG,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAO,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CACjF,eAAe,CAAC,oBAAoB,CACrC,CAAA;IACH,CAAC;IAED,IAAI,WAAW,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACvF,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAA;IAC1G,CAAC;IAED,2EAA2E;IAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC/C,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAA;QAC1C,uCAAuC;QACvC,qBAAqB;QACrB,MAAM,gBAAgB,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,YAAY,CAAA;QAC3D,IAAI,gBAAgB,KAAK,WAAW,EAAE,CAAC;YACrC,OAAO,KAAK,CAAC,CAAC,IAAI,gBAAgB,CAAC,EAAE,kBAAkB,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAA;QACxG,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAE1C,6EAA6E;QAC7E,KAAK,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAEzD,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAA;QAEnD,OAAO,EAAE,SAAS,EAAE,CAAA;IACtB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAA;IAEnC,kEAAkE;IAClE,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACzB,MAAM,gBAAgB,GAAG,GAAG,CAAC,aAAa,EAAE,CAAA;QAE5C,6CAA6C;QAC7C,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;YAC5C,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC9C,YAAY;gBACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;aACpE,CAAC,CAAC;YACH,QAAQ,EAAE,WAAW,CAAC,cAAc;YACpC,SAAS;SACV,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAA;QAEvE,iCAAiC;QACjC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,6BAA6B;YAC7B,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;gBACvB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;YACpG,CAAC;YAED,iFAAiF;YACjF,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;gBACpC,wBAAwB;gBACxB,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAA;gBAE7F,+FAA+F;gBAC/F,sFAAsF;gBACtF,6EAA6E;gBAC7E,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;oBAClD,MAAM,GAAG,GAAoC;wBAC3C,IAAI,EAAE,OAAO;wBACb,SAAS;wBACT,MAAM,EAAE,CAAC,UAAU,CAAC;qBACrB,CAAA;oBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;gBAChC,CAAC;YACH,CAAC;YAED,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,gBAAgB,CAAC,MAAM,oBAAoB,CAAC,CAAA;QACvF,CAAC;QAED,4EAA4E;QAC5E,IAAI,gBAAgB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,gBAAgB,CAAC,MAAM,EAAE,EACzB,CAAC,YAAY,EAAE,EAAE,CACf,kBAAkB,CAAC;gBACjB,aAAa,EAAE,YAAY,CAAC,aAAa;gBACzC,GAAG;gBACH,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,MAAM,EAAE,CAAC,UAAU,CAAC;aACrB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,EAChD,EAAE,WAAW,EAAE,WAAW,EAAE,CAC7B,CAAA;YAED,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,gBAAgB,CAAC,IAAI,cAAc,CAAC,CAAA;QAC/E,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,iBAAiB,EACxB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EACrC,MAAM,CAAC,eAAe,EAAE,+DAA+D;IACvF,MAAM,CAAC,IAAI,CACZ,CAAA;IAED,8GAA8G;IAC9G,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAA;IAExB,OAAO,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACrC,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,OAAO;IAC1B,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;QACvB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;IACpG,CAAC;AACH,CAAC,CAAC,CACH,EACD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAC7D,CAAA;AAEL;;GAEG;AACH,MAAM,qBAAqB,GACzB,CAAC,GAA+B,EAAE,EAAE,CACpC,CAAU,GAA2B,EAAE,EAAE,CACvC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,EAAK,CAAA;IAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CACtC,GAAG,CAAC,qBAAqB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAC1F,CAAA;IAED,OAAO,KAAK,CAAC,CAAC,IAAI,CAAA;AACpB,CAAC,CAAC,CAAA"}
+{"version":3,"file":"push.js","sourceRoot":"","sources":["../../../src/cf-worker/do/push.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,eAAe,GAChB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAgB,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAmB,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAA8D,yBAAyB,EAAE,MAAM,cAAc,CAAA;AACpH,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,MAAM,CAAC,MAAM,QAAQ,GACnB,CAAC,EACC,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,EACH,GAAG,GAOJ,EAAE,EAAE,CACL,CAAC,WAAkD,EAAE,EAAE,CACrD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,2FAA2F;IAC3F,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAEhG,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAO,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CACjF,eAAe,CAAC,oBAAoB,CACrC,CAAA;IACH,CAAC;IAED,IAAI,WAAW,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACvF,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAA;IAC1G,CAAC;IAED,2EAA2E;IAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC/C,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAA;QAC1C,uCAAuC;QACvC,qBAAqB;QACrB,MAAM,gBAAgB,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,YAAY,CAAA;QAC3D,IAAI,gBAAgB,KAAK,WAAW,EAAE,CAAC;YACrC,OAAO,KAAK,CAAC,CAAC,IAAI,gBAAgB,CAAC,EAAE,kBAAkB,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAA;QACxG,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAE1C,6EAA6E;QAC7E,KAAK,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAEzD,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAA;QAEnD,OAAO,EAAE,SAAS,EAAE,CAAA;IACtB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAA;IAEnC,kEAAkE;IAClE,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACzB,MAAM,gBAAgB,GAAG,GAAG,CAAC,aAAa,EAAE,CAAA;QAE5C,6CAA6C;QAC7C,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;YAC5C,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC9C,YAAY;gBACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;aACpE,CAAC,CAAC;YACH,QAAQ,EAAE,WAAW,CAAC,cAAc;YACpC,SAAS;SACV,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAA;QAEvE,iCAAiC;QACjC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,6BAA6B;YAC7B,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;gBACvB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;YACpG,CAAC;YAED,iFAAiF;YACjF,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;gBACpC,wBAAwB;gBACxB,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAA;gBAE7F,+FAA+F;gBAC/F,sFAAsF;gBACtF,6EAA6E;gBAC7E,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;oBAClD,MAAM,GAAG,GAAoC;wBAC3C,IAAI,EAAE,OAAO;wBACb,SAAS;wBACT,MAAM,EAAE,CAAC,UAAU,CAAC;qBACrB,CAAA;oBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;gBAChC,CAAC;YACH,CAAC;YAED,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,gBAAgB,CAAC,MAAM,oBAAoB,CAAC,CAAA;QACvF,CAAC;QAED,4EAA4E;QAC5E,IAAI,gBAAgB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,gBAAgB,CAAC,MAAM,EAAE,EACzB,CAAC,YAAY,EAAE,EAAE,CACf,kBAAkB,CAAC;gBACjB,aAAa,EAAE,YAAY,CAAC,aAAa;gBACzC,GAAG;gBACH,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,MAAM,EAAE,CAAC,UAAU,CAAC;aACrB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,EAChD,EAAE,WAAW,EAAE,WAAW,EAAE,CAC7B,CAAA;YAED,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,gBAAgB,CAAC,IAAI,cAAc,CAAC,CAAA;QAC/E,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,iBAAiB,EACxB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EACrC,MAAM,CAAC,eAAe,EAAE,+DAA+D;IACvF,MAAM,CAAC,IAAI,CACZ,CAAA;IAED,8GAA8G;IAC9G,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAA;IAExB,OAAO,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACrC,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,OAAO;IAC1B,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;QACvB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;IACpG,CAAC;AACH,CAAC,CAAC,CACH,EACD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,EAC5D,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CACrG,CAAA;AAEL;;GAEG;AACH,MAAM,qBAAqB,GACzB,CAAC,GAA+B,EAAE,EAAE,CACpC,CAAU,GAA2B,EAAE,EAAE,CACvC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,EAAK,CAAA;IAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CACtC,GAAG,CAAC,qBAAqB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAC1F,CAAA;IAED,OAAO,KAAK,CAAC,CAAC,IAAI,CAAA;AACpB,CAAC,CAAC,CAAA"}

package/dist/cf-worker/do/sync-storage.d.ts

@@ -1,15 +1,18 @@
 import { UnexpectedError } from '@livestore/common';
 import type { LiveStoreEvent } from '@livestore/common/schema';
 import type { CfTypes } from '@livestore/common-cf';
-import { Effect, Option } from '@livestore/utils/effect';
+import { Effect, Option, Stream } from '@livestore/utils/effect';
 import { SyncMetadata } from '../../common/sync-message-types.ts';
 import { type Env, type StoreId } from '../shared.ts';
 export type SyncStorage = {
     dbName: string;
-    getEvents: (cursor: number | undefined) => Effect.Effect<ReadonlyArray<{
-        eventEncoded: LiveStoreEvent.AnyEncodedGlobal;
-        metadata: Option.Option<SyncMetadata>;
-    }>, UnexpectedError>;
+    getEvents: (cursor: number | undefined) => Effect.Effect<{
+        total: number;
+        stream: Stream.Stream<{
+            eventEncoded: LiveStoreEvent.AnyEncodedGlobal;
+            metadata: Option.Option<SyncMetadata>;
+        }, UnexpectedError>;
+    }, UnexpectedError>;
     appendEvents: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>, createdAt: string) => Effect.Effect<void, UnexpectedError>;
     resetStore: Effect.Effect<void, UnexpectedError>;
 };

package/dist/cf-worker/do/sync-storage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sync-storage.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/sync-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAU,MAAM,yBAAyB,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAA;AACjE,OAAO,EAAE,KAAK,GAAG,EAA8B,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AAGjF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAA;IAEd,SAAS,EAAE,CACT,MAAM,EAAE,MAAM,GAAG,SAAS,KACvB,MAAM,CAAC,MAAM,CAChB,aAAa,CAAC;QAAE,YAAY,EAAE,cAAc,CAAC,gBAAgB,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;KAAE,CAAC,EACvG,eAAe,CAChB,CAAA;IACD,YAAY,EAAE,CACZ,KAAK,EAAE,aAAa,CAAC,cAAc,CAAC,gBAAgB,CAAC,EACrD,SAAS,EAAE,MAAM,KACd,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IACzC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;CACjD,CAAA;AAED,eAAO,MAAM,WAAW,GAAI,KAAK,OAAO,CAAC,kBAAkB,EAAE,KAAK,GAAG,EAAE,SAAS,OAAO,KAAG,WAmGzF,CAAA"}
+{"version":3,"file":"sync-storage.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/sync-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAS,MAAM,EAAE,MAAM,EAAU,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAA;AACjE,OAAO,EAAE,KAAK,GAAG,EAA8B,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AAGjF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,KAAK,MAAM,CAAC,MAAM,CACtD;QACE,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,EAAE,MAAM,CAAC,MAAM,CACnB;YAAE,YAAY,EAAE,cAAc,CAAC,gBAAgB,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;SAAE,EACxF,eAAe,CAChB,CAAA;KACF,EACD,eAAe,CAChB,CAAA;IACD,YAAY,EAAE,CACZ,KAAK,EAAE,aAAa,CAAC,cAAc,CAAC,gBAAgB,CAAC,EACrD,SAAS,EAAE,MAAM,KACd,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IACzC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;CACjD,CAAA;AAED,eAAO,MAAM,WAAW,GAAI,KAAK,OAAO,CAAC,kBAAkB,EAAE,KAAK,GAAG,EAAE,SAAS,OAAO,KAAG,WA+KzF,CAAA"}

package/dist/cf-worker/do/sync-storage.js

@@ -1,5 +1,5 @@
 import { UnexpectedError } from '@livestore/common';
-import { Effect, Option, Schema } from '@livestore/utils/effect';
+import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect';
 import { SyncMetadata } from "../../common/sync-message-types.js";
 import { PERSISTENCE_FORMAT_VERSION } from "../shared.js";
 import { eventlogTable } from "./sqlite.js";
@@ -9,25 +9,70 @@ export const makeStorage = (ctx, env, storeId) => {
         try: () => cb(env.DB),
         catch: (error) => new UnexpectedError({ cause: error, payload: { dbName } }),
     }).pipe(Effect.map((_) => _.results), Effect.withSpan('@livestore/sync-cf:durable-object:execDb'));
-    // const getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError> = Effect.gen(
-    //   function* () {
-    //     const result = yield* execDb<{ seqNum: EventSequenceNumber.GlobalEventSequenceNumber }>((db) =>
-    //       db.prepare(`SELECT seqNum FROM ${dbName} ORDER BY seqNum DESC LIMIT 1`).all(),
-    //     )
-    //     return result[0]?.seqNum ?? EventSequenceNumber.ROOT.global
-    //   },
-    // ).pipe(UnexpectedError.mapToUnexpectedError)
-    // TODO support streaming
+    // Cloudflare's D1 HTTP endpoint rejects JSON responses once they exceed ~1MB.
+    // Keep individual SELECT batches comfortably below that threshold so we can
+    // serve large histories without tripping the limit.
+    const D1_MAX_JSON_RESPONSE_BYTES = 1_000_000;
+    const D1_RESPONSE_SAFETY_MARGIN_BYTES = 64 * 1024;
+    const D1_TARGET_RESPONSE_BYTES = D1_MAX_JSON_RESPONSE_BYTES - D1_RESPONSE_SAFETY_MARGIN_BYTES;
+    const D1_INITIAL_PAGE_SIZE = 256;
+    const D1_MIN_PAGE_SIZE = 1;
+    const decodeEventlogRows = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema));
+    const textEncoder = new TextEncoder();
+    const decreaseLimit = (limit) => Math.max(D1_MIN_PAGE_SIZE, Math.floor(limit / 2));
+    const increaseLimit = (limit) => Math.min(D1_INITIAL_PAGE_SIZE, limit * 2);
+    const computeNextLimit = (limit, encodedSize) => {
+        if (encodedSize > D1_TARGET_RESPONSE_BYTES && limit > D1_MIN_PAGE_SIZE) {
+            const next = decreaseLimit(limit);
+            return next === limit ? limit : next;
+        }
+        if (encodedSize < D1_TARGET_RESPONSE_BYTES / 2 && limit < D1_INITIAL_PAGE_SIZE) {
+            const next = increaseLimit(limit);
+            return next === limit ? limit : next;
+        }
+        return limit;
+    };
     const getEvents = (cursor) => Effect.gen(function* () {
-        const whereClause = cursor === undefined ? '' : `WHERE seqNum > ${cursor}`;
-        const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY seqNum ASC`;
-        // TODO handle case where `cursor` was not found
-        const rawEvents = yield* execDb((db) => db.prepare(sql).all());
-        const events = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))(rawEvents).map(({ createdAt, ...eventEncoded }) => ({
-            eventEncoded,
-            metadata: Option.some(SyncMetadata.make({ createdAt })),
-        }));
-        return events;
+        const countStatement = cursor === undefined
+            ? `SELECT COUNT(*) as total FROM ${dbName}`
+            : `SELECT COUNT(*) as total FROM ${dbName} WHERE seqNum > ?`;
+        const countRows = yield* execDb((db) => {
+            const prepared = db.prepare(countStatement);
+            return cursor === undefined ? prepared.all() : prepared.bind(cursor).all();
+        });
+        const total = Number(countRows[0]?.total ?? 0);
+        const initialState = { cursor, limit: D1_INITIAL_PAGE_SIZE };
+        const fetchPage = (state) => Effect.gen(function* () {
+            const statement = state.cursor === undefined
+                ? `SELECT * FROM ${dbName} ORDER BY seqNum ASC LIMIT ?`
+                : `SELECT * FROM ${dbName} WHERE seqNum > ? ORDER BY seqNum ASC LIMIT ?`;
+            const rawEvents = yield* execDb((db) => {
+                const prepared = db.prepare(statement);
+                return state.cursor === undefined
+                    ? prepared.bind(state.limit).all()
+                    : prepared.bind(state.cursor, state.limit).all();
+            });
+            if (rawEvents.length === 0) {
+                return Option.none();
+            }
+            const encodedSize = textEncoder.encode(JSON.stringify(rawEvents)).byteLength;
+            if (encodedSize > D1_TARGET_RESPONSE_BYTES && state.limit > D1_MIN_PAGE_SIZE) {
+                const nextLimit = decreaseLimit(state.limit);
+                if (nextLimit !== state.limit) {
+                    return yield* fetchPage({ cursor: state.cursor, limit: nextLimit });
+                }
+            }
+            const decodedRows = Chunk.fromIterable(decodeEventlogRows(rawEvents));
+            const eventsChunk = Chunk.map(decodedRows, ({ createdAt, ...eventEncoded }) => ({
+                eventEncoded,
+                metadata: Option.some(SyncMetadata.make({ createdAt })),
+            }));
+            const lastSeqNum = Chunk.unsafeLast(decodedRows).seqNum;
+            const nextState = { cursor: lastSeqNum, limit: computeNextLimit(state.limit, encodedSize) };
+            return Option.some([eventsChunk, nextState]);
+        });
+        const stream = Stream.unfoldChunkEffect(initialState, fetchPage);
+        return { total, stream };
     }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/sync-cf:durable-object:getEvents', { attributes: { dbName, cursor } }));
     const appendEvents = (batch, createdAt) => Effect.gen(function* () {
         // If there are no events, do nothing.

package/dist/cf-worker/do/sync-storage.js.map

@@ -1 +1 @@
-{"version":3,"file":"sync-storage.js","sourceRoot":"","sources":["../../../src/cf-worker/do/sync-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAGnD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAA;AACjE,OAAO,EAAY,0BAA0B,EAAgB,MAAM,cAAc,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAkB3C,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAA+B,EAAE,GAAQ,EAAE,OAAgB,EAAe,EAAE;IACtG,MAAM,MAAM,GAAG,YAAY,0BAA0B,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAA;IAEpF,MAAM,MAAM,GAAG,CAAI,EAA4D,EAAE,EAAE,CACjF,MAAM,CAAC,UAAU,CAAC;QAChB,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;KAC7E,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAC5B,MAAM,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAC5D,CAAA;IAEH,6GAA6G;IAC7G,mBAAmB;IACnB,sGAAsG;IACtG,uFAAuF;IACvF,QAAQ;IAER,kEAAkE;IAClE,OAAO;IACP,+CAA+C;IAE/C,yBAAyB;IACzB,MAAM,SAAS,GAAG,CAChB,MAA0B,EAI1B,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,WAAW,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB,MAAM,EAAE,CAAA;QAC1E,MAAM,GAAG,GAAG,iBAAiB,MAAM,IAAI,WAAW,sBAAsB,CAAA;QACxE,gDAAgD;QAChD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAC3F,CAAC,EAAE,SAAS,EAAE,GAAG,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;YACnC,YAAY;YACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;SACxD,CAAC,CACH,CAAA;QACD,OAAO,MAAM,CAAA;IACf,CAAC,CAAC,CAAC,IAAI,CACL,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,6CAA6C,EAAE,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CACnG,CAAA;IAEH,MAAM,YAAY,GAAgC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CACrE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,sCAAsC;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE9B,gBAAgB;QAChB,gFAAgF;QAChF,6EAA6E;QAC7E,MAAM,UAAU,GAAG,EAAE,CAAA;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;YAE5C,wFAAwF;YACxF,MAAM,kBAAkB,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC9E,MAAM,GAAG,GAAG,eAAe,MAAM,8EAA8E,kBAAkB,EAAE,CAAA;YACnI,wDAAwD;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;gBACtC,KAAK,CAAC,MAAM;gBACZ,KAAK,CAAC,YAAY;gBAClB,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;gBAC5D,KAAK,CAAC,IAAI;gBACV,SAAS;gBACT,KAAK,CAAC,QAAQ;gBACd,KAAK,CAAC,SAAS;aAChB,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACnB,EAAE;iBACC,OAAO,CAAC,GAAG,CAAC;iBACZ,IAAI,CAAC,GAAG,MAAM,CAAC;iBACf,GAAG,EAAE,CACT,CAAA;QACH,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,gDAAgD,EAAE;QAChE,UAAU,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE;KAClD,CAAC,CACH,CAAA;IAEH,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CACnE,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAChE,CAAA;IAED,OAAO;QACL,MAAM;QACN,WAAW;QACX,SAAS;QACT,YAAY;QACZ,UAAU;KACX,CAAA;AACH,CAAC,CAAA;AAED,MAAM,gBAAgB,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,eAAe,EAAE,GAAG,CAAC,CAAA"}
+{"version":3,"file":"sync-storage.js","sourceRoot":"","sources":["../../../src/cf-worker/do/sync-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAGnD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAA;AACjE,OAAO,EAAY,0BAA0B,EAAgB,MAAM,cAAc,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAqB3C,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAA+B,EAAE,GAAQ,EAAE,OAAgB,EAAe,EAAE;IACtG,MAAM,MAAM,GAAG,YAAY,0BAA0B,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAA;IAEpF,MAAM,MAAM,GAAG,CAAI,EAA4D,EAAE,EAAE,CACjF,MAAM,CAAC,UAAU,CAAC;QAChB,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;KAC7E,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAC5B,MAAM,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAC5D,CAAA;IAEH,8EAA8E;IAC9E,4EAA4E;IAC5E,oDAAoD;IACpD,MAAM,0BAA0B,GAAG,SAAS,CAAA;IAC5C,MAAM,+BAA+B,GAAG,EAAE,GAAG,IAAI,CAAA;IACjD,MAAM,wBAAwB,GAAG,0BAA0B,GAAG,+BAA+B,CAAA;IAC7F,MAAM,oBAAoB,GAAG,GAAG,CAAA;IAChC,MAAM,gBAAgB,GAAG,CAAC,CAAA;IAE1B,MAAM,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAA;IAC1F,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAErC,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAA;IAC1F,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;IAElF,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAE,WAAmB,EAAE,EAAE;QAC9D,IAAI,WAAW,GAAG,wBAAwB,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;YACvE,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAA;YACjC,OAAO,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;QACtC,CAAC;QAED,IAAI,WAAW,GAAG,wBAAwB,GAAG,CAAC,IAAI,KAAK,GAAG,oBAAoB,EAAE,CAAC;YAC/E,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAA;YACjC,OAAO,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;QACtC,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC,CAAA;IAED,MAAM,SAAS,GAAG,CAChB,MAA0B,EAU1B,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,cAAc,GAClB,MAAM,KAAK,SAAS;YAClB,CAAC,CAAC,iCAAiC,MAAM,EAAE;YAC3C,CAAC,CAAC,iCAAiC,MAAM,mBAAmB,CAAA;QAEhE,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,MAAM,CAAoB,CAAC,EAAE,EAAE,EAAE;YACxD,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC3C,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAA;QAC5E,CAAC,CAAC,CAAA;QAEF,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAA;QAK9C,MAAM,YAAY,GAAU,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAA;QAEnE,MAAM,SAAS,GAAG,CAChB,KAAY,EACgF,EAAE,CAC9F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,MAAM,SAAS,GACb,KAAK,CAAC,MAAM,KAAK,SAAS;gBACxB,CAAC,CAAC,iBAAiB,MAAM,8BAA8B;gBACvD,CAAC,CAAC,iBAAiB,MAAM,+CAA+C,CAAA;YAE5E,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;gBACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;gBACtC,OAAO,KAAK,CAAC,MAAM,KAAK,SAAS;oBAC/B,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE;oBAClC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAA;YACpD,CAAC,CAAC,CAAA;YAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO,MAAM,CAAC,IAAI,EAAE,CAAA;YACtB,CAAC;YAED,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;YAE5E,IAAI,WAAW,GAAG,wBAAwB,IAAI,KAAK,CAAC,KAAK,GAAG,gBAAgB,EAAE,CAAC;gBAC7E,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAE5C,IAAI,SAAS,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC9B,OAAO,KAAK,CAAC,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;gBACrE,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,KAAK,CAAC,YAAY,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAA;YAErE,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC9E,YAAY;gBACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;aACxD,CAAC,CAAC,CAAA;YAEH,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,MAAM,CAAA;YACvD,MAAM,SAAS,GAAU,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,gBAAgB,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,CAAA;YAElG,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,SAAS,CAAU,CAAC,CAAA;QACvD,CAAC,CAAC,CAAA;QAEJ,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC,YAAY,EAAE,SAAS,CAAC,CAAA;QAEhE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;IAC1B,CAAC,CAAC,CAAC,IAAI,CACL,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,6CAA6C,EAAE,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CACnG,CAAA;IAEH,MAAM,YAAY,GAAgC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CACrE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,sCAAsC;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE9B,gBAAgB;QAChB,gFAAgF;QAChF,6EAA6E;QAC7E,MAAM,UAAU,GAAG,EAAE,CAAA;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;YAE5C,wFAAwF;YACxF,MAAM,kBAAkB,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC9E,MAAM,GAAG,GAAG,eAAe,MAAM,8EAA8E,kBAAkB,EAAE,CAAA;YACnI,wDAAwD;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;gBACtC,KAAK,CAAC,MAAM;gBACZ,KAAK,CAAC,YAAY;gBAClB,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;gBAC5D,KAAK,CAAC,IAAI;gBACV,SAAS;gBACT,KAAK,CAAC,QAAQ;gBACd,KAAK,CAAC,SAAS;aAChB,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACnB,EAAE;iBACC,OAAO,CAAC,GAAG,CAAC;iBACZ,IAAI,CAAC,GAAG,MAAM,CAAC;iBACf,GAAG,EAAE,CACT,CAAA;QACH,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,gDAAgD,EAAE;QAChE,UAAU,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE;KAClD,CAAC,CACH,CAAA;IAEH,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CACnE,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAChE,CAAA;IAED,OAAO;QACL,MAAM;QACN,WAAW;QACX,SAAS;QACT,YAAY;QACZ,UAAU;KACX,CAAA;AACH,CAAC,CAAA;AAED,MAAM,gBAAgB,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,eAAe,EAAE,GAAG,CAAC,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livestore/sync-cf",
-  "version": "0.4.0-dev.7",
+  "version": "0.4.0-dev.8",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -10,9 +10,9 @@
   },
   "dependencies": {
     "@cloudflare/workers-types": "4.20250823.0",
-    "@livestore/common": "0.4.0-dev.7",
-    "@livestore/common-cf": "0.4.0-dev.7",
-    "@livestore/utils": "0.4.0-dev.7"
+    "@livestore/common": "0.4.0-dev.8",
+    "@livestore/utils": "0.4.0-dev.8",
+    "@livestore/common-cf": "0.4.0-dev.8"
   },
   "files": [
     "dist",

package/src/cf-worker/do/pull.ts

@@ -1,5 +1,5 @@
 import { BackendIdMismatchError, InvalidPullError, SyncBackend, UnexpectedError } from '@livestore/common'
-import { Effect, Option, pipe, ReadonlyArray, type Schema, Stream } from '@livestore/utils/effect'
+import { Chunk, Effect, Option, type Schema, Stream } from '@livestore/utils/effect'
 import { SyncMessage } from '../../common/mod.ts'
 import { PULL_CHUNK_SIZE } from '../shared.ts'
 import { DoCtx } from './layer.ts'
@@ -30,24 +30,25 @@ export const makeEndingPullStream = (
       return yield* new BackendIdMismatchError({ expected: backendId, received: req.cursor.value.backendId })
     }
 
-    // TODO use streaming for db results
-    const remainingEvents = yield* storage.getEvents(Option.getOrUndefined(req.cursor)?.eventSequenceNumber)
+    const { stream: storedEvents, total } = yield* storage.getEvents(
+      Option.getOrUndefined(req.cursor)?.eventSequenceNumber,
+    )
 
-    const batches = pipe(
-      remainingEvents,
-      ReadonlyArray.chunksOf(PULL_CHUNK_SIZE),
-      ReadonlyArray.map((batch, i) => {
-        const remaining = Math.max(0, remainingEvents.length - (i + 1) * PULL_CHUNK_SIZE)
+    return storedEvents.pipe(
+      Stream.grouped(PULL_CHUNK_SIZE),
+      Stream.mapAccum(total, (remaining, chunk) => {
+        const asArray = Chunk.toReadonlyArray(chunk)
+        const nextRemaining = Math.max(0, remaining - asArray.length)
 
-        return SyncMessage.PullResponse.make({
-          batch,
-          pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
-          backendId,
-        })
+        return [
+          nextRemaining,
+          SyncMessage.PullResponse.make({
+            batch: asArray,
+            pageInfo: nextRemaining > 0 ? SyncBackend.pageInfoMoreKnown(nextRemaining) : SyncBackend.pageInfoNoMore,
+            backendId,
+          }),
+        ] as const
       }),
-    )
-
-    return Stream.fromIterable(batches).pipe(
       Stream.tap(
         Effect.fn(function* (res) {
           if (doOptions?.onPullRes) {

package/src/cf-worker/do/push.ts

@@ -144,6 +144,7 @@ export const makePush =
         }),
       ),
       Effect.mapError((cause) => InvalidPushError.make({ cause })),
+      Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }),
     )
 
 /**

package/src/cf-worker/do/sync-storage.ts

@@ -1,18 +1,21 @@
 import { UnexpectedError } from '@livestore/common'
 import type { LiveStoreEvent } from '@livestore/common/schema'
 import type { CfTypes } from '@livestore/common-cf'
-import { Effect, Option, Schema } from '@livestore/utils/effect'
+import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 import { SyncMetadata } from '../../common/sync-message-types.ts'
 import { type Env, PERSISTENCE_FORMAT_VERSION, type StoreId } from '../shared.ts'
 import { eventlogTable } from './sqlite.ts'
 
 export type SyncStorage = {
   dbName: string
-  // getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError>
-  getEvents: (
-    cursor: number | undefined,
-  ) => Effect.Effect<
-    ReadonlyArray<{ eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
+  getEvents: (cursor: number | undefined) => Effect.Effect<
+    {
+      total: number
+      stream: Stream.Stream<
+        { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> },
+        UnexpectedError
+      >
+    },
     UnexpectedError
   >
   appendEvents: (
@@ -34,35 +37,111 @@ export const makeStorage = (ctx: CfTypes.DurableObjectState, env: Env, storeId:
       Effect.withSpan('@livestore/sync-cf:durable-object:execDb'),
     )
 
-  // const getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError> = Effect.gen(
-  //   function* () {
-  //     const result = yield* execDb<{ seqNum: EventSequenceNumber.GlobalEventSequenceNumber }>((db) =>
-  //       db.prepare(`SELECT seqNum FROM ${dbName} ORDER BY seqNum DESC LIMIT 1`).all(),
-  //     )
+  // Cloudflare's D1 HTTP endpoint rejects JSON responses once they exceed ~1MB.
+  // Keep individual SELECT batches comfortably below that threshold so we can
+  // serve large histories without tripping the limit.
+  const D1_MAX_JSON_RESPONSE_BYTES = 1_000_000
+  const D1_RESPONSE_SAFETY_MARGIN_BYTES = 64 * 1024
+  const D1_TARGET_RESPONSE_BYTES = D1_MAX_JSON_RESPONSE_BYTES - D1_RESPONSE_SAFETY_MARGIN_BYTES
+  const D1_INITIAL_PAGE_SIZE = 256
+  const D1_MIN_PAGE_SIZE = 1
+
+  const decodeEventlogRows = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))
+  const textEncoder = new TextEncoder()
+
+  const decreaseLimit = (limit: number) => Math.max(D1_MIN_PAGE_SIZE, Math.floor(limit / 2))
+  const increaseLimit = (limit: number) => Math.min(D1_INITIAL_PAGE_SIZE, limit * 2)
 
-  //     return result[0]?.seqNum ?? EventSequenceNumber.ROOT.global
-  //   },
-  // ).pipe(UnexpectedError.mapToUnexpectedError)
+  const computeNextLimit = (limit: number, encodedSize: number) => {
+    if (encodedSize > D1_TARGET_RESPONSE_BYTES && limit > D1_MIN_PAGE_SIZE) {
+      const next = decreaseLimit(limit)
+      return next === limit ? limit : next
+    }
+
+    if (encodedSize < D1_TARGET_RESPONSE_BYTES / 2 && limit < D1_INITIAL_PAGE_SIZE) {
+      const next = increaseLimit(limit)
+      return next === limit ? limit : next
+    }
+
+    return limit
+  }
 
-  // TODO support streaming
   const getEvents = (
     cursor: number | undefined,
   ): Effect.Effect<
-    ReadonlyArray<{ eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
+    {
+      total: number
+      stream: Stream.Stream<
+        { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> },
+        UnexpectedError
+      >
+    },
     UnexpectedError
   > =>
     Effect.gen(function* () {
-      const whereClause = cursor === undefined ? '' : `WHERE seqNum > ${cursor}`
-      const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY seqNum ASC`
-      // TODO handle case where `cursor` was not found
-      const rawEvents = yield* execDb((db) => db.prepare(sql).all())
-      const events = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))(rawEvents).map(
-        ({ createdAt, ...eventEncoded }) => ({
-          eventEncoded,
-          metadata: Option.some(SyncMetadata.make({ createdAt })),
-        }),
-      )
-      return events
+      const countStatement =
+        cursor === undefined
+          ? `SELECT COUNT(*) as total FROM ${dbName}`
+          : `SELECT COUNT(*) as total FROM ${dbName} WHERE seqNum > ?`
+
+      const countRows = yield* execDb<{ total: number }>((db) => {
+        const prepared = db.prepare(countStatement)
+        return cursor === undefined ? prepared.all() : prepared.bind(cursor).all()
+      })
+
+      const total = Number(countRows[0]?.total ?? 0)
+
+      type State = { cursor: number | undefined; limit: number }
+      type EmittedEvent = { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }
+
+      const initialState: State = { cursor, limit: D1_INITIAL_PAGE_SIZE }
+
+      const fetchPage = (
+        state: State,
+      ): Effect.Effect<Option.Option<readonly [Chunk.Chunk<EmittedEvent>, State]>, UnexpectedError> =>
+        Effect.gen(function* () {
+          const statement =
+            state.cursor === undefined
+              ? `SELECT * FROM ${dbName} ORDER BY seqNum ASC LIMIT ?`
+              : `SELECT * FROM ${dbName} WHERE seqNum > ? ORDER BY seqNum ASC LIMIT ?`
+
+          const rawEvents = yield* execDb((db) => {
+            const prepared = db.prepare(statement)
+            return state.cursor === undefined
+              ? prepared.bind(state.limit).all()
+              : prepared.bind(state.cursor, state.limit).all()
+          })
+
+          if (rawEvents.length === 0) {
+            return Option.none()
+          }
+
+          const encodedSize = textEncoder.encode(JSON.stringify(rawEvents)).byteLength
+
+          if (encodedSize > D1_TARGET_RESPONSE_BYTES && state.limit > D1_MIN_PAGE_SIZE) {
+            const nextLimit = decreaseLimit(state.limit)
+
+            if (nextLimit !== state.limit) {
+              return yield* fetchPage({ cursor: state.cursor, limit: nextLimit })
+            }
+          }
+
+          const decodedRows = Chunk.fromIterable(decodeEventlogRows(rawEvents))
+
+          const eventsChunk = Chunk.map(decodedRows, ({ createdAt, ...eventEncoded }) => ({
+            eventEncoded,
+            metadata: Option.some(SyncMetadata.make({ createdAt })),
+          }))
+
+          const lastSeqNum = Chunk.unsafeLast(decodedRows).seqNum
+          const nextState: State = { cursor: lastSeqNum, limit: computeNextLimit(state.limit, encodedSize) }
+
+          return Option.some([eventsChunk, nextState] as const)
+        })
+
+      const stream = Stream.unfoldChunkEffect(initialState, fetchPage)
+
+      return { total, stream }
     }).pipe(
       UnexpectedError.mapToUnexpectedError,
       Effect.withSpan('@livestore/sync-cf:durable-object:getEvents', { attributes: { dbName, cursor } }),

package/dist/cf-worker/cf-types.d.ts

@@ -1,2 +0,0 @@
-export { type D1Database, type D1Result, type DurableObject, type DurableObjectState, Request, Response, Rpc, WebSocket, WebSocketPair, WebSocketRequestResponsePair, } from '@cloudflare/workers-types';
-//# sourceMappingURL=cf-types.d.ts.map

package/dist/cf-worker/cf-types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"cf-types.d.ts","sourceRoot":"","sources":["../../src/cf-worker/cf-types.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,UAAU,EACf,KAAK,QAAQ,EACb,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,OAAO,EACP,QAAQ,EACR,GAAG,EACH,SAAS,EACT,aAAa,EACb,4BAA4B,GAC7B,MAAM,2BAA2B,CAAA"}

package/dist/cf-worker/cf-types.js

@@ -1,2 +0,0 @@
-export { Request, Response, Rpc, WebSocket, WebSocketPair, WebSocketRequestResponsePair, } from '@cloudflare/workers-types';
-//# sourceMappingURL=cf-types.js.map

package/dist/cf-worker/cf-types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cf-types.js","sourceRoot":"","sources":["../../src/cf-worker/cf-types.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,OAAO,EACP,QAAQ,EACR,GAAG,EACH,SAAS,EACT,aAAa,EACb,4BAA4B,GAC7B,MAAM,2BAA2B,CAAA"}