@livestore/sync-cf 0.4.0-dev.7 → 0.4.0-dev.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/cf-worker/do/durable-object.d.ts.map +1 -1
  3. package/dist/cf-worker/do/durable-object.js +5 -9
  4. package/dist/cf-worker/do/durable-object.js.map +1 -1
  5. package/dist/cf-worker/do/layer.d.ts +1 -1
  6. package/dist/cf-worker/do/pull.d.ts +1 -1
  7. package/dist/cf-worker/do/pull.d.ts.map +1 -1
  8. package/dist/cf-worker/do/pull.js +21 -13
  9. package/dist/cf-worker/do/pull.js.map +1 -1
  10. package/dist/cf-worker/do/push.d.ts.map +1 -1
  11. package/dist/cf-worker/do/push.js +66 -35
  12. package/dist/cf-worker/do/push.js.map +1 -1
  13. package/dist/cf-worker/do/sync-storage.d.ts +8 -5
  14. package/dist/cf-worker/do/sync-storage.d.ts.map +1 -1
  15. package/dist/cf-worker/do/sync-storage.js +64 -19
  16. package/dist/cf-worker/do/sync-storage.js.map +1 -1
  17. package/dist/cf-worker/do/transport/do-rpc-server.d.ts +2 -1
  18. package/dist/cf-worker/do/transport/do-rpc-server.d.ts.map +1 -1
  19. package/dist/cf-worker/do/transport/do-rpc-server.js.map +1 -1
  20. package/dist/cf-worker/do/transport/http-rpc-server.d.ts +1 -1
  21. package/dist/cf-worker/do/ws-chunking.d.ts +22 -0
  22. package/dist/cf-worker/do/ws-chunking.d.ts.map +1 -0
  23. package/dist/cf-worker/do/ws-chunking.js +49 -0
  24. package/dist/cf-worker/do/ws-chunking.js.map +1 -0
  25. package/dist/cf-worker/shared.d.ts +19 -13
  26. package/dist/cf-worker/shared.d.ts.map +1 -1
  27. package/dist/cf-worker/shared.js +15 -4
  28. package/dist/cf-worker/shared.js.map +1 -1
  29. package/dist/cf-worker/worker.d.ts +30 -45
  30. package/dist/cf-worker/worker.d.ts.map +1 -1
  31. package/dist/cf-worker/worker.js +30 -25
  32. package/dist/cf-worker/worker.js.map +1 -1
  33. package/dist/common/sync-message-types.d.ts +5 -5
  34. package/package.json +5 -5
  35. package/src/cf-worker/do/durable-object.ts +6 -10
  36. package/src/cf-worker/do/pull.ts +30 -17
  37. package/src/cf-worker/do/push.ts +85 -38
  38. package/src/cf-worker/do/sync-storage.ts +106 -27
  39. package/src/cf-worker/do/transport/do-rpc-server.ts +4 -2
  40. package/src/cf-worker/do/ws-chunking.ts +76 -0
  41. package/src/cf-worker/shared.ts +19 -6
  42. package/src/cf-worker/worker.ts +46 -69
  43. package/dist/cf-worker/cf-types.d.ts +0 -2
  44. package/dist/cf-worker/cf-types.d.ts.map +0 -1
  45. package/dist/cf-worker/cf-types.js +0 -2
  46. package/dist/cf-worker/cf-types.js.map +0 -1
  47. package/dist/cf-worker/durable-object.d.ts +0 -189
  48. package/dist/cf-worker/durable-object.d.ts.map +0 -1
  49. package/dist/cf-worker/durable-object.js +0 -317
  50. package/dist/cf-worker/durable-object.js.map +0 -1
  51. package/dist/common/ws-message-types.d.ts +0 -270
  52. package/dist/common/ws-message-types.d.ts.map +0 -1
  53. package/dist/common/ws-message-types.js +0 -57
  54. package/dist/common/ws-message-types.js.map +0 -1
  55. package/dist/sync-impl/mod.d.ts +0 -2
  56. package/dist/sync-impl/mod.d.ts.map +0 -1
  57. package/dist/sync-impl/mod.js +0 -2
  58. package/dist/sync-impl/mod.js.map +0 -1
  59. package/dist/sync-impl/ws-impl.d.ts +0 -7
  60. package/dist/sync-impl/ws-impl.d.ts.map +0 -1
  61. package/dist/sync-impl/ws-impl.js +0 -175
  62. package/dist/sync-impl/ws-impl.js.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"durable-object.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/durable-object.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,OAAO,EAAkC,MAAM,sBAAsB,CAAA;AAanF,OAAO,EACL,KAAK,GAAG,EAER,KAAK,6BAA6B,EAClC,KAAK,uBAAuB,EAE7B,MAAM,cAAc,CAAA;AAmBrB,MAAM,MAAM,OAAO,GAAG,OAAO,CAAC,kBAAkB,CAAA;AAChD,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI,OAAO,CAAC,aAAa,GAAG,CAAC,CAAA;AAEnD,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,CAAC,EAAE,6BAA6B,KAAK;IAChF,KAAK,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,QAAQ,CAAC,uBAAuB,CAAC,CAAA;CAChE,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,eAAO,MAAM,iBAAiB,EAAE,sBA4J/B,CAAA"}
1
+ {"version":3,"file":"durable-object.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/durable-object.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,OAAO,EAAkC,MAAM,sBAAsB,CAAA;AAanF,OAAO,EACL,KAAK,GAAG,EACR,KAAK,6BAA6B,EAElC,KAAK,uBAAuB,EAE7B,MAAM,cAAc,CAAA;AAmBrB,MAAM,MAAM,OAAO,GAAG,OAAO,CAAC,kBAAkB,CAAA;AAChD,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI,OAAO,CAAC,aAAa,GAAG,CAAC,CAAA;AAEnD,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,CAAC,EAAE,6BAA6B,KAAK;IAChF,KAAK,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,QAAQ,CAAC,uBAAuB,CAAC,CAAA;CAChE,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,eAAO,MAAM,iBAAiB,EAAE,sBAwJ/B,CAAA"}
@@ -3,12 +3,12 @@ import { DurableObject } from 'cloudflare:workers';
3
3
  import { setupDurableObjectWebSocketRpc } from '@livestore/common-cf';
4
4
  import { CfDeclare } from '@livestore/common-cf/declare';
5
5
  import { Effect, FetchHttpClient, Layer, Logger, LogLevel, Otlp, RpcMessage, Schema, } from '@livestore/utils/effect';
6
- import { getSyncRequestSearchParams, WebSocketAttachmentSchema, } from "../shared.js";
6
+ import { matchSyncRequest, WebSocketAttachmentSchema, } from "../shared.js";
7
7
  import { DoCtx } from "./layer.js";
8
8
  import { createDoRpcHandler } from "./transport/do-rpc-server.js";
9
9
  import { createHttpRpcHandler } from "./transport/http-rpc-server.js";
10
10
  import { makeRpcServer } from "./transport/ws-rpc-server.js";
11
- const DurableObjectBase = DurableObject;
11
+ const DurableObjectBase = (DurableObject);
12
12
  /**
13
13
  * Creates a Durable Object class for handling WebSocket-based sync.
14
14
  * A sync durable object is uniquely scoped to a specific `storeId`.
@@ -59,12 +59,8 @@ export const makeDurableObject = (options) => {
59
59
  : Layer.empty;
60
60
  return class SyncBackendDOBase extends DurableObjectBase {
61
61
  __DURABLE_OBJECT_BRAND = 'SyncBackendDOBase';
62
- ctx;
63
- env;
64
62
  constructor(ctx, env) {
65
63
  super(ctx, env);
66
- this.ctx = ctx;
67
- this.env = env;
68
64
  const WebSocketRpcServerLive = makeRpcServer({ doSelf: this, doOptions: options });
69
65
  // This registers the `webSocketMessage` and `webSocketClose` handlers
70
66
  if (enabledTransports.has('ws')) {
@@ -100,11 +96,11 @@ export const makeDurableObject = (options) => {
100
96
  }
101
97
  }
102
98
  fetch = async (request) => Effect.gen(this, function* () {
103
- const requestParamsResult = getSyncRequestSearchParams(request);
104
- if (requestParamsResult._tag === 'None') {
99
+ const searchParams = matchSyncRequest(request);
100
+ if (searchParams === undefined) {
105
101
  throw new Error('No search params found in request URL');
106
102
  }
107
- const { storeId, payload, transport } = requestParamsResult.value;
103
+ const { storeId, payload, transport } = searchParams;
108
104
  if (enabledTransports.has(transport) === false) {
109
105
  throw new Error(`Transport ${transport} is not enabled (based on \`options.enabledTransports\`)`);
110
106
  }
@@ -1 +1 @@
1
- {"version":3,"file":"durable-object.js","sourceRoot":"","sources":["../../../src/cf-worker/do/durable-object.ts"],"names":[],"mappings":"AAAA,mDAAmD;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAgB,8BAA8B,EAAE,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AACxD,OAAO,EACL,MAAM,EACN,eAAe,EACf,KAAK,EACL,MAAM,EACN,QAAQ,EACR,IAAI,EACJ,UAAU,EACV,MAAM,GAEP,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAEL,0BAA0B,EAG1B,yBAAyB,GAC1B,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAClC,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAS5D,MAAM,iBAAiB,GAAG,aAGA,CAAA;AAU1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA2B,CAAC,OAAO,EAAE,EAAE;IACnE,MAAM,iBAAiB,GAAG,OAAO,EAAE,iBAAiB,IAAI,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAA;IAEzF,MAAM,OAAO,GAAG,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAElD,MAAM,aAAa,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO;QAC1C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACT,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO;YAC7B,oBAAoB,EAAE,EAAE;YACxB,QAAQ,EAAE;gBACR,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,IAAI,YAAY;aACtD;SACF,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC/C,CAAC,CAAC,KAAK,CAAC,KAAK,CAAA;IAEf,OAAO,MAAM,iBAAkB,SAAQ,iBAAiB;QACtD,sBAAsB,GAAG,mBAA4B,CAAA;QACrD,GAAG,CAA4B;QAC/B,GAAG,CAAK;QAER,YAAY,GAA+B,EAAE,GAAQ;YACnD,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;YACf,IAAI,CAAC,GAAG,GAAG,GAAG,CAAA;YACd,IAAI,CAAC,GAAG,GAAG,GAAG,CAAA;YAEd,MAAM,sBAAsB,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YAElF,sEAAsE;YACtE,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChC,8BAA8B,CAAC;oBAC7B,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,sBAAsB;oBAChC,aAAa,EAAE,WAAW;oBAC1B,oFAAoF;oBACpF,qCAAqC;oBACrC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE;wBACzB,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK,gBAAgB,EAAE,CAAC;4BACnE,mDAAmD;4BACnD,MAAM,UAAU,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAA;4BAC7C,MAAM,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAAA;4BAC5F,EAAE,CAAC,mBAAmB,CACpB,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC;gCAC3C,GAAG,IAAI;gCACP,cAAc,EAAE,CAAC,GAAG,cAAc,EAAE,OAAO,CAAC,EAAE,CAAC;6BAChD,CAAC,CACH,CAAA;wBACH,CAAC;6BAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;4BACxC,6DAA6D;4BAC7D,MAAM,UAAU,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAA;4BAC7C,MAAM,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAAA;4BAC5F,EAAE,CAAC,mBAAmB,CACpB,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC;gCAC3C,GAAG,IAAI;gCACP,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,SAAS,CAAC;6BACxE,CAAC,CACH,CAAA;4BACD,2CAA2C;wBAC7C,CAAC;oBACH,CAAC;oBACD,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QAED,KAAK,GAAG,KAAK,EAAE,OAAgB,EAAqB,EAAE,CACpD,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;YACxB,MAAM,mBAAmB,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAA;YAC/D,IAAI,mBAAmB,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAA;YAC1D,CAAC;YAED,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,mBAAmB,CAAC,KAAK,CAAA;YAEjE,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,KAAK,EAAE,CAAC;gBAC/C,MAAM,IAAI,KAAK,CAAC,aAAa,SAAS,0DAA0D,CAAC,CAAA;YACnG,CAAC;YAED,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;YACxC,CAAC;YAED,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gBACvB,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,aAAa,EAAE,CAAA;gBAEpD,mHAAmH;gBACnH,MAAM,CAAC,mBAAmB,CACxB,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CACvF,CAAA;gBAED,8FAA8F;gBAE9F,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;gBAEhC,kDAAkD;gBAClD,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAC/B,IAAI,4BAA4B,CAC9B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EACpC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,CACrC,CACF,CAAA;gBAED,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;oBACxB,MAAM,EAAE,GAAG;oBACX,SAAS,EAAE,MAAM;iBAClB,CAAC,CAAA;YACJ,CAAC;YAED,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,CAAA;YAE1C,OAAO,IAAI,QAAQ,CAAC,cAAc,EAAE;gBAClC,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;QACJ,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,iBAAiB,EAAE,kDAAkD;QAC5E,MAAM,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE,CAC7B,MAAM,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CACrF,EACD,MAAM,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAC1D,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,EAClF,IAAI,CAAC,kBAAkB,CACxB,CAAA;QAEH;;WAEG;QACH,KAAK,CAAC,GAAG,CAAC,OAAgC;YACxC,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAE,CAAC;gBAC9C,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAA;YAC3F,CAAC;YAED,OAAO,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,CACtF,MAAM,CAAC,QAAQ,CAAC,uCAAuC,CAAC,EACxD,IAAI,CAAC,kBAAkB,CACxB,CAAA;QACH,CAAC;QAED;;;;WAIG;QACK,UAAU,GAAG,CAAC,OAAwB,EAAE,EAAE,CAChD,oBAAoB,CAAC;YACnB,OAAO;SACR,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAAC,CAAA;QAElE,kBAAkB,GAAG,CAAe,MAAwC,EAAc,EAAE,CAClG,MAAM,CAAC,IAAI,CACT,MAAM,CAAC,iBAAiB,EACxB,MAAM,CAAC,mBAAmB,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC1C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,EACtD,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,CAClB,CAAA;KACJ,CAAA;AACH,CAAC,CAAA"}
1
+ {"version":3,"file":"durable-object.js","sourceRoot":"","sources":["../../../src/cf-worker/do/durable-object.ts"],"names":[],"mappings":"AAAA,mDAAmD;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAgB,8BAA8B,EAAE,MAAM,sBAAsB,CAAA;AACnF,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AACxD,OAAO,EACL,MAAM,EACN,eAAe,EACf,KAAK,EACL,MAAM,EACN,QAAQ,EACR,IAAI,EACJ,UAAU,EACV,MAAM,GAEP,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAGL,gBAAgB,EAEhB,yBAAyB,GAC1B,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAClC,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAA;AAS5D,MAAM,iBAAiB,GAAG,CAAA,aAGgD,CAAA,CAAA;AAU1E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA2B,CAAC,OAAO,EAAE,EAAE;IACnE,MAAM,iBAAiB,GAAG,OAAO,EAAE,iBAAiB,IAAI,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAA;IAEzF,MAAM,OAAO,GAAG,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAElD,MAAM,aAAa,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO;QAC1C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACT,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO;YAC7B,oBAAoB,EAAE,EAAE;YACxB,QAAQ,EAAE;gBACR,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,WAAW,IAAI,YAAY;aACtD;SACF,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC/C,CAAC,CAAC,KAAK,CAAC,KAAK,CAAA;IAEf,OAAO,MAAM,iBAAkB,SAAQ,iBAAiB;QACtD,sBAAsB,GAAG,mBAA4B,CAAA;QAErD,YAAY,GAA+B,EAAE,GAAQ;YACnD,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;YAEf,MAAM,sBAAsB,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAA;YAElF,sEAAsE;YACtE,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChC,8BAA8B,CAAC;oBAC7B,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,sBAAsB;oBAChC,aAAa,EAAE,WAAW;oBAC1B,oFAAoF;oBACpF,qCAAqC;oBACrC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,EAAE,EAAE;wBACzB,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK,gBAAgB,EAAE,CAAC;4BACnE,mDAAmD;4BACnD,MAAM,UAAU,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAA;4BAC7C,MAAM,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAAA;4BAC5F,EAAE,CAAC,mBAAmB,CACpB,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC;gCAC3C,GAAG,IAAI;gCACP,cAAc,EAAE,CAAC,GAAG,cAAc,EAAE,OAAO,CAAC,EAAE,CAAC;6BAChD,CAAC,CACH,CAAA;wBACH,CAAC;6BAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;4BACxC,6DAA6D;4BAC7D,MAAM,UAAU,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAA;4BAC7C,MAAM,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAAA;4BAC5F,EAAE,CAAC,mBAAmB,CACpB,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC;gCAC3C,GAAG,IAAI;gCACP,cAAc,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,OAAO,CAAC,SAAS,CAAC;6BACxE,CAAC,CACH,CAAA;4BACD,2CAA2C;wBAC7C,CAAC;oBACH,CAAC;oBACD,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QAED,KAAK,GAAG,KAAK,EAAE,OAAgB,EAAqB,EAAE,CACpD,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;YACxB,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;YAC9C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAA;YAC1D,CAAC;YAED,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,YAAY,CAAA;YAEpD,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,KAAK,EAAE,CAAC;gBAC/C,MAAM,IAAI,KAAK,CAAC,aAAa,SAAS,0DAA0D,CAAC,CAAA;YACnG,CAAC;YAED,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;YACxC,CAAC;YAED,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gBACvB,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,aAAa,EAAE,CAAA;gBAEpD,mHAAmH;gBACnH,MAAM,CAAC,mBAAmB,CACxB,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CACvF,CAAA;gBAED,8FAA8F;gBAE9F,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;gBAEhC,kDAAkD;gBAClD,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAC/B,IAAI,4BAA4B,CAC9B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EACpC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,CACrC,CACF,CAAA;gBAED,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;oBACxB,MAAM,EAAE,GAAG;oBACX,SAAS,EAAE,MAAM;iBAClB,CAAC,CAAA;YACJ,CAAC;YAED,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,CAAA;YAE1C,OAAO,IAAI,QAAQ,CAAC,cAAc,EAAE;gBAClC,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAA;QACJ,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,iBAAiB,EAAE,kDAAkD;QAC5E,MAAM,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE,CAC7B,MAAM,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CACrF,EACD,MAAM,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAC1D,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,EAClF,IAAI,CAAC,kBAAkB,CACxB,CAAA;QAEH;;WAEG;QACH,KAAK,CAAC,GAAG,CAAC,OAAgC;YACxC,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAE,CAAC;gBAC9C,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAA;YAC3F,CAAC;YAED,OAAO,kBAAkB,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,CACtF,MAAM,CAAC,QAAQ,CAAC,uCAAuC,CAAC,EACxD,IAAI,CAAC,kBAAkB,CACxB,CAAA;QACH,CAAC;QAED;;;;WAIG;QACK,UAAU,GAAG,CAAC,OAAwB,EAAE,EAAE,CAChD,oBAAoB,CAAC;YACnB,OAAO;SACR,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAAC,CAAA;QAElE,kBAAkB,GAAG,CAAe,MAAwC,EAAc,EAAE,CAClG,MAAM,CAAC,IAAI,CACT,MAAM,CAAC,iBAAiB,EACxB,MAAM,CAAC,mBAAmB,CAAC,QAAQ,CAAC,KAAK,CAAC,EAC1C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,EACtD,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,CAClB,CAAA;KACJ,CAAA;AACH,CAAC,CAAA"}
@@ -24,7 +24,7 @@ declare const DoCtx_base: Effect.Service.Class<DoCtx, "DoCtx", {
24
24
  storage: import("./sync-storage.ts").SyncStorage;
25
25
  doOptions: MakeDurableObjectClassOptions | undefined;
26
26
  env: Env;
27
- ctx: CfTypes.DurableObjectState;
27
+ ctx: CfTypes.DurableObjectState<unknown>;
28
28
  rpcSubscriptions: Map<string, RpcSubscription>;
29
29
  }, UnexpectedError, never>>;
30
30
  }>;
@@ -1,5 +1,5 @@
1
1
  import { InvalidPullError } from '@livestore/common';
2
- import { type Schema, Stream } from '@livestore/utils/effect';
2
+ import { Schema, Stream } from '@livestore/utils/effect';
3
3
  import { SyncMessage } from '../../common/mod.ts';
4
4
  import { DoCtx } from './layer.ts';
5
5
  export declare const makeEndingPullStream: (req: SyncMessage.PullRequest, payload: Schema.JsonValue | undefined) => Stream.Stream<SyncMessage.PullResponse, InvalidPullError, DoCtx>;
@@ -1 +1 @@
1
- {"version":3,"file":"pull.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,gBAAgB,EAAgC,MAAM,mBAAmB,CAAA;AAC1G,OAAO,EAAuC,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAClG,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAWlC,eAAO,MAAM,oBAAoB,GAC/B,KAAK,WAAW,CAAC,WAAW,EAC5B,SAAS,MAAM,CAAC,SAAS,GAAG,SAAS,KACpC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,CA6C/D,CAAA"}
1
+ {"version":3,"file":"pull.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,gBAAgB,EAAgC,MAAM,mBAAmB,CAAA;AAC1G,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAC/E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAclC,eAAO,MAAM,oBAAoB,GAC/B,KAAK,WAAW,CAAC,WAAW,EAC5B,SAAS,MAAM,CAAC,SAAS,GAAG,SAAS,KACpC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,CAuD/D,CAAA"}
@@ -1,8 +1,10 @@
1
1
  import { BackendIdMismatchError, InvalidPullError, SyncBackend, UnexpectedError } from '@livestore/common';
2
- import { Effect, Option, pipe, ReadonlyArray, Stream } from '@livestore/utils/effect';
2
+ import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect';
3
3
  import { SyncMessage } from "../../common/mod.js";
4
- import { PULL_CHUNK_SIZE } from "../shared.js";
4
+ import { MAX_PULL_EVENTS_PER_MESSAGE, MAX_WS_MESSAGE_BYTES } from "../shared.js";
5
5
  import { DoCtx } from "./layer.js";
6
+ import { splitChunkBySize } from "./ws-chunking.js";
7
+ const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse);
6
8
  // Notes on stream handling:
7
9
  // We're intentionally closing the stream once we've read all existing events
8
10
  //
@@ -20,17 +22,23 @@ export const makeEndingPullStream = (req, payload) => Effect.gen(function* () {
20
22
  if (req.cursor._tag === 'Some' && req.cursor.value.backendId !== backendId) {
21
23
  return yield* new BackendIdMismatchError({ expected: backendId, received: req.cursor.value.backendId });
22
24
  }
23
- // TODO use streaming for db results
24
- const remainingEvents = yield* storage.getEvents(Option.getOrUndefined(req.cursor)?.eventSequenceNumber);
25
- const batches = pipe(remainingEvents, ReadonlyArray.chunksOf(PULL_CHUNK_SIZE), ReadonlyArray.map((batch, i) => {
26
- const remaining = Math.max(0, remainingEvents.length - (i + 1) * PULL_CHUNK_SIZE);
27
- return SyncMessage.PullResponse.make({
28
- batch,
29
- pageInfo: remaining > 0 ? SyncBackend.pageInfoMoreKnown(remaining) : SyncBackend.pageInfoNoMore,
30
- backendId,
31
- });
32
- }));
33
- return Stream.fromIterable(batches).pipe(Stream.tap(Effect.fn(function* (res) {
25
+ const { stream: storedEvents, total } = yield* storage.getEvents(Option.getOrUndefined(req.cursor)?.eventSequenceNumber);
26
+ return storedEvents.pipe(Stream.mapChunks(splitChunkBySize({
27
+ maxItems: MAX_PULL_EVENTS_PER_MESSAGE,
28
+ maxBytes: MAX_WS_MESSAGE_BYTES,
29
+ encode: (batch) => encodePullResponse(SyncMessage.PullResponse.make({ batch, pageInfo: SyncBackend.pageInfoNoMore, backendId })),
30
+ })), Stream.mapAccum(total, (remaining, chunk) => {
31
+ const asArray = Chunk.toReadonlyArray(chunk);
32
+ const nextRemaining = Math.max(0, remaining - asArray.length);
33
+ return [
34
+ nextRemaining,
35
+ SyncMessage.PullResponse.make({
36
+ batch: asArray,
37
+ pageInfo: nextRemaining > 0 ? SyncBackend.pageInfoMoreKnown(nextRemaining) : SyncBackend.pageInfoNoMore,
38
+ backendId,
39
+ }),
40
+ ];
41
+ }), Stream.tap(Effect.fn(function* (res) {
34
42
  if (doOptions?.onPullRes) {
35
43
  yield* Effect.tryAll(() => doOptions.onPullRes(res)).pipe(UnexpectedError.mapToUnexpectedError);
36
44
  }
@@ -1 +1 @@
1
- {"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/cf-worker/do/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAC1G,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAe,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAClG,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,4BAA4B;AAC5B,6EAA6E;AAC7E,EAAE;AACF,aAAa;AACb,yDAAyD;AACzD,6GAA6G;AAC7G,UAAU;AACV,yDAAyD;AACzD,wDAAwD;AACxD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,GAA4B,EAC5B,OAAqC,EAC6B,EAAE,CACpE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAE/D,IAAI,SAAS,EAAE,MAAM,EAAE,CAAC;QACtB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAU,CAAC,MAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAC5E,eAAe,CAAC,oBAAoB,CACrC,CAAA;IACH,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC3E,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA;IACzG,CAAC;IAED,oCAAoC;IACpC,MAAM,eAAe,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,mBAAmB,CAAC,CAAA;IAExG,MAAM,OAAO,GAAG,IAAI,CAClB,eAAe,EACf,aAAa,CAAC,QAAQ,CAAC,eAAe,CAAC,EACvC,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,eAAe,CAAC,CAAA;QAEjF,OAAO,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;YACnC,KAAK;YACL,QAAQ,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC/F,SAAS;SACV,CAAC,CAAA;IACJ,CAAC,CAAC,CACH,CAAA;IAED,OAAO,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,CACtC,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,GAAG;QACtB,IAAI,SAAS,EAAE,SAAS,EAAE,CAAC;YACzB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,SAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;QAClG,CAAC;IACH,CAAC,CAAC,CACH,EACD,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAC7D,CAAA;AACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,EAC5D,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAC5C,CAAA"}
1
+ {"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/cf-worker/do/pull.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAC1G,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAC/E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,2BAA2B,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAA;AAChF,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEnD,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;AAEtE,4BAA4B;AAC5B,6EAA6E;AAC7E,EAAE;AACF,aAAa;AACb,yDAAyD;AACzD,6GAA6G;AAC7G,UAAU;AACV,yDAAyD;AACzD,wDAAwD;AACxD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,GAA4B,EAC5B,OAAqC,EAC6B,EAAE,CACpE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAE/D,IAAI,SAAS,EAAE,MAAM,EAAE,CAAC;QACtB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAU,CAAC,MAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAC5E,eAAe,CAAC,oBAAoB,CACrC,CAAA;IACH,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC3E,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA;IACzG,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,CAC9D,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,mBAAmB,CACvD,CAAA;IAED,OAAO,YAAY,CAAC,IAAI,CACtB,MAAM,CAAC,SAAS,CACd,gBAAgB,CAAC;QACf,QAAQ,EAAE,2BAA2B;QACrC,QAAQ,EAAE,oBAAoB;QAC9B,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAChB,kBAAkB,CAChB,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,cAAc,EAAE,SAAS,EAAE,CAAC,CAC1F;KACJ,CAAC,CACH,EACD,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;QAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;QAE7D,OAAO;YACL,aAAa;YACb,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;gBAC5B,KAAK,EAAE,OAAO;gBACd,QAAQ,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;gBACvG,SAAS;aACV,CAAC;SACM,CAAA;IACZ,CAAC,CAAC,EACF,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,GAAG;QACtB,IAAI,SAAS,EAAE,SAAS,EAAE,CAAC;YACzB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,SAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;QAClG,CAAC;IACH,CAAC,CAAC,CACH,EACD,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAC7D,CAAA;AACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,EAC5D,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAC5C,CAAA"}
@@ -1 +1 @@
1
- {"version":3,"file":"push.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/push.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB,EAIjB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,KAAK,OAAO,EAAsB,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAE,MAAM,EAA2B,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,6BAA6B,EAAE,KAAK,OAAO,EAA6B,MAAM,cAAc,CAAA;AACpH,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,eAAO,MAAM,QAAQ,GAClB,0CAME;IACD,OAAO,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IACrC,OAAO,EAAE,6BAA6B,GAAG,SAAS,CAAA;IAClD,OAAO,EAAE,OAAO,CAAA;IAChB,GAAG,EAAE,OAAO,CAAC,kBAAkB,CAAA;IAC/B,GAAG,EAAE,GAAG,CAAA;CACT,MACA,aAAa,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,CAAC,+CAuHhD,CAAA"}
1
+ {"version":3,"file":"push.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/push.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB,EAIjB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,KAAK,OAAO,EAAsB,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAS,MAAM,EAA2B,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACxF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EACL,KAAK,GAAG,EAGR,KAAK,6BAA6B,EAClC,KAAK,OAAO,EAEb,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAMlC,eAAO,MAAM,QAAQ,GAClB,0CAME;IACD,OAAO,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IACrC,OAAO,EAAE,6BAA6B,GAAG,SAAS,CAAA;IAClD,OAAO,EAAE,OAAO,CAAA;IAChB,GAAG,EAAE,OAAO,CAAC,kBAAkB,CAAA;IAC/B,GAAG,EAAE,GAAG,CAAA;CACT,MACA,aAAa,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,CAAC,+CA2JhD,CAAA"}
@@ -1,9 +1,11 @@
1
1
  import { BackendIdMismatchError, InvalidPushError, ServerAheadError, SyncBackend, UnexpectedError, } from '@livestore/common';
2
2
  import { emitStreamResponse } from '@livestore/common-cf';
3
- import { Effect, Option, Schema } from '@livestore/utils/effect';
3
+ import { Chunk, Effect, Option, Schema } from '@livestore/utils/effect';
4
4
  import { SyncMessage } from "../../common/mod.js";
5
- import { WebSocketAttachmentSchema } from "../shared.js";
5
+ import { MAX_PULL_EVENTS_PER_MESSAGE, MAX_WS_MESSAGE_BYTES, WebSocketAttachmentSchema, } from "../shared.js";
6
6
  import { DoCtx } from "./layer.js";
7
+ import { splitChunkBySize } from "./ws-chunking.js";
8
+ const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse);
7
9
  export const makePush = ({ payload, options, storeId, ctx, env, }) => (pushRequest) => Effect.gen(function* () {
8
10
  // yield* Effect.log(`Pushing ${decodedMessage.batch.length} events`, decodedMessage.batch)
9
11
  const { backendId, storage, currentHeadRef, updateCurrentHead, rpcSubscriptions } = yield* DoCtx;
@@ -23,6 +25,12 @@ export const makePush = ({ payload, options, storeId, ctx, env, }) => (pushReque
23
25
  // Validate the batch
24
26
  const firstEventParent = pushRequest.batch[0].parentSeqNum;
25
27
  if (firstEventParent !== currentHead) {
28
+ // yield* Effect.logDebug('ServerAheadError: backend head mismatch', {
29
+ // expectedHead: currentHead,
30
+ // providedHead: firstEventParent,
31
+ // batchSize: pushRequest.batch.length,
32
+ // backendId,
33
+ // })
26
34
  return yield* new ServerAheadError({ minimumExpectedNum: currentHead, providedNum: firstEventParent });
27
35
  }
28
36
  const createdAt = new Date().toISOString();
@@ -34,48 +42,71 @@ export const makePush = ({ payload, options, storeId, ctx, env, }) => (pushReque
34
42
  // Run in background but already return the push ack to the client
35
43
  yield* Effect.gen(function* () {
36
44
  const connectedClients = ctx.getWebSockets();
37
- // Dual broadcasting: WebSocket + RPC clients
38
- const pullRes = SyncMessage.PullResponse.make({
39
- batch: pushRequest.batch.map((eventEncoded) => ({
45
+ // Preparing chunks of responses to make sure we don't exceed the WS message size limit.
46
+ const responses = Chunk.fromIterable(pushRequest.batch).pipe(splitChunkBySize({
47
+ maxItems: MAX_PULL_EVENTS_PER_MESSAGE,
48
+ maxBytes: MAX_WS_MESSAGE_BYTES,
49
+ encode: (items) => encodePullResponse(SyncMessage.PullResponse.make({
50
+ batch: items.map((eventEncoded) => ({
51
+ eventEncoded,
52
+ metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
53
+ })),
54
+ pageInfo: SyncBackend.pageInfoNoMore,
55
+ backendId,
56
+ })),
57
+ }), Chunk.map((eventsChunk) => {
58
+ const batchWithMetadata = Chunk.toReadonlyArray(eventsChunk).map((eventEncoded) => ({
40
59
  eventEncoded,
41
60
  metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
42
- })),
43
- pageInfo: SyncBackend.pageInfoNoMore,
44
- backendId,
45
- });
46
- const pullResEnc = Schema.encodeSync(SyncMessage.PullResponse)(pullRes);
61
+ }));
62
+ const response = SyncMessage.PullResponse.make({
63
+ batch: batchWithMetadata,
64
+ pageInfo: SyncBackend.pageInfoNoMore,
65
+ backendId,
66
+ });
67
+ return {
68
+ response,
69
+ encoded: Schema.encodeSync(SyncMessage.PullResponse)(response),
70
+ };
71
+ }));
72
+ // Dual broadcasting: WebSocket + RPC clients
47
73
  // Broadcast to WebSocket clients
48
74
  if (connectedClients.length > 0) {
49
- // Only calling once for now.
50
- if (options?.onPullRes) {
51
- yield* Effect.tryAll(() => options.onPullRes(pullRes)).pipe(UnexpectedError.mapToUnexpectedError);
52
- }
53
- // NOTE we're also sending the pullRes to the pushing ws client as a confirmation
54
- for (const conn of connectedClients) {
55
- // conn.send(pullResEnc)
56
- const attachment = Schema.decodeSync(WebSocketAttachmentSchema)(conn.deserializeAttachment());
57
- // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
58
- // response messsages on the Effect RPC-managed websocket connection to the WS client.
59
- // For this we need to get the RPC `requestId` from the WebSocket attachment.
60
- for (const requestId of attachment.pullRequestIds) {
61
- const res = {
62
- _tag: 'Chunk',
63
- requestId,
64
- values: [pullResEnc],
65
- };
66
- conn.send(JSON.stringify(res));
75
+ for (const { response, encoded } of responses) {
76
+ // Only calling once for now.
77
+ if (options?.onPullRes) {
78
+ yield* Effect.tryAll(() => options.onPullRes(response)).pipe(UnexpectedError.mapToUnexpectedError);
79
+ }
80
+ // NOTE we're also sending the pullRes chunk to the pushing ws client as confirmation
81
+ for (const conn of connectedClients) {
82
+ const attachment = Schema.decodeSync(WebSocketAttachmentSchema)(conn.deserializeAttachment());
83
+ // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
84
+ // response messsages on the Effect RPC-managed websocket connection to the WS client.
85
+ // For this we need to get the RPC `requestId` from the WebSocket attachment.
86
+ for (const requestId of attachment.pullRequestIds) {
87
+ const res = {
88
+ _tag: 'Chunk',
89
+ requestId,
90
+ values: [encoded],
91
+ };
92
+ conn.send(JSON.stringify(res));
93
+ }
67
94
  }
68
95
  }
69
96
  yield* Effect.logDebug(`Broadcasted to ${connectedClients.length} WebSocket clients`);
70
97
  }
71
98
  // RPC broadcasting would require reconstructing client stubs from clientIds
72
99
  if (rpcSubscriptions.size > 0) {
73
- yield* Effect.forEach(rpcSubscriptions.values(), (subscription) => emitStreamResponse({
74
- callerContext: subscription.callerContext,
75
- env,
76
- requestId: subscription.requestId,
77
- values: [pullResEnc],
78
- }).pipe(Effect.tapCauseLogPretty, Effect.exit), { concurrency: 'unbounded' });
100
+ for (const subscription of rpcSubscriptions.values()) {
101
+ for (const { encoded } of responses) {
102
+ yield* emitStreamResponse({
103
+ callerContext: subscription.callerContext,
104
+ env,
105
+ requestId: subscription.requestId,
106
+ values: [encoded],
107
+ }).pipe(Effect.tapCauseLogPretty, Effect.exit);
108
+ }
109
+ }
79
110
  yield* Effect.logDebug(`Broadcasted to ${rpcSubscriptions.size} RPC clients`);
80
111
  }
81
112
  }).pipe(Effect.tapCauseLogPretty, Effect.withSpan('push-rpc-broadcast'), Effect.uninterruptible, // We need to make sure Effect RPC doesn't interrupt this fiber
@@ -87,7 +118,7 @@ export const makePush = ({ payload, options, storeId, ctx, env, }) => (pushReque
87
118
  if (options?.onPushRes) {
88
119
  yield* Effect.tryAll(() => options.onPushRes(message)).pipe(UnexpectedError.mapToUnexpectedError);
89
120
  }
90
- })), Effect.mapError((cause) => InvalidPushError.make({ cause })));
121
+ })), Effect.mapError((cause) => InvalidPushError.make({ cause })), Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }));
91
122
  /**
92
123
  * @see https://developers.cloudflare.com/durable-objects/api/state/#blockconcurrencywhile
93
124
  */
@@ -1 +1 @@
1
- {"version":3,"file":"push.js","sourceRoot":"","sources":["../../../src/cf-worker/do/push.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,eAAe,GAChB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAgB,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAmB,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAA8D,yBAAyB,EAAE,MAAM,cAAc,CAAA;AACpH,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,MAAM,CAAC,MAAM,QAAQ,GACnB,CAAC,EACC,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,EACH,GAAG,GAOJ,EAAE,EAAE,CACL,CAAC,WAAkD,EAAE,EAAE,CACrD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,2FAA2F;IAC3F,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAEhG,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAO,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CACjF,eAAe,CAAC,oBAAoB,CACrC,CAAA;IACH,CAAC;IAED,IAAI,WAAW,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACvF,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAA;IAC1G,CAAC;IAED,2EAA2E;IAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC/C,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAA;QAC1C,uCAAuC;QACvC,qBAAqB;QACrB,MAAM,gBAAgB,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,YAAY,CAAA;QAC3D,IAAI,gBAAgB,KAAK,WAAW,EAAE,CAAC;YACrC,OAAO,KAAK,CAAC,CAAC,IAAI,gBAAgB,CAAC,EAAE,kBAAkB,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAA;QACxG,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAE1C,6EAA6E;QAC7E,KAAK,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAEzD,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAA;QAEnD,OAAO,EAAE,SAAS,EAAE,CAAA;IACtB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAA;IAEnC,kEAAkE;IAClE,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACzB,MAAM,gBAAgB,GAAG,GAAG,CAAC,aAAa,EAAE,CAAA;QAE5C,6CAA6C;QAC7C,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;YAC5C,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC9C,YAAY;gBACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;aACpE,CAAC,CAAC;YACH,QAAQ,EAAE,WAAW,CAAC,cAAc;YACpC,SAAS;SACV,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAA;QAEvE,iCAAiC;QACjC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,6BAA6B;YAC7B,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;gBACvB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;YACpG,CAAC;YAED,iFAAiF;YACjF,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;gBACpC,wBAAwB;gBACxB,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAA;gBAE7F,+FAA+F;gBAC/F,sFAAsF;gBACtF,6EAA6E;gBAC7E,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;oBAClD,MAAM,GAAG,GAAoC;wBAC3C,IAAI,EAAE,OAAO;wBACb,SAAS;wBACT,MAAM,EAAE,CAAC,UAAU,CAAC;qBACrB,CAAA;oBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;gBAChC,CAAC;YACH,CAAC;YAED,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,gBAAgB,CAAC,MAAM,oBAAoB,CAAC,CAAA;QACvF,CAAC;QAED,4EAA4E;QAC5E,IAAI,gBAAgB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnB,gBAAgB,CAAC,MAAM,EAAE,EACzB,CAAC,YAAY,EAAE,EAAE,CACf,kBAAkB,CAAC;gBACjB,aAAa,EAAE,YAAY,CAAC,aAAa;gBACzC,GAAG;gBACH,SAAS,EAAE,YAAY,CAAC,SAAS;gBACjC,MAAM,EAAE,CAAC,UAAU,CAAC;aACrB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,EAChD,EAAE,WAAW,EAAE,WAAW,EAAE,CAC7B,CAAA;YAED,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,gBAAgB,CAAC,IAAI,cAAc,CAAC,CAAA;QAC/E,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,iBAAiB,EACxB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EACrC,MAAM,CAAC,eAAe,EAAE,+DAA+D;IACvF,MAAM,CAAC,IAAI,CACZ,CAAA;IAED,8GAA8G;IAC9G,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAA;IAExB,OAAO,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACrC,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,OAAO;IAC1B,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;QACvB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;IACpG,CAAC;AACH,CAAC,CAAC,CACH,EACD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAC7D,CAAA;AAEL;;GAEG;AACH,MAAM,qBAAqB,GACzB,CAAC,GAA+B,EAAE,EAAE,CACpC,CAAU,GAA2B,EAAE,EAAE,CACvC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,EAAK,CAAA;IAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CACtC,GAAG,CAAC,qBAAqB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAC1F,CAAA;IAED,OAAO,KAAK,CAAC,CAAC,IAAI,CAAA;AACpB,CAAC,CAAC,CAAA"}
1
+ {"version":3,"file":"push.js","sourceRoot":"","sources":["../../../src/cf-worker/do/push.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,eAAe,GAChB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAgB,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAmB,MAAM,EAAE,MAAM,yBAAyB,CAAA;AACxF,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAEL,2BAA2B,EAC3B,oBAAoB,EAGpB,yBAAyB,GAC1B,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEnD,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAA;AAGtE,MAAM,CAAC,MAAM,QAAQ,GACnB,CAAC,EACC,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,EACH,GAAG,GAOJ,EAAE,EAAE,CACL,CAAC,WAAkD,EAAE,EAAE,CACrD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,2FAA2F;IAC3F,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAEhG,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAO,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CACjF,eAAe,CAAC,oBAAoB,CACrC,CAAA;IACH,CAAC;IAED,IAAI,WAAW,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,IAAI,WAAW,CAAC,SAAS,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACvF,OAAO,KAAK,CAAC,CAAC,IAAI,sBAAsB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAA;IAC1G,CAAC;IAED,2EAA2E;IAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAC/C,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAA;QAC1C,uCAAuC;QACvC,qBAAqB;QACrB,MAAM,gBAAgB,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,YAAY,CAAA;QAC3D,IAAI,gBAAgB,KAAK,WAAW,EAAE,CAAC;YACrC,sEAAsE;YACtE,+BAA+B;YAC/B,oCAAoC;YACpC,yCAAyC;YACzC,eAAe;YACf,KAAK;YAEL,OAAO,KAAK,CAAC,CAAC,IAAI,gBAAgB,CAAC,EAAE,kBAAkB,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,CAAA;QACxG,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAE1C,6EAA6E;QAC7E,KAAK,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAEzD,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAA;QAEnD,OAAO,EAAE,SAAS,EAAE,CAAA;IACtB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAA;IAEnC,kEAAkE;IAClE,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACzB,MAAM,gBAAgB,GAAG,GAAG,CAAC,aAAa,EAAE,CAAA;QAE5C,wFAAwF;QACxF,MAAM,SAAS,GAAG,KAAK,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CAC1D,gBAAgB,CAAC;YACf,QAAQ,EAAE,2BAA2B;YACrC,QAAQ,EAAE,oBAAoB;YAC9B,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAChB,kBAAkB,CAChB,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;gBAC5B,KAAK,EAAE,KAAK,CAAC,GAAG,CACd,CAAC,YAAY,EAAiB,EAAE,CAAC,CAAC;oBAChC,YAAY;oBACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;iBACpE,CAAC,CACH;gBACD,QAAQ,EAAE,WAAW,CAAC,cAAc;gBACpC,SAAS;aACV,CAAC,CACH;SACJ,CAAC,EACF,KAAK,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YACxB,MAAM,iBAAiB,GAAG,KAAK,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAClF,YAAY;gBACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;aACpE,CAAC,CAAC,CAAA;YAEH,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC;gBAC7C,KAAK,EAAE,iBAAiB;gBACxB,QAAQ,EAAE,WAAW,CAAC,cAAc;gBACpC,SAAS;aACV,CAAC,CAAA;YAEF,OAAO;gBACL,QAAQ;gBACR,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC;aAC/D,CAAA;QACH,CAAC,CAAC,CACH,CAAA;QAED,6CAA6C;QAE7C,iCAAiC;QACjC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,KAAK,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;gBAC9C,6BAA6B;gBAC7B,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;oBACvB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;gBACrG,CAAC;gBAED,qFAAqF;gBACrF,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;oBACpC,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAA;oBAE7F,+FAA+F;oBAC/F,sFAAsF;oBACtF,6EAA6E;oBAC7E,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;wBAClD,MAAM,GAAG,GAAoC;4BAC3C,IAAI,EAAE,OAAO;4BACb,SAAS;4BACT,MAAM,EAAE,CAAC,OAAO,CAAC;yBAClB,CAAA;wBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;oBAChC,CAAC;gBACH,CAAC;YACH,CAAC;YAED,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,gBAAgB,CAAC,MAAM,oBAAoB,CAAC,CAAA;QACvF,CAAC;QAED,4EAA4E;QAC5E,IAAI,gBAAgB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC9B,KAAK,MAAM,YAAY,IAAI,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrD,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;oBACpC,KAAK,CAAC,CAAC,kBAAkB,CAAC;wBACxB,aAAa,EAAE,YAAY,CAAC,aAAa;wBACzC,GAAG;wBACH,SAAS,EAAE,YAAY,CAAC,SAAS;wBACjC,MAAM,EAAE,CAAC,OAAO,CAAC;qBAClB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;gBAChD,CAAC;YACH,CAAC;YAED,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,gBAAgB,CAAC,IAAI,cAAc,CAAC,CAAA;QAC/E,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,iBAAiB,EACxB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EACrC,MAAM,CAAC,eAAe,EAAE,+DAA+D;IACvF,MAAM,CAAC,IAAI,CACZ,CAAA;IAED,8GAA8G;IAC9G,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAA;IAExB,OAAO,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACrC,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,OAAO;IAC1B,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;QACvB,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;IACpG,CAAC;AACH,CAAC,CAAC,CACH,EACD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,EAC5D,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CACrG,CAAA;AAEL;;GAEG;AACH,MAAM,qBAAqB,GACzB,CAAC,GAA+B,EAAE,EAAE,CACpC,CAAU,GAA2B,EAAE,EAAE,CACvC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,EAAK,CAAA;IAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CACtC,GAAG,CAAC,qBAAqB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAC1F,CAAA;IAED,OAAO,KAAK,CAAC,CAAC,IAAI,CAAA;AACpB,CAAC,CAAC,CAAA"}
@@ -1,15 +1,18 @@
1
1
  import { UnexpectedError } from '@livestore/common';
2
2
  import type { LiveStoreEvent } from '@livestore/common/schema';
3
3
  import type { CfTypes } from '@livestore/common-cf';
4
- import { Effect, Option } from '@livestore/utils/effect';
4
+ import { Effect, Option, Stream } from '@livestore/utils/effect';
5
5
  import { SyncMetadata } from '../../common/sync-message-types.ts';
6
6
  import { type Env, type StoreId } from '../shared.ts';
7
7
  export type SyncStorage = {
8
8
  dbName: string;
9
- getEvents: (cursor: number | undefined) => Effect.Effect<ReadonlyArray<{
10
- eventEncoded: LiveStoreEvent.AnyEncodedGlobal;
11
- metadata: Option.Option<SyncMetadata>;
12
- }>, UnexpectedError>;
9
+ getEvents: (cursor: number | undefined) => Effect.Effect<{
10
+ total: number;
11
+ stream: Stream.Stream<{
12
+ eventEncoded: LiveStoreEvent.AnyEncodedGlobal;
13
+ metadata: Option.Option<SyncMetadata>;
14
+ }, UnexpectedError>;
15
+ }, UnexpectedError>;
13
16
  appendEvents: (batch: ReadonlyArray<LiveStoreEvent.AnyEncodedGlobal>, createdAt: string) => Effect.Effect<void, UnexpectedError>;
14
17
  resetStore: Effect.Effect<void, UnexpectedError>;
15
18
  };
@@ -1 +1 @@
1
- {"version":3,"file":"sync-storage.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/sync-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAU,MAAM,yBAAyB,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAA;AACjE,OAAO,EAAE,KAAK,GAAG,EAA8B,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AAGjF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAA;IAEd,SAAS,EAAE,CACT,MAAM,EAAE,MAAM,GAAG,SAAS,KACvB,MAAM,CAAC,MAAM,CAChB,aAAa,CAAC;QAAE,YAAY,EAAE,cAAc,CAAC,gBAAgB,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;KAAE,CAAC,EACvG,eAAe,CAChB,CAAA;IACD,YAAY,EAAE,CACZ,KAAK,EAAE,aAAa,CAAC,cAAc,CAAC,gBAAgB,CAAC,EACrD,SAAS,EAAE,MAAM,KACd,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IACzC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;CACjD,CAAA;AAED,eAAO,MAAM,WAAW,GAAI,KAAK,OAAO,CAAC,kBAAkB,EAAE,KAAK,GAAG,EAAE,SAAS,OAAO,KAAG,WAmGzF,CAAA"}
1
+ {"version":3,"file":"sync-storage.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/sync-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAS,MAAM,EAAE,MAAM,EAAU,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAA;AACjE,OAAO,EAAE,KAAK,GAAG,EAA8B,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AAGjF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,KAAK,MAAM,CAAC,MAAM,CACtD;QACE,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,EAAE,MAAM,CAAC,MAAM,CACnB;YAAE,YAAY,EAAE,cAAc,CAAC,gBAAgB,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;SAAE,EACxF,eAAe,CAChB,CAAA;KACF,EACD,eAAe,CAChB,CAAA;IACD,YAAY,EAAE,CACZ,KAAK,EAAE,aAAa,CAAC,cAAc,CAAC,gBAAgB,CAAC,EACrD,SAAS,EAAE,MAAM,KACd,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IACzC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;CACjD,CAAA;AAED,eAAO,MAAM,WAAW,GAAI,KAAK,OAAO,CAAC,kBAAkB,EAAE,KAAK,GAAG,EAAE,SAAS,OAAO,KAAG,WA+KzF,CAAA"}
@@ -1,5 +1,5 @@
1
1
  import { UnexpectedError } from '@livestore/common';
2
- import { Effect, Option, Schema } from '@livestore/utils/effect';
2
+ import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect';
3
3
  import { SyncMetadata } from "../../common/sync-message-types.js";
4
4
  import { PERSISTENCE_FORMAT_VERSION } from "../shared.js";
5
5
  import { eventlogTable } from "./sqlite.js";
@@ -9,25 +9,70 @@ export const makeStorage = (ctx, env, storeId) => {
9
9
  try: () => cb(env.DB),
10
10
  catch: (error) => new UnexpectedError({ cause: error, payload: { dbName } }),
11
11
  }).pipe(Effect.map((_) => _.results), Effect.withSpan('@livestore/sync-cf:durable-object:execDb'));
12
- // const getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError> = Effect.gen(
13
- // function* () {
14
- // const result = yield* execDb<{ seqNum: EventSequenceNumber.GlobalEventSequenceNumber }>((db) =>
15
- // db.prepare(`SELECT seqNum FROM ${dbName} ORDER BY seqNum DESC LIMIT 1`).all(),
16
- // )
17
- // return result[0]?.seqNum ?? EventSequenceNumber.ROOT.global
18
- // },
19
- // ).pipe(UnexpectedError.mapToUnexpectedError)
20
- // TODO support streaming
12
+ // Cloudflare's D1 HTTP endpoint rejects JSON responses once they exceed ~1MB.
13
+ // Keep individual SELECT batches comfortably below that threshold so we can
14
+ // serve large histories without tripping the limit.
15
+ const D1_MAX_JSON_RESPONSE_BYTES = 1_000_000;
16
+ const D1_RESPONSE_SAFETY_MARGIN_BYTES = 64 * 1024;
17
+ const D1_TARGET_RESPONSE_BYTES = D1_MAX_JSON_RESPONSE_BYTES - D1_RESPONSE_SAFETY_MARGIN_BYTES;
18
+ const D1_INITIAL_PAGE_SIZE = 256;
19
+ const D1_MIN_PAGE_SIZE = 1;
20
+ const decodeEventlogRows = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema));
21
+ const textEncoder = new TextEncoder();
22
+ const decreaseLimit = (limit) => Math.max(D1_MIN_PAGE_SIZE, Math.floor(limit / 2));
23
+ const increaseLimit = (limit) => Math.min(D1_INITIAL_PAGE_SIZE, limit * 2);
24
+ const computeNextLimit = (limit, encodedSize) => {
25
+ if (encodedSize > D1_TARGET_RESPONSE_BYTES && limit > D1_MIN_PAGE_SIZE) {
26
+ const next = decreaseLimit(limit);
27
+ return next === limit ? limit : next;
28
+ }
29
+ if (encodedSize < D1_TARGET_RESPONSE_BYTES / 2 && limit < D1_INITIAL_PAGE_SIZE) {
30
+ const next = increaseLimit(limit);
31
+ return next === limit ? limit : next;
32
+ }
33
+ return limit;
34
+ };
21
35
  const getEvents = (cursor) => Effect.gen(function* () {
22
- const whereClause = cursor === undefined ? '' : `WHERE seqNum > ${cursor}`;
23
- const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY seqNum ASC`;
24
- // TODO handle case where `cursor` was not found
25
- const rawEvents = yield* execDb((db) => db.prepare(sql).all());
26
- const events = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))(rawEvents).map(({ createdAt, ...eventEncoded }) => ({
27
- eventEncoded,
28
- metadata: Option.some(SyncMetadata.make({ createdAt })),
29
- }));
30
- return events;
36
+ const countStatement = cursor === undefined
37
+ ? `SELECT COUNT(*) as total FROM ${dbName}`
38
+ : `SELECT COUNT(*) as total FROM ${dbName} WHERE seqNum > ?`;
39
+ const countRows = yield* execDb((db) => {
40
+ const prepared = db.prepare(countStatement);
41
+ return cursor === undefined ? prepared.all() : prepared.bind(cursor).all();
42
+ });
43
+ const total = Number(countRows[0]?.total ?? 0);
44
+ const initialState = { cursor, limit: D1_INITIAL_PAGE_SIZE };
45
+ const fetchPage = (state) => Effect.gen(function* () {
46
+ const statement = state.cursor === undefined
47
+ ? `SELECT * FROM ${dbName} ORDER BY seqNum ASC LIMIT ?`
48
+ : `SELECT * FROM ${dbName} WHERE seqNum > ? ORDER BY seqNum ASC LIMIT ?`;
49
+ const rawEvents = yield* execDb((db) => {
50
+ const prepared = db.prepare(statement);
51
+ return state.cursor === undefined
52
+ ? prepared.bind(state.limit).all()
53
+ : prepared.bind(state.cursor, state.limit).all();
54
+ });
55
+ if (rawEvents.length === 0) {
56
+ return Option.none();
57
+ }
58
+ const encodedSize = textEncoder.encode(JSON.stringify(rawEvents)).byteLength;
59
+ if (encodedSize > D1_TARGET_RESPONSE_BYTES && state.limit > D1_MIN_PAGE_SIZE) {
60
+ const nextLimit = decreaseLimit(state.limit);
61
+ if (nextLimit !== state.limit) {
62
+ return yield* fetchPage({ cursor: state.cursor, limit: nextLimit });
63
+ }
64
+ }
65
+ const decodedRows = Chunk.fromIterable(decodeEventlogRows(rawEvents));
66
+ const eventsChunk = Chunk.map(decodedRows, ({ createdAt, ...eventEncoded }) => ({
67
+ eventEncoded,
68
+ metadata: Option.some(SyncMetadata.make({ createdAt })),
69
+ }));
70
+ const lastSeqNum = Chunk.unsafeLast(decodedRows).seqNum;
71
+ const nextState = { cursor: lastSeqNum, limit: computeNextLimit(state.limit, encodedSize) };
72
+ return Option.some([eventsChunk, nextState]);
73
+ });
74
+ const stream = Stream.unfoldChunkEffect(initialState, fetchPage);
75
+ return { total, stream };
31
76
  }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/sync-cf:durable-object:getEvents', { attributes: { dbName, cursor } }));
32
77
  const appendEvents = (batch, createdAt) => Effect.gen(function* () {
33
78
  // If there are no events, do nothing.
@@ -1 +1 @@
1
- {"version":3,"file":"sync-storage.js","sourceRoot":"","sources":["../../../src/cf-worker/do/sync-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAGnD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAA;AACjE,OAAO,EAAY,0BAA0B,EAAgB,MAAM,cAAc,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAkB3C,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAA+B,EAAE,GAAQ,EAAE,OAAgB,EAAe,EAAE;IACtG,MAAM,MAAM,GAAG,YAAY,0BAA0B,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAA;IAEpF,MAAM,MAAM,GAAG,CAAI,EAA4D,EAAE,EAAE,CACjF,MAAM,CAAC,UAAU,CAAC;QAChB,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;KAC7E,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAC5B,MAAM,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAC5D,CAAA;IAEH,6GAA6G;IAC7G,mBAAmB;IACnB,sGAAsG;IACtG,uFAAuF;IACvF,QAAQ;IAER,kEAAkE;IAClE,OAAO;IACP,+CAA+C;IAE/C,yBAAyB;IACzB,MAAM,SAAS,GAAG,CAChB,MAA0B,EAI1B,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,WAAW,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB,MAAM,EAAE,CAAA;QAC1E,MAAM,GAAG,GAAG,iBAAiB,MAAM,IAAI,WAAW,sBAAsB,CAAA;QACxE,gDAAgD;QAChD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAC3F,CAAC,EAAE,SAAS,EAAE,GAAG,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;YACnC,YAAY;YACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;SACxD,CAAC,CACH,CAAA;QACD,OAAO,MAAM,CAAA;IACf,CAAC,CAAC,CAAC,IAAI,CACL,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,6CAA6C,EAAE,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CACnG,CAAA;IAEH,MAAM,YAAY,GAAgC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CACrE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,sCAAsC;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE9B,gBAAgB;QAChB,gFAAgF;QAChF,6EAA6E;QAC7E,MAAM,UAAU,GAAG,EAAE,CAAA;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;YAE5C,wFAAwF;YACxF,MAAM,kBAAkB,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC9E,MAAM,GAAG,GAAG,eAAe,MAAM,8EAA8E,kBAAkB,EAAE,CAAA;YACnI,wDAAwD;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;gBACtC,KAAK,CAAC,MAAM;gBACZ,KAAK,CAAC,YAAY;gBAClB,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;gBAC5D,KAAK,CAAC,IAAI;gBACV,SAAS;gBACT,KAAK,CAAC,QAAQ;gBACd,KAAK,CAAC,SAAS;aAChB,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACnB,EAAE;iBACC,OAAO,CAAC,GAAG,CAAC;iBACZ,IAAI,CAAC,GAAG,MAAM,CAAC;iBACf,GAAG,EAAE,CACT,CAAA;QACH,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,gDAAgD,EAAE;QAChE,UAAU,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE;KAClD,CAAC,CACH,CAAA;IAEH,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CACnE,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAChE,CAAA;IAED,OAAO;QACL,MAAM;QACN,WAAW;QACX,SAAS;QACT,YAAY;QACZ,UAAU;KACX,CAAA;AACH,CAAC,CAAA;AAED,MAAM,gBAAgB,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,eAAe,EAAE,GAAG,CAAC,CAAA"}
1
+ {"version":3,"file":"sync-storage.js","sourceRoot":"","sources":["../../../src/cf-worker/do/sync-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAGnD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAA;AACjE,OAAO,EAAY,0BAA0B,EAAgB,MAAM,cAAc,CAAA;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAqB3C,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,GAA+B,EAAE,GAAQ,EAAE,OAAgB,EAAe,EAAE;IACtG,MAAM,MAAM,GAAG,YAAY,0BAA0B,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAA;IAEpF,MAAM,MAAM,GAAG,CAAI,EAA4D,EAAE,EAAE,CACjF,MAAM,CAAC,UAAU,CAAC;QAChB,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;KAC7E,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAC5B,MAAM,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAC5D,CAAA;IAEH,8EAA8E;IAC9E,4EAA4E;IAC5E,oDAAoD;IACpD,MAAM,0BAA0B,GAAG,SAAS,CAAA;IAC5C,MAAM,+BAA+B,GAAG,EAAE,GAAG,IAAI,CAAA;IACjD,MAAM,wBAAwB,GAAG,0BAA0B,GAAG,+BAA+B,CAAA;IAC7F,MAAM,oBAAoB,GAAG,GAAG,CAAA;IAChC,MAAM,gBAAgB,GAAG,CAAC,CAAA;IAE1B,MAAM,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAA;IAC1F,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;IAErC,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAA;IAC1F,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;IAElF,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAE,WAAmB,EAAE,EAAE;QAC9D,IAAI,WAAW,GAAG,wBAAwB,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;YACvE,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAA;YACjC,OAAO,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;QACtC,CAAC;QAED,IAAI,WAAW,GAAG,wBAAwB,GAAG,CAAC,IAAI,KAAK,GAAG,oBAAoB,EAAE,CAAC;YAC/E,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAA;YACjC,OAAO,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;QACtC,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC,CAAA;IAED,MAAM,SAAS,GAAG,CAChB,MAA0B,EAU1B,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,cAAc,GAClB,MAAM,KAAK,SAAS;YAClB,CAAC,CAAC,iCAAiC,MAAM,EAAE;YAC3C,CAAC,CAAC,iCAAiC,MAAM,mBAAmB,CAAA;QAEhE,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,MAAM,CAAoB,CAAC,EAAE,EAAE,EAAE;YACxD,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC3C,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAA;QAC5E,CAAC,CAAC,CAAA;QAEF,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAA;QAK9C,MAAM,YAAY,GAAU,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAA;QAEnE,MAAM,SAAS,GAAG,CAChB,KAAY,EACgF,EAAE,CAC9F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,MAAM,SAAS,GACb,KAAK,CAAC,MAAM,KAAK,SAAS;gBACxB,CAAC,CAAC,iBAAiB,MAAM,8BAA8B;gBACvD,CAAC,CAAC,iBAAiB,MAAM,+CAA+C,CAAA;YAE5E,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;gBACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;gBACtC,OAAO,KAAK,CAAC,MAAM,KAAK,SAAS;oBAC/B,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE;oBAClC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAA;YACpD,CAAC,CAAC,CAAA;YAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO,MAAM,CAAC,IAAI,EAAE,CAAA;YACtB,CAAC;YAED,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAA;YAE5E,IAAI,WAAW,GAAG,wBAAwB,IAAI,KAAK,CAAC,KAAK,GAAG,gBAAgB,EAAE,CAAC;gBAC7E,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAE5C,IAAI,SAAS,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC9B,OAAO,KAAK,CAAC,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;gBACrE,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,KAAK,CAAC,YAAY,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAA;YAErE,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC9E,YAAY;gBACZ,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;aACxD,CAAC,CAAC,CAAA;YAEH,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,MAAM,CAAA;YACvD,MAAM,SAAS,GAAU,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,gBAAgB,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,CAAA;YAElG,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,SAAS,CAAU,CAAC,CAAA;QACvD,CAAC,CAAC,CAAA;QAEJ,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC,YAAY,EAAE,SAAS,CAAC,CAAA;QAEhE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;IAC1B,CAAC,CAAC,CAAC,IAAI,CACL,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,6CAA6C,EAAE,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CACnG,CAAA;IAEH,MAAM,YAAY,GAAgC,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CACrE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,sCAAsC;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE9B,gBAAgB;QAChB,gFAAgF;QAChF,6EAA6E;QAC7E,MAAM,UAAU,GAAG,EAAE,CAAA;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAA;YAE5C,wFAAwF;YACxF,MAAM,kBAAkB,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC9E,MAAM,GAAG,GAAG,eAAe,MAAM,8EAA8E,kBAAkB,EAAE,CAAA;YACnI,wDAAwD;YACxD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;gBACtC,KAAK,CAAC,MAAM;gBACZ,KAAK,CAAC,YAAY;gBAClB,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;gBAC5D,KAAK,CAAC,IAAI;gBACV,SAAS;gBACT,KAAK,CAAC,QAAQ;gBACd,KAAK,CAAC,SAAS;aAChB,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACnB,EAAE;iBACC,OAAO,CAAC,GAAG,CAAC;iBACZ,IAAI,CAAC,GAAG,MAAM,CAAC;iBACf,GAAG,EAAE,CACT,CAAA;QACH,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,gDAAgD,EAAE;QAChE,UAAU,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE;KAClD,CAAC,CACH,CAAA;IAEH,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CACnE,eAAe,CAAC,oBAAoB,EACpC,MAAM,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAChE,CAAA;IAED,OAAO;QACL,MAAM;QACN,WAAW;QACX,SAAS;QACT,YAAY;QACZ,UAAU;KACX,CAAA;AACH,CAAC,CAAA;AAED,MAAM,gBAAgB,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,eAAe,EAAE,GAAG,CAAC,CAAA"}
@@ -1,8 +1,9 @@
1
+ import { type CfTypes } from '@livestore/common-cf';
1
2
  import { Effect } from '@livestore/utils/effect';
2
3
  import { type DoCtxInput } from '../layer.ts';
3
4
  export interface DoRpcHandlerOptions {
4
5
  payload: Uint8Array<ArrayBuffer>;
5
6
  input: Omit<DoCtxInput, 'from'>;
6
7
  }
7
- export declare const createDoRpcHandler: (options: DoRpcHandlerOptions) => Effect.Effect<import("@cloudflare/workers-types").ReadableStream<any> | Uint8Array<ArrayBuffer>, never, never>;
8
+ export declare const createDoRpcHandler: (options: DoRpcHandlerOptions) => Effect.Effect<Uint8Array<ArrayBuffer> | CfTypes.ReadableStream>;
8
9
  //# sourceMappingURL=do-rpc-server.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"do-rpc-server.d.ts","sourceRoot":"","sources":["../../../../src/cf-worker/do/transport/do-rpc-server.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,MAAM,EASP,MAAM,yBAAyB,CAAA;AAGhC,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAA;AAIpD,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,UAAU,CAAC,WAAW,CAAC,CAAA;IAChC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;CAChC;AAED,eAAO,MAAM,kBAAkB,GAAI,SAAS,mBAAmB,mHAyDf,CAAA"}
1
+ {"version":3,"file":"do-rpc-server.d.ts","sourceRoot":"","sources":["../../../../src/cf-worker/do/transport/do-rpc-server.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,OAAO,EAA0B,MAAM,sBAAsB,CAAA;AAC3E,OAAO,EACL,MAAM,EASP,MAAM,yBAAyB,CAAA;AAGhC,OAAO,EAAS,KAAK,UAAU,EAAE,MAAM,aAAa,CAAA;AAIpD,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,UAAU,CAAC,WAAW,CAAC,CAAA;IAChC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;CAChC;AAED,eAAO,MAAM,kBAAkB,GAC7B,SAAS,mBAAmB,KAC3B,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,cAAc,CAyDjB,CAAA"}
@@ -1 +1 @@
1
- {"version":3,"file":"do-rpc-server.js","sourceRoot":"","sources":["../../../../src/cf-worker/do/transport/do-rpc-server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AAC7D,OAAO,EACL,MAAM,EACN,OAAO,EACP,UAAU,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,MAAM,EACN,gBAAgB,EAChB,MAAM,GACP,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAA;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAmB,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAOrC,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,OAA4B,EAAE,EAAE,CACjE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;IAClC,4EAA4E;IAE5E,sBAAsB;IACtB,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;QAChC,gBAAgB,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QAClD,CAAC;QACD,gBAAgB,EAAE,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CACrC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;YACxB,MAAM,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;YAEzC,6DAA6D;YAC7D,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;gBACnB,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE;oBAChC,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE;oBACxB,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;oBAC3E,aAAa,EAAE,GAAG,CAAC,UAAU,CAAC,aAAa;iBAC5C,CAAC,CAAA;YACJ,CAAC;YAED,OAAO,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnB,GAAG,GAAG;YACN,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;SAC/E,CAAC,CAAC,EACH,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAChF,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAC1G,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CACjC;QACH,gBAAgB,EAAE,CAAC,GAAG,EAAE,EAAE,CACxB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;YACxB,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;YACrD,MAAM,IAAI,GAAG,QAAQ,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;YAEtF,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACzB,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAC3E,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAC1G,MAAM,CAAC,iBAAiB,CACzB;KACJ,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,sBAAsB,CAAC,SAAS,EAAE;QAChD,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC,IAAI,CACtF,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,EACjD,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CACtD;KACF,CAAC,CAAA;IAEF,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;AAChC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAA"}
1
+ {"version":3,"file":"do-rpc-server.js","sourceRoot":"","sources":["../../../../src/cf-worker/do/transport/do-rpc-server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACtE,OAAO,EAAgB,sBAAsB,EAAE,MAAM,sBAAsB,CAAA;AAC3E,OAAO,EACL,MAAM,EACN,OAAO,EACP,UAAU,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,MAAM,EACN,gBAAgB,EAChB,MAAM,GACP,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAA;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAmB,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAOrC,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,OAA4B,EACqC,EAAE,CACnE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;IAClC,4EAA4E;IAE5E,sBAAsB;IACtB,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;QAChC,gBAAgB,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QAClD,CAAC;QACD,gBAAgB,EAAE,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CACrC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;YACxB,MAAM,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;YAEzC,6DAA6D;YAC7D,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;gBACnB,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE;oBAChC,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE;oBACxB,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;oBAC3E,aAAa,EAAE,GAAG,CAAC,UAAU,CAAC,aAAa;iBAC5C,CAAC,CAAA;YACJ,CAAC;YAED,OAAO,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnB,GAAG,GAAG;YACN,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;SAC/E,CAAC,CAAC,EACH,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAChF,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAC1G,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CACjC;QACH,gBAAgB,EAAE,CAAC,GAAG,EAAE,EAAE,CACxB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;YACxB,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;YACrD,MAAM,IAAI,GAAG,QAAQ,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;YAEtF,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACzB,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAC3E,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAC1G,MAAM,CAAC,iBAAiB,CACzB;KACJ,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,sBAAsB,CAAC,SAAS,EAAE;QAChD,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC,IAAI,CACtF,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,EACjD,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CACtD;KACF,CAAC,CAAA;IAEF,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;AAChC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAA"}
@@ -3,5 +3,5 @@ import { Effect } from '@livestore/utils/effect';
3
3
  import { DoCtx } from '../layer.ts';
4
4
  export declare const createHttpRpcHandler: ({ request }: {
5
5
  request: CfTypes.Request;
6
- }) => Effect.Effect<CfTypes.Response, import("effect/Cause").TimeoutException, DoCtx | import("effect/Scope").Scope>;
6
+ }) => Effect.Effect<CfTypes.Response, import("effect/Cause").TimeoutException, import("effect/Scope").Scope | DoCtx>;
7
7
  //# sourceMappingURL=http-rpc-server.d.ts.map
@@ -0,0 +1,22 @@
1
+ import { Chunk } from '@livestore/utils/effect';
2
+ /**
3
+ * Configuration describing how to break a chunk into smaller payload-safe chunks.
4
+ */
5
+ export interface ChunkingOptions<A> {
6
+ /** Maximum number of items that may appear in any emitted chunk. */
7
+ readonly maxItems: number;
8
+ /** Maximum encoded byte size allowed for any emitted chunk. */
9
+ readonly maxBytes: number;
10
+ /**
11
+ * Callback that produces a JSON-serialisable structure whose byte size should
12
+ * fit within {@link maxBytes}. This lets callers control framing overhead.
13
+ */
14
+ readonly encode: (items: ReadonlyArray<A>) => unknown;
15
+ }
16
+ /**
17
+ * Derives a function that splits an input chunk into sub-chunks confined by
18
+ * both item count and encoded byte size limits. Designed for transports with
19
+ * strict frame caps (e.g. Cloudflare hibernated WebSockets).
20
+ */
21
+ export declare const splitChunkBySize: <A>(options: ChunkingOptions<A>) => (chunk: Chunk.Chunk<A>) => Chunk.Chunk<Chunk.Chunk<A>>;
22
+ //# sourceMappingURL=ws-chunking.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ws-chunking.d.ts","sourceRoot":"","sources":["../../../src/cf-worker/do/ws-chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAA;AAI/C;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,oEAAoE;IACpE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,OAAO,CAAA;CACtD;AAED;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,GAC1B,CAAC,EAAE,SAAS,eAAe,CAAC,CAAC,CAAC,MAC9B,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAiDlD,CAAA"}