@dxos/echo-pipeline 0.7.4 → 0.7.5-feature-compute.4d9d99a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/lib/browser/{chunk-LZK5YFYE.mjs → chunk-QBMTPEMY.mjs} +111 -38
  2. package/dist/lib/browser/chunk-QBMTPEMY.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +171 -77
  4. package/dist/lib/browser/index.mjs.map +4 -4
  5. package/dist/lib/browser/meta.json +1 -1
  6. package/dist/lib/browser/testing/index.mjs +1 -1
  7. package/dist/lib/node/{chunk-MACQJ2EP.cjs → chunk-NPZ57MV5.cjs} +110 -40
  8. package/dist/lib/node/chunk-NPZ57MV5.cjs.map +7 -0
  9. package/dist/lib/node/index.cjs +184 -94
  10. package/dist/lib/node/index.cjs.map +4 -4
  11. package/dist/lib/node/meta.json +1 -1
  12. package/dist/lib/node/testing/index.cjs +10 -10
  13. package/dist/lib/node-esm/{chunk-JIZPSASG.mjs → chunk-OY5N3ZIV.mjs} +111 -38
  14. package/dist/lib/node-esm/chunk-OY5N3ZIV.mjs.map +7 -0
  15. package/dist/lib/node-esm/index.mjs +171 -77
  16. package/dist/lib/node-esm/index.mjs.map +4 -4
  17. package/dist/lib/node-esm/meta.json +1 -1
  18. package/dist/lib/node-esm/testing/index.mjs +1 -1
  19. package/dist/types/src/automerge/automerge-host.d.ts +5 -1
  20. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  21. package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -0
  22. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  23. package/dist/types/src/automerge/echo-network-adapter.d.ts +1 -0
  24. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  25. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
  26. package/dist/types/src/db-host/echo-host.d.ts +3 -2
  27. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  28. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  29. package/dist/types/src/edge/inflight-request-limiter.d.ts +24 -0
  30. package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -0
  31. package/dist/types/src/pipeline/pipeline.d.ts +1 -0
  32. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  33. package/dist/types/src/space/control-pipeline.d.ts +9 -0
  34. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  35. package/dist/types/src/space/space-manager.d.ts +1 -0
  36. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  37. package/dist/types/tsconfig.tsbuildinfo +1 -0
  38. package/package.json +34 -34
  39. package/src/automerge/automerge-host.ts +49 -14
  40. package/src/automerge/collection-synchronizer.ts +8 -4
  41. package/src/automerge/echo-network-adapter.ts +7 -0
  42. package/src/automerge/mesh-echo-replicator.ts +2 -2
  43. package/src/db-host/echo-host.ts +4 -1
  44. package/src/edge/echo-edge-replicator.ts +34 -18
  45. package/src/edge/inflight-request-limiter.ts +69 -0
  46. package/src/pipeline/pipeline.ts +9 -1
  47. package/src/space/control-pipeline.ts +25 -2
  48. package/src/space/space-manager.ts +17 -1
  49. package/dist/lib/browser/chunk-LZK5YFYE.mjs.map +0 -7
  50. package/dist/lib/node/chunk-MACQJ2EP.cjs.map +0 -7
  51. package/dist/lib/node-esm/chunk-JIZPSASG.mjs.map +0 -7
@@ -108,6 +108,7 @@ export declare class Pipeline implements PipelineAccessor {
108
108
  getFeeds(): FeedWrapper<FeedMessage>[];
109
109
  addFeed(feed: FeedWrapper<FeedMessage>): Promise<void>;
110
110
  setWriteFeed(feed: FeedWrapper<FeedMessage>): void;
111
+ retryMessage(message: FeedMessageBlock): void;
111
112
  start(): Promise<void>;
112
113
  stop(): Promise<void>;
113
114
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAA2C,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAmB,MAAM,eAAe,CAAC;AAEzD,OAAO,EAAmB,KAAK,WAAW,EAAE,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEtF,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,OAAO,EAAyC,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAG1F,MAAM,MAAM,4BAA4B,GAAG;IACzC;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;IAgCtB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IA1BzB,SAAgB,eAAe,mBAA+B;IAE9D,SAAgB,OAAO,cAAe;IAOtC;;OAEG;IACH,OAAO,CAAC,gBAAgB,CAAwB;gBAatC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC,EACvD,eAAe,EAAE,cAAc;IAGzC;;;OAGG;IAEH,IAAI,YAAY,cASf;IAED,IAAI,cAAc,cAEjB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,gBAAgB,cAEnB;IAED,IAAI,eAAe,cAElB;IAED,IAAI,aAAa,YAEhB;IAED,IAAI,KAAK,+BAER;IAEK,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAI1C,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAIpC;;;;;;OAMG;IACG,+BAA+B,CAAC,EACpC,GAAmB,EACnB,OAAO,EACP,YAAmB,GACpB,GAAE,4BAAiC;CAyCrC;AAGD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,QAAS,YAAW,gBAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAuC;IACvE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IAGtD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuF;IAGlH,OAAO,CAAC,gBAAgB,CAAC,CAA+B;IAGxD,OAAO,CAAC,OAAO,CAA8C;IAE7D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,SAAS,CAAS;IAE1B,IAAI,KAAK,kBAER;IAED,IAAI,MAAM,IAAI,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAG5C;IAED,OAAO,CAAC,OAAO,EAAE,SAAS;IAI1B,QAAQ,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE;IAMhC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAY5C,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAcrC,KAAK;IAgBL,IAAI;IAiBV;;;OAGG;IAEG,SAAS,CAAC,SAAS,EAAE,SAAS;IAcpC;;OAEG;IAEG,KAAK;IAWL,OAAO;IAWb;;;OAGG;IACI,OAAO,IAAI,aAAa,CAAC,gBAAgB,CAAC;IAkCjD,OAAO,CAAC,qBAAqB;YAoBf,aAAa;CAe5B"}
1
+ {"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/pipeline/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,EAA2C,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAmB,MAAM,eAAe,CAAC;AAEzD,OAAO,EAAmB,KAAK,WAAW,EAAE,KAAK,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEtF,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,OAAO,EAAyC,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAG1F,MAAM,MAAM,4BAA4B,GAAG;IACzC;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,qBAAa,aAAa;IAgCtB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IA1BzB,SAAgB,eAAe,mBAA+B;IAE9D,SAAgB,OAAO,cAAe;IAOtC;;OAEG;IACH,OAAO,CAAC,gBAAgB,CAAwB;gBAatC,MAAM,EAAE,UAAU,CAAC,SAAS,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC,EACvD,eAAe,EAAE,cAAc;IAGzC;;;OAGG;IAEH,IAAI,YAAY,cASf;IAED,IAAI,cAAc,cAEjB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,gBAAgB,cAEnB;IAED,IAAI,eAAe,cAElB;IAED,IAAI,aAAa,YAEhB;IAED,IAAI,KAAK,+BAER;IAEK,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAI1C,kBAAkB,CAAC,MAAM,EAAE,SAAS;IAIpC;;;;;;OAMG;IACG,+BAA+B,CAAC,EACpC,GAAmB,EACnB,OAAO,EACP,YAAmB,GACpB,GAAE,4BAAiC;CAyCrC;AAGD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,QAAS,YAAW,gBAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAuC;IACvE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuE;IAG9F,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IAGtD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuF;IAGlH,OAAO,CAAC,gBAAgB,CAAC,CAA+B;IAGxD,OAAO,CAAC,OAAO,CAA8C;IAE7D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,SAAS,CAAS;IAE1B,IAAI,KAAK,kBAER;IAED,IAAI,MAAM,IAAI,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAG5C;IAED,OAAO,CAAC,OAAO,EAAE,SAAS;IAI1B,QAAQ,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE;IAMhC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAY5C,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAa3C,YAAY,CAAC,OAAO,EAAE,gBAAgB;IAMhC,KAAK;IAgBL,IAAI;IAiBV;;;OAGG;IAEG,SAAS,CAAC,SAAS,EAAE,SAAS;IAcpC;;OAEG;IAEG,KAAK;IAWL,OAAO;IAWb;;;OAGG;IACI,OAAO,IAAI,aAAa,CAAC,gBAAgB,CAAC;IAkCjD,OAAO,CAAC,qBAAqB;YAoBf,aAAa;CAkB5B"}
@@ -19,6 +19,10 @@ export declare class ControlPipeline {
19
19
  private readonly _ctx;
20
20
  private readonly _pipeline;
21
21
  private readonly _spaceStateMachine;
22
+ /**
23
+ * Map to keep track of failed messages.
24
+ */
25
+ private readonly _failedMessages;
22
26
  private readonly _spaceKey;
23
27
  private readonly _metadata;
24
28
  private _targetTimeframe?;
@@ -40,6 +44,11 @@ export declare class ControlPipeline {
40
44
  private _saveSnapshot;
41
45
  private _consumePipeline;
42
46
  private _processMessage;
47
+ /**
48
+ * If it first failure, it will be retried once the pipeline is processed fully.
49
+ * If it fails again, it will be ignored.
50
+ */
51
+ private _retryMessage;
43
52
  private _noteTargetStateIfNeeded;
44
53
  stop(): Promise<void>;
45
54
  private _saveTargetTimeframe;
@@ -1 +1 @@
1
- {"version":3,"file":"control-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/space/control-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,QAAQ,EACb,KAAK,4BAA4B,EAClC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAExE,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,6CAA6C,CAAC;AAG5F,OAAO,EAAE,KAAK,aAAa,EAAE,QAAQ,EAAU,MAAM,YAAY,CAAC;AAElE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAY,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE9D,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,EAAE,SAAS,CAAC;IACpB,WAAW,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;IACtC,YAAY,EAAE,CAAC,OAAO,EAAE,SAAS,KAAK,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;IACxE,aAAa,EAAE,aAAa,CAAC;CAC9B,CAAC;AAQF;;GAEG;AACH,qBAEa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoB;IAEvD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;IAC1C,OAAO,CAAC,gBAAgB,CAAC,CAAY;IACrC,OAAO,CAAC,sBAAsB,CAAsB;IAEpD,SAAgB,cAAc,oCAA2C;IACzE,SAAgB,mBAAmB,EAAE,QAAQ,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC3E,SAAgB,qBAAqB,EAAE,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3E,SAAgB,qBAAqB,EAAE,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC,CAAC;IAC7F,SAAgB,4BAA4B,EAAE,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC,CAAC;IAGpG,OAAO,CAAC,MAAM,CAA0B;IAGxC,OAAO,CAAC,UAAU,CAA2B;IAE7C,OAAO,CAAC,aAAa,CAGlB;gBAES,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,EAAE,qBAAqB;IAkCzF,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,QAAQ,IAAI,gBAAgB,CAE/B;IAEK,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAM3C,KAAK;YAgBG,gBAAgB;YAehB,aAAa;YAgBb,gBAAgB;YAgBhB,eAAe;YAoBf,wBAAwB;IAUhC,IAAI;YAQI,oBAAoB;CASnC"}
1
+ {"version":3,"file":"control-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/space/control-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,QAAQ,EACb,KAAK,4BAA4B,EAClC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAExE,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,6CAA6C,CAAC;AAG5F,OAAO,EAAE,KAAK,aAAa,EAAE,QAAQ,EAAsB,MAAM,YAAY,CAAC;AAE9E,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAY,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE9D,MAAM,MAAM,qBAAqB,GAAG;IAClC,QAAQ,EAAE,SAAS,CAAC;IACpB,WAAW,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;IACtC,YAAY,EAAE,CAAC,OAAO,EAAE,SAAS,KAAK,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC;IACxE,aAAa,EAAE,aAAa,CAAC;CAC9B,CAAC;AAQF;;GAEG;AACH,qBAEa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoB;IAEvD;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAE9B;IAEF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;IAC1C,OAAO,CAAC,gBAAgB,CAAC,CAAY;IACrC,OAAO,CAAC,sBAAsB,CAAsB;IAEpD,SAAgB,cAAc,oCAA2C;IACzE,SAAgB,mBAAmB,EAAE,QAAQ,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAC3E,SAAgB,qBAAqB,EAAE,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3E,SAAgB,qBAAqB,EAAE,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC,CAAC;IAC7F,SAAgB,4BAA4B,EAAE,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC,CAAC;IAGpG,OAAO,CAAC,MAAM,CAA0B;IAGxC,OAAO,CAAC,UAAU,CAA2B;IAE7C,OAAO,CAAC,aAAa,CAGlB;gBAES,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,EAAE,qBAAqB;IAkCzF,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,QAAQ,IAAI,gBAAgB,CAE/B;IAEK,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC;IAM3C,KAAK;YAgBG,gBAAgB;YAehB,aAAa;YAgBb,gBAAgB;YAgBhB,eAAe;IAqB7B;;;OAGG;IACH,OAAO,CAAC,aAAa;YAWP,wBAAwB;IAUhC,IAAI;YAQI,oBAAoB;CASnC"}
@@ -53,5 +53,6 @@ export declare class SpaceManager {
53
53
  close(): Promise<void>;
54
54
  constructSpace({ metadata, swarmIdentity, onAuthorizedConnection, onAuthFailure, onDelegatedInvitationStatusChange, onMemberRolesChanged, memberKey, }: ConstructSpaceParams): Promise<Space>;
55
55
  requestSpaceAdmissionCredential(params: RequestSpaceAdmissionCredentialParams): Promise<Credential>;
56
+ findSpaceByRootDocumentId(documentId: string): Space | undefined;
56
57
  }
57
58
  //# sourceMappingURL=space-manager.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"space-manager.d.ts","sourceRoot":"","sources":["../../../../src/space/space-manager.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,4BAA4B,EAAE,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEvF,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,KAAK,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,0CAA0C,CAAC;AAC9E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6CAA6C,CAAC;AAC9E,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,sCAAsC,CAAC;AACtE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAErE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;IAClC,cAAc,EAAE,mBAAmB,CAAC;IACpC,aAAa,EAAE,aAAa,CAAC;IAE7B,SAAS,EAAE,SAAS,CAAC;IAErB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,aAAa,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB;;OAEG;IACH,sBAAsB,EAAE,CAAC,OAAO,EAAE,QAAQ,KAAK,IAAI,CAAC;IACpD,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC5C,iCAAiC,EAAE,CAAC,UAAU,EAAE,4BAA4B,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClH,oBAAoB,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D,CAAC;AAEF,MAAM,MAAM,qCAAqC,GAAG;IAClD,QAAQ,EAAE,SAAS,CAAC;IACpB,WAAW,EAAE,SAAS,CAAC;IACvB,aAAa,EAAE,aAAa,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,qBACa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAC5E,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;IAC1D,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;gBAErC,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,qBAAqB,EAAE,EAAE,kBAAkB;IAU9G,IAAI,MAAM,iCAET;IAGK,IAAI;IAGJ,KAAK;IAIL,cAAc,CAAC,EACnB,QAAQ,EACR,aAAa,EACb,sBAAsB,EACtB,aAAa,EACb,iCAAiC,EACjC,oBAAoB,EACpB,SAAS,GACV,EAAE,oBAAoB;IAoCV,+BAA+B,CAAC,MAAM,EAAE,qCAAqC,GAAG,OAAO,CAAC,UAAU,CAAC;CAoCjH"}
1
+ {"version":3,"file":"space-manager.d.ts","sourceRoot":"","sources":["../../../../src/space/space-manager.ts"],"names":[],"mappings":"AAMA,OAAO,EAA0B,KAAK,4BAA4B,EAAE,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/G,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,KAAK,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,0CAA0C,CAAC;AAC9E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6CAA6C,CAAC;AAC9E,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,sCAAsC,CAAC;AACtE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAChC,OAAO,EAAiB,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAErE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;IAClC,cAAc,EAAE,mBAAmB,CAAC;IACpC,aAAa,EAAE,aAAa,CAAC;IAE7B,SAAS,EAAE,SAAS,CAAC;IAErB,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,aAAa,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB;;OAEG;IACH,sBAAsB,EAAE,CAAC,OAAO,EAAE,QAAQ,KAAK,IAAI,CAAC;IACpD,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC5C,iCAAiC,EAAE,CAAC,UAAU,EAAE,4BAA4B,EAAE,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClH,oBAAoB,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/D,CAAC;AAEF,MAAM,MAAM,qCAAqC,GAAG;IAClD,QAAQ,EAAE,SAAS,CAAC;IACpB,WAAW,EAAE,SAAS,CAAC;IACvB,aAAa,EAAE,aAAa,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,qBACa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoD;IAC5E,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;IAC1D,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;gBAErC,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,qBAAqB,EAAE,EAAE,kBAAkB;IAU9G,IAAI,MAAM,iCAET;IAGK,IAAI;IAGJ,KAAK;IAIL,cAAc,CAAC,EACnB,QAAQ,EACR,aAAa,EACb,sBAAsB,EACtB,aAAa,EACb,iCAAiC,EACjC,oBAAoB,EACpB,SAAS,GACV,EAAE,oBAAoB;IAoCV,+BAA+B,CAAC,MAAM,EAAE,qCAAqC,GAAG,OAAO,CAAC,UAAU,CAAC;IAqCzG,yBAAyB,CAAC,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS;CAcxE"}
@@ -0,0 +1 @@
1
+ {"version":"5.7.2"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/echo-pipeline",
3
- "version": "0.7.4",
3
+ "version": "0.7.5-feature-compute.4d9d99a",
4
4
  "description": "ECHO database.",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -37,44 +37,44 @@
37
37
  "crc-32": "^1.2.2",
38
38
  "level-transcoder": "^1.0.1",
39
39
  "lodash.isequal": "^4.5.0",
40
- "@dxos/async": "0.7.4",
41
- "@dxos/automerge": "0.7.4",
42
- "@dxos/context": "0.7.4",
43
- "@dxos/codec-protobuf": "0.7.4",
44
- "@dxos/credentials": "0.7.4",
45
- "@dxos/crypto": "0.7.4",
46
- "@dxos/debug": "0.7.4",
47
- "@dxos/feed-store": "0.7.4",
48
- "@dxos/hypercore": "0.7.4",
49
- "@dxos/echo-protocol": "0.7.4",
50
- "@dxos/echo-schema": "0.7.4",
51
- "@dxos/edge-client": "0.7.4",
52
- "@dxos/invariant": "0.7.4",
53
- "@dxos/indexing": "0.7.4",
54
- "@dxos/keyring": "0.7.4",
55
- "@dxos/kv-store": "0.7.4",
56
- "@dxos/log": "0.7.4",
57
- "@dxos/keys": "0.7.4",
58
- "@dxos/messaging": "0.7.4",
59
- "@dxos/node-std": "0.7.4",
60
- "@dxos/protocols": "0.7.4",
61
- "@dxos/random-access-storage": "0.7.4",
62
- "@dxos/network-manager": "0.7.4",
63
- "@dxos/teleport": "0.7.4",
64
- "@dxos/teleport-extension-automerge-replicator": "0.7.4",
65
- "@dxos/teleport-extension-object-sync": "0.7.4",
66
- "@dxos/teleport-extension-gossip": "0.7.4",
67
- "@dxos/timeframe": "0.7.4",
68
- "@dxos/teleport-extension-replicator": "0.7.4",
69
- "@dxos/typings": "0.7.4",
70
- "@dxos/tracing": "0.7.4",
71
- "@dxos/util": "0.7.4"
40
+ "@dxos/async": "0.7.5-feature-compute.4d9d99a",
41
+ "@dxos/codec-protobuf": "0.7.5-feature-compute.4d9d99a",
42
+ "@dxos/context": "0.7.5-feature-compute.4d9d99a",
43
+ "@dxos/automerge": "0.7.5-feature-compute.4d9d99a",
44
+ "@dxos/credentials": "0.7.5-feature-compute.4d9d99a",
45
+ "@dxos/crypto": "0.7.5-feature-compute.4d9d99a",
46
+ "@dxos/debug": "0.7.5-feature-compute.4d9d99a",
47
+ "@dxos/echo-protocol": "0.7.5-feature-compute.4d9d99a",
48
+ "@dxos/echo-schema": "0.7.5-feature-compute.4d9d99a",
49
+ "@dxos/edge-client": "0.7.5-feature-compute.4d9d99a",
50
+ "@dxos/feed-store": "0.7.5-feature-compute.4d9d99a",
51
+ "@dxos/hypercore": "0.7.5-feature-compute.4d9d99a",
52
+ "@dxos/indexing": "0.7.5-feature-compute.4d9d99a",
53
+ "@dxos/invariant": "0.7.5-feature-compute.4d9d99a",
54
+ "@dxos/keyring": "0.7.5-feature-compute.4d9d99a",
55
+ "@dxos/log": "0.7.5-feature-compute.4d9d99a",
56
+ "@dxos/messaging": "0.7.5-feature-compute.4d9d99a",
57
+ "@dxos/keys": "0.7.5-feature-compute.4d9d99a",
58
+ "@dxos/network-manager": "0.7.5-feature-compute.4d9d99a",
59
+ "@dxos/protocols": "0.7.5-feature-compute.4d9d99a",
60
+ "@dxos/random-access-storage": "0.7.5-feature-compute.4d9d99a",
61
+ "@dxos/kv-store": "0.7.5-feature-compute.4d9d99a",
62
+ "@dxos/teleport": "0.7.5-feature-compute.4d9d99a",
63
+ "@dxos/teleport-extension-automerge-replicator": "0.7.5-feature-compute.4d9d99a",
64
+ "@dxos/teleport-extension-gossip": "0.7.5-feature-compute.4d9d99a",
65
+ "@dxos/teleport-extension-object-sync": "0.7.5-feature-compute.4d9d99a",
66
+ "@dxos/teleport-extension-replicator": "0.7.5-feature-compute.4d9d99a",
67
+ "@dxos/timeframe": "0.7.5-feature-compute.4d9d99a",
68
+ "@dxos/tracing": "0.7.5-feature-compute.4d9d99a",
69
+ "@dxos/node-std": "0.7.5-feature-compute.4d9d99a",
70
+ "@dxos/util": "0.7.5-feature-compute.4d9d99a",
71
+ "@dxos/typings": "0.7.5-feature-compute.4d9d99a"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@types/lodash.isequal": "^4.5.0",
75
75
  "fast-check": "^3.19.0",
76
76
  "get-port-please": "^3.1.1",
77
- "@dxos/test-utils": "0.7.4"
77
+ "@dxos/test-utils": "0.7.5-feature-compute.4d9d99a"
78
78
  },
79
79
  "publishConfig": {
80
80
  "access": "public"
@@ -44,6 +44,8 @@ import { LevelDBStorageAdapter, type BeforeSaveParams } from './leveldb-storage-
44
44
 
45
45
  export type PeerIdProvider = () => string | undefined;
46
46
 
47
+ export type RootDocumentSpaceKeyProvider = (documentId: string) => PublicKey | undefined;
48
+
47
49
  export type AutomergeHostParams = {
48
50
  db: LevelDB;
49
51
 
@@ -54,6 +56,7 @@ export type AutomergeHostParams = {
54
56
  * Used for creating stable ids. A random key is generated on open, if no value is provided.
55
57
  */
56
58
  peerIdProvider?: PeerIdProvider;
59
+ getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
57
60
  };
58
61
 
59
62
  export type LoadDocOptions = {
@@ -90,10 +93,17 @@ export class AutomergeHost extends Resource {
90
93
  private _peerId!: PeerId;
91
94
 
92
95
  private readonly _peerIdProvider?: PeerIdProvider;
96
+ private readonly _getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
93
97
 
94
98
  public readonly collectionStateUpdated = new Event<{ collectionId: CollectionId }>();
95
99
 
96
- constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider }: AutomergeHostParams) {
100
+ constructor({
101
+ db,
102
+ indexMetadataStore,
103
+ dataMonitor,
104
+ peerIdProvider,
105
+ getSpaceKeyByRootDocumentId,
106
+ }: AutomergeHostParams) {
97
107
  super();
98
108
  this._db = db;
99
109
  this._storage = new LevelDBStorageAdapter({
@@ -114,6 +124,7 @@ export class AutomergeHost extends Resource {
114
124
  this._headsStore = new HeadsStore({ db: db.sublevel('heads') });
115
125
  this._indexMetadataStore = indexMetadataStore;
116
126
  this._peerIdProvider = peerIdProvider;
127
+ this._getSpaceKeyByRootDocumentId = getSpaceKeyByRootDocumentId;
117
128
  }
118
129
 
119
130
  protected override async _open() {
@@ -132,14 +143,28 @@ export class AutomergeHost extends Resource {
132
143
  ],
133
144
  });
134
145
 
135
- Event.wrap(this._echoNetworkAdapter, 'peer-candidate').on(this._ctx, ((e: PeerCandidatePayload) =>
136
- this._onPeerConnected(e.peerId)) as any);
137
- Event.wrap(this._echoNetworkAdapter, 'peer-disconnected').on(this._ctx, ((e: PeerDisconnectedPayload) =>
138
- this._onPeerDisconnected(e.peerId)) as any);
146
+ let updatingAuthScope = false;
147
+ Event.wrap(this._echoNetworkAdapter, 'peer-candidate').on(
148
+ this._ctx,
149
+ ((e: PeerCandidatePayload) => !updatingAuthScope && this._onPeerConnected(e.peerId)) as any,
150
+ );
151
+ Event.wrap(this._echoNetworkAdapter, 'peer-disconnected').on(
152
+ this._ctx,
153
+ ((e: PeerDisconnectedPayload) => !updatingAuthScope && this._onPeerDisconnected(e.peerId)) as any,
154
+ );
139
155
 
140
- this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
156
+ this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId, newDocsAppeared }) => {
141
157
  this._onRemoteCollectionStateUpdated(collectionId, peerId);
142
158
  this.collectionStateUpdated.emit({ collectionId: collectionId as CollectionId });
159
+ // We use collection lookups during share policy check, so we might need to update share policy for the new doc
160
+ if (newDocsAppeared) {
161
+ updatingAuthScope = true;
162
+ try {
163
+ this._echoNetworkAdapter.onConnectionAuthScopeChanged(peerId);
164
+ } finally {
165
+ updatingAuthScope = false;
166
+ }
167
+ }
143
168
  });
144
169
 
145
170
  await this._echoNetworkAdapter.open();
@@ -351,16 +376,23 @@ export class AutomergeHost extends Resource {
351
376
 
352
377
  private async _getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null> {
353
378
  const doc = this._repo.handles[documentId as any]?.docSync();
354
- if (!doc) {
355
- return null;
379
+ if (doc) {
380
+ const spaceKeyHex = getSpaceKeyFromDoc(doc);
381
+ if (spaceKeyHex) {
382
+ return PublicKey.from(spaceKeyHex);
383
+ }
356
384
  }
357
-
358
- const spaceKeyHex = getSpaceKeyFromDoc(doc);
359
- if (!spaceKeyHex) {
360
- return null;
385
+ /**
386
+ * Edge case on the initial space setup.
387
+ * A peer is maybe trying to share space root document with us after a successful invitation.
388
+ * We don't have a document to check access block locally, so we need to rely on external sources (space metada).
389
+ */
390
+ const rootDocSpaceKey = this._getSpaceKeyByRootDocumentId?.(documentId);
391
+ if (rootDocSpaceKey) {
392
+ return rootDocSpaceKey;
361
393
  }
362
394
 
363
- return PublicKey.from(spaceKeyHex);
395
+ return null;
364
396
  }
365
397
 
366
398
  /**
@@ -496,7 +528,10 @@ export class AutomergeHost extends Resource {
496
528
  return;
497
529
  }
498
530
 
499
- log.info('replication documents after collection sync', {
531
+ log.info('replicating documents after collection sync', {
532
+ collectionId,
533
+ peerId,
534
+ toReplicate,
500
535
  count: toReplicate.length,
501
536
  });
502
537
 
@@ -35,7 +35,7 @@ export class CollectionSynchronizer extends Resource {
35
35
 
36
36
  private readonly _connectedPeers = new Set<PeerId>();
37
37
 
38
- public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId }>();
38
+ public readonly remoteStateUpdated = new Event<{ collectionId: string; peerId: PeerId; newDocsAppeared: boolean }>();
39
39
 
40
40
  constructor(params: CollectionSynchronizerParams) {
41
41
  super();
@@ -121,7 +121,7 @@ export class CollectionSynchronizer extends Resource {
121
121
  return;
122
122
  }
123
123
  for (const [collectionId, state] of this._perCollectionStates.entries()) {
124
- if (this._shouldSyncCollection(collectionId, peerId)) {
124
+ if (this._activeCollections.has(collectionId) && this._shouldSyncCollection(collectionId, peerId)) {
125
125
  state.interestedPeers.add(peerId);
126
126
  state.lastQueried.set(peerId, Date.now());
127
127
  this._queryCollectionState(collectionId, peerId);
@@ -159,8 +159,12 @@ export class CollectionSynchronizer extends Resource {
159
159
  log('onRemoteStateReceived', { collectionId, peerId, state });
160
160
  validateCollectionState(state);
161
161
  const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
162
- perCollectionState.remoteStates.set(peerId, state);
163
- this.remoteStateUpdated.emit({ peerId, collectionId });
162
+ const existingState = perCollectionState.remoteStates.get(peerId) ?? { documents: {} };
163
+ const diff = diffCollectionState(existingState, state);
164
+ if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
165
+ perCollectionState.remoteStates.set(peerId, state);
166
+ this.remoteStateUpdated.emit({ peerId, collectionId, newDocsAppeared: diff.missingOnLocal.length > 0 });
167
+ }
164
168
  }
165
169
 
166
170
  private _getOrCreatePerCollectionState(collectionId: string): PerCollectionState {
@@ -246,6 +246,13 @@ export class EchoNetworkAdapter extends NetworkAdapter {
246
246
  this._params.monitor?.recordMessageReceived(message);
247
247
  }
248
248
 
249
+ public onConnectionAuthScopeChanged(peer: PeerId) {
250
+ const entry = this._connections.get(peer);
251
+ if (entry) {
252
+ this._onConnectionAuthScopeChanged(entry.connection);
253
+ }
254
+ }
255
+
249
256
  /**
250
257
  * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
251
258
  * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
@@ -88,10 +88,10 @@ export class MeshEchoReplicator implements EchoReplicator {
88
88
  documentId: params.documentId,
89
89
  peerId: connection.peerId,
90
90
  });
91
- log('document not found locally for share policy check, accepting the remote document', {
91
+ log('document not found locally for share policy check', {
92
92
  peerId: connection.peerId,
93
93
  documentId: params.documentId,
94
- remoteDocumentExists,
94
+ acceptDocument: remoteDocumentExists,
95
95
  });
96
96
  // If a document is not present locally return true if another peer claims to have it.
97
97
  // Simply returning true will add the peer to "generous peers list" for this document which will
@@ -33,6 +33,7 @@ import {
33
33
  type EchoReplicator,
34
34
  type EchoDataStats,
35
35
  type PeerIdProvider,
36
+ type RootDocumentSpaceKeyProvider,
36
37
  } from '../automerge';
37
38
 
38
39
  const INDEXER_CONFIG: IndexConfig = {
@@ -43,6 +44,7 @@ const INDEXER_CONFIG: IndexConfig = {
43
44
  export type EchoHostParams = {
44
45
  kv: LevelDB;
45
46
  peerIdProvider?: PeerIdProvider;
47
+ getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
46
48
  };
47
49
 
48
50
  /**
@@ -60,7 +62,7 @@ export class EchoHost extends Resource {
60
62
  private readonly _spaceStateManager = new SpaceStateManager();
61
63
  private readonly _echoDataMonitor: EchoDataMonitor;
62
64
 
63
- constructor({ kv, peerIdProvider }: EchoHostParams) {
65
+ constructor({ kv, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
64
66
  super();
65
67
 
66
68
  this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
@@ -72,6 +74,7 @@ export class EchoHost extends Resource {
72
74
  dataMonitor: this._echoDataMonitor,
73
75
  indexMetadataStore: this._indexMetadataStore,
74
76
  peerIdProvider,
77
+ getSpaceKeyByRootDocumentId,
75
78
  });
76
79
 
77
80
  this._indexer = new Indexer({
@@ -3,7 +3,6 @@
3
3
  //
4
4
 
5
5
  import { Mutex, scheduleTask, scheduleMicroTask } from '@dxos/async';
6
- import * as A from '@dxos/automerge/automerge';
7
6
  import { cbor } from '@dxos/automerge/automerge-repo';
8
7
  import { Context, Resource } from '@dxos/context';
9
8
  import { randomUUID } from '@dxos/crypto';
@@ -20,6 +19,7 @@ import {
20
19
  } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
21
20
  import { bufferToArray } from '@dxos/util';
22
21
 
22
+ import { InflightRequestLimiter } from './inflight-request-limiter';
23
23
  import {
24
24
  getSpaceIdFromCollectionId,
25
25
  type EchoReplicator,
@@ -184,9 +184,12 @@ type EdgeReplicatorConnectionsParams = {
184
184
  onRestartRequested: () => Promise<void>;
185
185
  };
186
186
 
187
+ const MAX_INFLIGHT_REQUESTS = 5;
188
+ const MAX_RATE_LIMIT_WAIT_TIME_MS = 3000;
189
+
187
190
  class EdgeReplicatorConnection extends Resource implements ReplicatorConnection {
188
191
  private readonly _edgeConnection: EdgeConnection;
189
- private _remotePeerId: string | null = null;
192
+ private readonly _remotePeerId: string | null = null;
190
193
  private readonly _targetServiceId: string;
191
194
  private readonly _spaceId: SpaceId;
192
195
  private readonly _context: EchoReplicatorContext;
@@ -195,6 +198,11 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
195
198
  private readonly _onRemoteDisconnected: () => Promise<void>;
196
199
  private readonly _onRestartRequested: () => void;
197
200
 
201
+ private _requestLimiter = new InflightRequestLimiter({
202
+ maxInflightRequests: MAX_INFLIGHT_REQUESTS,
203
+ resetBalanceTimeoutMs: MAX_RATE_LIMIT_WAIT_TIME_MS,
204
+ });
205
+
198
206
  private _readableStreamController!: ReadableStreamDefaultController<AutomergeProtocolMessage>;
199
207
 
200
208
  public readable: ReadableStream<AutomergeProtocolMessage>;
@@ -213,7 +221,6 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
213
221
  this._edgeConnection = edgeConnection;
214
222
  this._spaceId = spaceId;
215
223
  this._context = context;
216
-
217
224
  // Generate a unique peer id for every connection.
218
225
  // This way automerge-repo will have separate sync states for every connection.
219
226
  // This is important because the previous connection might have had some messages that failed to deliver
@@ -233,6 +240,8 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
233
240
 
234
241
  this.writable = new WritableStream<AutomergeProtocolMessage>({
235
242
  write: async (message: AutomergeProtocolMessage, controller) => {
243
+ await this._requestLimiter.rateLimit(message);
244
+
236
245
  await this._sendMessage(message);
237
246
  },
238
247
  });
@@ -240,6 +249,9 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
240
249
 
241
250
  protected override async _open(ctx: Context): Promise<void> {
242
251
  log('open');
252
+
253
+ await this._requestLimiter.open();
254
+
243
255
  // TODO: handle reconnects
244
256
  this._ctx.onDispose(
245
257
  this._edgeConnection.onMessage((msg: RouterMessage) => {
@@ -253,6 +265,9 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
253
265
  protected override async _close(): Promise<void> {
254
266
  log('close');
255
267
  this._readableStreamController.close();
268
+
269
+ await this._requestLimiter.close();
270
+
256
271
  await this._onRemoteDisconnected();
257
272
  }
258
273
 
@@ -272,9 +287,10 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
272
287
  peerId: this._remotePeerId as PeerId,
273
288
  });
274
289
 
275
- log.info('document not found locally for share policy check, accepting the remote document', {
290
+ log.verbose('edge-replicator document not found locally for share policy check', {
276
291
  documentId: params.documentId,
277
- remoteDocumentExists,
292
+ acceptDocument: remoteDocumentExists,
293
+ remoteId: this._remotePeerId,
278
294
  });
279
295
 
280
296
  // If a document is not present locally return true only if it already exists on edge.
@@ -290,7 +306,8 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
290
306
  return true;
291
307
  }
292
308
  const spaceId = getSpaceIdFromCollectionId(params.collectionId as CollectionId);
293
- return spaceId === this._spaceId;
309
+ // Only sync collections of form space:id:rootDoc, edge ignores legacy space:id collections
310
+ return spaceId === this._spaceId && params.collectionId.split(':').length === 3;
294
311
  }
295
312
 
296
313
  private _onMessage(message: RouterMessage) {
@@ -299,15 +316,12 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
299
316
  }
300
317
 
301
318
  const payload = cbor.decode(message.payload!.value) as AutomergeProtocolMessage;
302
- log('recv', () => {
303
- const decodedData =
304
- payload.type === 'sync' && payload.data
305
- ? A.decodeSyncMessage(payload.data)
306
- : payload.type === 'collection-state'
307
- ? (payload as any).state
308
- : payload;
309
- return { from: message.serviceId, type: payload.type, decodedData };
319
+ log.verbose('edge replicator receive', {
320
+ type: payload.type,
321
+ documentId: payload.type === 'sync' && payload.documentId,
322
+ remoteId: this._remotePeerId,
310
323
  });
324
+
311
325
  // Fix the peer id.
312
326
  payload.senderId = this._remotePeerId! as PeerId;
313
327
  this._processMessage(payload);
@@ -322,6 +336,8 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
322
336
  return;
323
337
  }
324
338
 
339
+ this._requestLimiter.handleResponse(message);
340
+
325
341
  this._readableStreamController.enqueue(message);
326
342
  }
327
343
 
@@ -329,12 +345,12 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
329
345
  // Fix the peer id.
330
346
  (message as any).targetId = this._targetServiceId as PeerId;
331
347
 
332
- log('send', {
348
+ log.verbose('edge replicator send', {
333
349
  type: message.type,
334
- senderId: message.senderId,
335
- targetId: (message as any).targetId,
336
- documentId: (message as any).documentId,
350
+ documentId: message.type === 'sync' && message.documentId,
351
+ remoteId: this._remotePeerId,
337
352
  });
353
+
338
354
  const encoded = cbor.encode(message);
339
355
 
340
356
  await this._edgeConnection.send(
@@ -0,0 +1,69 @@
1
+ //
2
+ // Copyright 2025 DXOS.org
3
+ //
4
+
5
+ import { Trigger } from '@dxos/async';
6
+ import { Resource } from '@dxos/context';
7
+ import { log } from '@dxos/log';
8
+ import { type AutomergeProtocolMessage } from '@dxos/protocols';
9
+
10
+ type InflightRequestLimiterConfig = {
11
+ maxInflightRequests: number;
12
+ resetBalanceTimeoutMs: number;
13
+ };
14
+
15
+ export class InflightRequestLimiter extends Resource {
16
+ /**
17
+ * Decrement when we receive a sync message, increment when we send one.
18
+ * Can't exceed _config.maxInflightRequests.
19
+ * Resets after timeout to avoid replicator being stuck.
20
+ */
21
+ private _inflightRequestBalance = 0;
22
+ private _requestBarrier = new Trigger();
23
+ private _resetBalanceTimeout: NodeJS.Timeout | undefined;
24
+
25
+ constructor(private readonly _config: InflightRequestLimiterConfig) {
26
+ super();
27
+ }
28
+
29
+ protected override async _open() {
30
+ this._inflightRequestBalance = 0;
31
+ this._requestBarrier.reset();
32
+ this._requestBarrier.wake();
33
+ }
34
+
35
+ protected override async _close() {
36
+ this._inflightRequestBalance = 0;
37
+ this._requestBarrier.throw(new Error('Rate limiter closed.'));
38
+ clearTimeout(this._resetBalanceTimeout);
39
+ }
40
+
41
+ public async rateLimit(message: AutomergeProtocolMessage) {
42
+ if (message.type !== 'sync') {
43
+ return;
44
+ }
45
+ while (this._inflightRequestBalance >= this._config.maxInflightRequests) {
46
+ await this._requestBarrier.wait();
47
+ }
48
+ this._inflightRequestBalance++;
49
+ if (this._inflightRequestBalance === this._config.maxInflightRequests) {
50
+ this._requestBarrier.reset();
51
+ this._resetBalanceTimeout = setTimeout(() => {
52
+ log.warn('Request balance has not changed during specified timeout, resetting request limiter.');
53
+ this._inflightRequestBalance = 0;
54
+ this._requestBarrier.wake();
55
+ }, this._config.resetBalanceTimeoutMs);
56
+ }
57
+ }
58
+
59
+ public handleResponse(message: AutomergeProtocolMessage) {
60
+ if (message.type !== 'sync') {
61
+ return;
62
+ }
63
+ this._inflightRequestBalance--;
64
+ if (this._inflightRequestBalance + 1 === this._config.maxInflightRequests) {
65
+ this._requestBarrier.wake();
66
+ clearInterval(this._resetBalanceTimeout);
67
+ }
68
+ }
69
+ }
@@ -279,6 +279,11 @@ export class Pipeline implements PipelineAccessor {
279
279
  );
280
280
  }
281
281
 
282
+ retryMessage(message: FeedMessageBlock) {
283
+ invariant(this._feedSetIterator, 'Iterator not initialized.');
284
+ this._feedSetIterator.reiterateBlock(message);
285
+ }
286
+
282
287
  @synchronized
283
288
  async start() {
284
289
  invariant(!this._isStarted, 'Pipeline is already started.');
@@ -423,7 +428,10 @@ export class Pipeline implements PipelineAccessor {
423
428
  });
424
429
 
425
430
  this._feedSetIterator.stalled.on((iterator) => {
426
- log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`);
431
+ log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {
432
+ currentTimeframe: this._timeframeClock.timeframe,
433
+ targetTimeframe: this._state.targetTimeframe,
434
+ });
427
435
  this._state.stalled.emit();
428
436
  });
429
437