@dxos/echo-pipeline 0.6.5 → 0.6.6-staging.23d123d

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/lib/browser/{chunk-2MII6KJX.mjs → chunk-P6XSIJKM.mjs} +2184 -2132
  2. package/dist/lib/browser/chunk-P6XSIJKM.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +1 -1
  4. package/dist/lib/browser/meta.json +1 -1
  5. package/dist/lib/browser/testing/index.mjs +8 -8
  6. package/dist/lib/browser/testing/index.mjs.map +3 -3
  7. package/dist/lib/node/{chunk-6MWU4MHX.cjs → chunk-IYTGTZ7D.cjs} +2177 -2125
  8. package/dist/lib/node/chunk-IYTGTZ7D.cjs.map +7 -0
  9. package/dist/lib/node/index.cjs +35 -35
  10. package/dist/lib/node/index.cjs.map +1 -1
  11. package/dist/lib/node/meta.json +1 -1
  12. package/dist/lib/node/testing/index.cjs +18 -18
  13. package/dist/lib/node/testing/index.cjs.map +3 -3
  14. package/dist/types/src/automerge/automerge-host.d.ts +1 -0
  15. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  16. package/dist/types/src/automerge/echo-network-adapter.d.ts +2 -1
  17. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  18. package/dist/types/src/automerge/echo-replicator.d.ts +10 -1
  19. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  20. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  21. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  22. package/dist/types/src/automerge/network-protocol.d.ts +3 -28
  23. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  24. package/dist/types/src/space/space-manager.d.ts +3 -1
  25. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  26. package/dist/types/src/space/space-protocol.d.ts +10 -3
  27. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  28. package/dist/types/src/space/space.d.ts.map +1 -1
  29. package/dist/types/src/testing/test-network-adapter.d.ts +2 -1
  30. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  31. package/package.json +33 -33
  32. package/src/automerge/automerge-host.ts +13 -1
  33. package/src/automerge/automerge-repo.test.ts +380 -366
  34. package/src/automerge/echo-network-adapter.test.ts +1 -0
  35. package/src/automerge/echo-network-adapter.ts +8 -0
  36. package/src/automerge/echo-replicator.ts +11 -1
  37. package/src/automerge/mesh-echo-replicator-connection.ts +18 -0
  38. package/src/automerge/mesh-echo-replicator.ts +10 -2
  39. package/src/automerge/network-protocol.ts +8 -34
  40. package/src/space/space-manager.ts +15 -2
  41. package/src/space/space-protocol.ts +42 -5
  42. package/src/space/space.ts +4 -4
  43. package/src/testing/test-network-adapter.ts +5 -3
  44. package/dist/lib/browser/chunk-2MII6KJX.mjs.map +0 -7
  45. package/dist/lib/node/chunk-6MWU4MHX.cjs.map +0 -7
@@ -26,8 +26,8 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
26
26
  mod
27
27
  ));
28
28
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
- var chunk_6MWU4MHX_exports = {};
30
- __export(chunk_6MWU4MHX_exports, {
29
+ var chunk_IYTGTZ7D_exports = {};
30
+ __export(chunk_IYTGTZ7D_exports, {
31
31
  AuthExtension: () => AuthExtension,
32
32
  AuthStatus: () => AuthStatus,
33
33
  AutomergeHost: () => AutomergeHost,
@@ -63,7 +63,7 @@ __export(chunk_6MWU4MHX_exports, {
63
63
  startAfter: () => startAfter,
64
64
  valueEncoding: () => valueEncoding
65
65
  });
66
- module.exports = __toCommonJS(chunk_6MWU4MHX_exports);
66
+ module.exports = __toCommonJS(chunk_IYTGTZ7D_exports);
67
67
  var import_hypercore = require("@dxos/hypercore");
68
68
  var import_protocols = require("@dxos/protocols");
69
69
  var import_invariant = require("@dxos/invariant");
@@ -82,97 +82,99 @@ var import_async2 = require("@dxos/async");
82
82
  var import_automerge2 = require("@dxos/automerge/automerge");
83
83
  var import_context3 = require("@dxos/context");
84
84
  var import_util = require("@dxos/util");
85
- var import_context4 = require("@dxos/context");
86
85
  var import_async3 = require("@dxos/async");
87
- var import_automerge3 = require("@dxos/automerge/automerge");
88
- var import_automerge_repo = require("@dxos/automerge/automerge-repo");
89
- var import_context5 = require("@dxos/context");
86
+ var import_context4 = require("@dxos/context");
87
+ var import_crypto2 = require("@dxos/crypto");
90
88
  var import_invariant3 = require("@dxos/invariant");
91
- var import_keys2 = require("@dxos/keys");
92
89
  var import_log2 = require("@dxos/log");
93
90
  var import_protocols4 = require("@dxos/protocols");
94
- var import_tracing = require("@dxos/tracing");
95
- var import_async4 = require("@dxos/async");
96
- var import_automerge_repo2 = require("@dxos/automerge/automerge-repo");
97
- var import_context6 = require("@dxos/context");
98
- var import_invariant4 = require("@dxos/invariant");
99
- var import_log3 = require("@dxos/log");
100
- var import_util2 = require("@dxos/util");
101
- var import_indexing = require("@dxos/indexing");
102
- var import_invariant5 = require("@dxos/invariant");
103
- var import_keys3 = require("@dxos/keys");
104
- var import_async5 = require("@dxos/async");
105
- var import_context7 = require("@dxos/context");
106
- var import_crypto2 = require("@dxos/crypto");
107
- var import_invariant6 = require("@dxos/invariant");
108
- var import_log4 = require("@dxos/log");
109
- var import_protocols5 = require("@dxos/protocols");
110
91
  var import_teleport = require("@dxos/teleport");
111
- var import_async6 = require("@dxos/async");
92
+ var import_async4 = require("@dxos/async");
112
93
  var import_debug = require("@dxos/debug");
113
- var import_log5 = require("@dxos/log");
94
+ var import_log3 = require("@dxos/log");
114
95
  var import_timeframe = require("@dxos/timeframe");
115
- var import_async7 = require("@dxos/async");
116
- var import_context8 = require("@dxos/context");
96
+ var import_async5 = require("@dxos/async");
97
+ var import_context5 = require("@dxos/context");
117
98
  var import_debug2 = require("@dxos/debug");
118
99
  var import_feed_store = require("@dxos/feed-store");
119
- var import_invariant7 = require("@dxos/invariant");
120
- var import_keys4 = require("@dxos/keys");
121
- var import_log6 = require("@dxos/log");
100
+ var import_invariant4 = require("@dxos/invariant");
101
+ var import_keys2 = require("@dxos/keys");
102
+ var import_log4 = require("@dxos/log");
122
103
  var import_timeframe2 = require("@dxos/timeframe");
123
- var import_util3 = require("@dxos/util");
124
- var import_invariant8 = require("@dxos/invariant");
125
- var import_log7 = require("@dxos/log");
126
- var import_async8 = require("@dxos/async");
127
- var import_context9 = require("@dxos/context");
104
+ var import_util2 = require("@dxos/util");
105
+ var import_invariant5 = require("@dxos/invariant");
106
+ var import_log5 = require("@dxos/log");
107
+ var import_async6 = require("@dxos/async");
108
+ var import_context6 = require("@dxos/context");
128
109
  var import_crypto3 = require("@dxos/crypto");
129
- var import_invariant9 = require("@dxos/invariant");
130
- var import_keys5 = require("@dxos/keys");
131
- var import_log8 = require("@dxos/log");
110
+ var import_invariant6 = require("@dxos/invariant");
111
+ var import_keys3 = require("@dxos/keys");
112
+ var import_log6 = require("@dxos/log");
132
113
  var import_credentials = require("@dxos/protocols/proto/dxos/halo/credentials");
133
- var import_tracing2 = require("@dxos/tracing");
134
- var import_util4 = require("@dxos/util");
135
- var import_async9 = require("@dxos/async");
136
- var import_context10 = require("@dxos/context");
114
+ var import_tracing = require("@dxos/tracing");
115
+ var import_util3 = require("@dxos/util");
116
+ var import_async7 = require("@dxos/async");
117
+ var import_context7 = require("@dxos/context");
137
118
  var import_credentials2 = require("@dxos/credentials");
138
- var import_keys6 = require("@dxos/keys");
139
- var import_log9 = require("@dxos/log");
119
+ var import_keys4 = require("@dxos/keys");
120
+ var import_log7 = require("@dxos/log");
140
121
  var import_credentials3 = require("@dxos/protocols/proto/dxos/halo/credentials");
141
122
  var import_timeframe3 = require("@dxos/timeframe");
142
- var import_tracing3 = require("@dxos/tracing");
143
- var import_util5 = require("@dxos/util");
144
- var import_async10 = require("@dxos/async");
145
- var import_context11 = require("@dxos/context");
146
- var import_protocols6 = require("@dxos/protocols");
123
+ var import_tracing2 = require("@dxos/tracing");
124
+ var import_util4 = require("@dxos/util");
125
+ var import_async8 = require("@dxos/async");
126
+ var import_context8 = require("@dxos/context");
127
+ var import_protocols5 = require("@dxos/protocols");
147
128
  var import_teleport2 = require("@dxos/teleport");
148
129
  var import_crypto4 = require("@dxos/crypto");
149
- var import_keys7 = require("@dxos/keys");
150
- var import_log10 = require("@dxos/log");
130
+ var import_keys5 = require("@dxos/keys");
131
+ var import_log8 = require("@dxos/log");
151
132
  var import_network_manager = require("@dxos/network-manager");
152
133
  var import_teleport3 = require("@dxos/teleport");
153
134
  var import_teleport_extension_object_sync = require("@dxos/teleport-extension-object-sync");
154
135
  var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
155
- var import_tracing4 = require("@dxos/tracing");
156
- var import_util6 = require("@dxos/util");
157
- var import_async11 = require("@dxos/async");
136
+ var import_tracing3 = require("@dxos/tracing");
137
+ var import_util5 = require("@dxos/util");
138
+ var import_async9 = require("@dxos/async");
158
139
  var import_debug3 = require("@dxos/debug");
140
+ var import_keys6 = require("@dxos/keys");
141
+ var import_log9 = require("@dxos/log");
142
+ var import_protocols6 = require("@dxos/protocols");
143
+ var import_util6 = require("@dxos/util");
144
+ var import_codec_protobuf = require("@dxos/codec-protobuf");
145
+ var import_invariant7 = require("@dxos/invariant");
146
+ var import_keys7 = require("@dxos/keys");
147
+ var import_log10 = require("@dxos/log");
148
+ var import_async10 = require("@dxos/async");
149
+ var import_automerge3 = require("@dxos/automerge/automerge");
150
+ var import_automerge_repo = require("@dxos/automerge/automerge-repo");
151
+ var import_context9 = require("@dxos/context");
152
+ var import_invariant8 = require("@dxos/invariant");
159
153
  var import_keys8 = require("@dxos/keys");
160
154
  var import_log11 = require("@dxos/log");
161
155
  var import_protocols7 = require("@dxos/protocols");
156
+ var import_tracing4 = require("@dxos/tracing");
157
+ var import_async11 = require("@dxos/async");
158
+ var import_automerge_repo2 = require("@dxos/automerge/automerge-repo");
159
+ var import_context10 = require("@dxos/context");
160
+ var import_invariant9 = require("@dxos/invariant");
161
+ var import_log12 = require("@dxos/log");
162
162
  var import_util7 = require("@dxos/util");
163
- var import_codec_protobuf = require("@dxos/codec-protobuf");
163
+ var import_protocols8 = require("@dxos/protocols");
164
+ var import_indexing = require("@dxos/indexing");
165
+ var import_context11 = require("@dxos/context");
164
166
  var import_invariant10 = require("@dxos/invariant");
165
167
  var import_keys9 = require("@dxos/keys");
166
- var import_log12 = require("@dxos/log");
167
- var import_invariant11 = require("@dxos/invariant");
168
- var import_keys10 = require("@dxos/keys");
169
168
  var import_log13 = require("@dxos/log");
170
169
  var import_util8 = require("@dxos/util");
170
+ var A2 = __toESM(require("@dxos/automerge/automerge"));
171
171
  var import_automerge_repo3 = require("@dxos/automerge/automerge-repo");
172
172
  var import_context12 = require("@dxos/context");
173
- var import_invariant12 = require("@dxos/invariant");
173
+ var import_invariant11 = require("@dxos/invariant");
174
174
  var import_log14 = require("@dxos/log");
175
175
  var import_teleport_extension_automerge_replicator = require("@dxos/teleport-extension-automerge-replicator");
176
+ var import_invariant12 = require("@dxos/invariant");
177
+ var import_keys10 = require("@dxos/keys");
176
178
  var import_tracing5 = require("@dxos/tracing");
177
179
  var import_util9 = require("@dxos/util");
178
180
  var import_crc_32 = __toESM(require("crc-32"));
@@ -181,7 +183,7 @@ var import_context13 = require("@dxos/context");
181
183
  var import_invariant13 = require("@dxos/invariant");
182
184
  var import_keys11 = require("@dxos/keys");
183
185
  var import_log15 = require("@dxos/log");
184
- var import_protocols8 = require("@dxos/protocols");
186
+ var import_protocols9 = require("@dxos/protocols");
185
187
  var import_services = require("@dxos/protocols/proto/dxos/client/services");
186
188
  var import_util10 = require("@dxos/util");
187
189
  var codec = import_protocols.schema.getCodecForType("dxos.echo.feed.FeedMessage");
@@ -578,113 +580,107 @@ var diffCollectionState = (local, remote) => {
578
580
  different
579
581
  };
580
582
  };
581
- var LevelDBStorageAdapter = class extends import_context4.Resource {
582
- constructor(_params) {
583
- super();
584
- this._params = _params;
585
- }
586
- async load(keyArray) {
587
- try {
588
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
589
- return void 0;
590
- }
591
- const startMs = Date.now();
592
- const chunk = await this._params.db.get(keyArray, {
593
- ...encodingOptions
594
- });
595
- this._params.monitor?.recordBytesLoaded(chunk.byteLength);
596
- this._params.monitor?.recordLoadDuration(Date.now() - startMs);
597
- return chunk;
598
- } catch (err) {
599
- if (isLevelDbNotFoundError(err)) {
600
- return void 0;
601
- }
602
- throw err;
603
- }
604
- }
605
- async save(keyArray, binary) {
606
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
607
- return void 0;
608
- }
609
- const startMs = Date.now();
610
- const batch = this._params.db.batch();
611
- await this._params.callbacks?.beforeSave?.({
612
- path: keyArray,
613
- batch
583
+ var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/auth.ts";
584
+ var AuthExtension = class extends import_teleport.RpcExtension {
585
+ constructor(_authParams) {
586
+ super({
587
+ requested: {
588
+ AuthService: import_protocols4.schema.getService("dxos.mesh.teleport.auth.AuthService")
589
+ },
590
+ exposed: {
591
+ AuthService: import_protocols4.schema.getService("dxos.mesh.teleport.auth.AuthService")
592
+ },
593
+ timeout: 60 * 1e3
614
594
  });
615
- batch.put(keyArray, Buffer.from(binary), {
616
- ...encodingOptions
595
+ this._authParams = _authParams;
596
+ this._ctx = new import_context4.Context({
597
+ onError: (err) => {
598
+ import_log2.log.catch(err, void 0, {
599
+ F: __dxlog_file3,
600
+ L: 28,
601
+ S: this,
602
+ C: (f, a) => f(...a)
603
+ });
604
+ }
605
+ }, {
606
+ F: __dxlog_file3,
607
+ L: 26
617
608
  });
618
- await batch.write();
619
- this._params.monitor?.recordBytesStored(binary.byteLength);
620
- await this._params.callbacks?.afterSave?.(keyArray);
621
- this._params.monitor?.recordStoreDuration(Date.now() - startMs);
622
609
  }
623
- async remove(keyArray) {
624
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
625
- return void 0;
626
- }
627
- await this._params.db.del(keyArray, {
628
- ...encodingOptions
610
+ async getHandlers() {
611
+ return {
612
+ AuthService: {
613
+ authenticate: async ({ challenge }) => {
614
+ try {
615
+ const credential = await this._authParams.provider(challenge);
616
+ if (!credential) {
617
+ throw new Error("auth rejected");
618
+ }
619
+ return {
620
+ credential
621
+ };
622
+ } catch (err) {
623
+ import_log2.log.error("failed to generate auth credentials", err, {
624
+ F: __dxlog_file3,
625
+ L: 55,
626
+ S: this,
627
+ C: (f, a) => f(...a)
628
+ });
629
+ throw new Error("auth rejected");
630
+ }
631
+ }
632
+ }
633
+ };
634
+ }
635
+ async onOpen(context) {
636
+ await super.onOpen(context);
637
+ (0, import_async3.scheduleTask)(this._ctx, async () => {
638
+ try {
639
+ const challenge = (0, import_crypto2.randomBytes)(32);
640
+ const { credential } = await this.rpc.AuthService.authenticate({
641
+ challenge
642
+ });
643
+ (0, import_invariant3.invariant)(credential?.length > 0, "invalid credential", {
644
+ F: __dxlog_file3,
645
+ L: 69,
646
+ S: this,
647
+ A: [
648
+ "credential?.length > 0",
649
+ "'invalid credential'"
650
+ ]
651
+ });
652
+ const success = await this._authParams.verifier(challenge, credential);
653
+ (0, import_invariant3.invariant)(success, "credential not verified", {
654
+ F: __dxlog_file3,
655
+ L: 71,
656
+ S: this,
657
+ A: [
658
+ "success",
659
+ "'credential not verified'"
660
+ ]
661
+ });
662
+ (0, import_async3.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
663
+ } catch (err) {
664
+ (0, import_log2.log)("auth failed", err, {
665
+ F: __dxlog_file3,
666
+ L: 74,
667
+ S: this,
668
+ C: (f, a) => f(...a)
669
+ });
670
+ this.close();
671
+ this._authParams.onAuthFailure();
672
+ }
629
673
  });
630
674
  }
631
- async loadRange(keyPrefix) {
632
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
633
- return [];
634
- }
635
- const startMs = Date.now();
636
- const result = [];
637
- for await (const [key, value] of this._params.db.iterator({
638
- gte: keyPrefix,
639
- lte: [
640
- ...keyPrefix,
641
- "\uFFFF"
642
- ],
643
- ...encodingOptions
644
- })) {
645
- result.push({
646
- key,
647
- data: value
648
- });
649
- this._params.monitor?.recordBytesLoaded(value.byteLength);
650
- }
651
- this._params.monitor?.recordLoadDuration(Date.now() - startMs);
652
- return result;
675
+ async onClose() {
676
+ await this._ctx.dispose();
677
+ await super.onClose();
653
678
  }
654
- async removeRange(keyPrefix) {
655
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
656
- return void 0;
657
- }
658
- const batch = this._params.db.batch();
659
- for await (const [key] of this._params.db.iterator({
660
- gte: keyPrefix,
661
- lte: [
662
- ...keyPrefix,
663
- "\uFFFF"
664
- ],
665
- ...encodingOptions
666
- })) {
667
- batch.del(key, {
668
- ...encodingOptions
669
- });
670
- }
671
- await batch.write();
679
+ async onAbort() {
680
+ await this._ctx.dispose();
681
+ await super.onAbort();
672
682
  }
673
683
  };
674
- var keyEncoder = {
675
- encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
676
- decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
677
- format: "buffer"
678
- };
679
- var encodingOptions = {
680
- keyEncoding: keyEncoder,
681
- valueEncoding: "buffer"
682
- };
683
- var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
684
- var MESSAGE_TYPE_COLLECTION_QUERY = "collection-query";
685
- var isCollectionQueryMessage = (message) => message.type === MESSAGE_TYPE_COLLECTION_QUERY;
686
- var MESSAGE_TYPE_COLLECTION_STATE = "collection-state";
687
- var isCollectionStateMessage = (message) => message.type === MESSAGE_TYPE_COLLECTION_STATE;
688
684
  function _ts_decorate(decorators, target, key, desc) {
689
685
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
690
686
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -695,1035 +691,1083 @@ function _ts_decorate(decorators, target, key, desc) {
695
691
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
696
692
  return c > 3 && r && Object.defineProperty(target, key, r), r;
697
693
  }
698
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
699
- var EchoNetworkAdapter = class extends import_automerge_repo2.NetworkAdapter {
700
- constructor(_params) {
701
- super();
702
- this._params = _params;
703
- this._replicators = /* @__PURE__ */ new Set();
704
- this._connections = /* @__PURE__ */ new Map();
705
- this._lifecycleState = import_context6.LifecycleState.CLOSED;
706
- this._connected = new import_async4.Trigger();
694
+ var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
695
+ var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
696
+ feedKey,
697
+ index
698
+ }));
699
+ var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
700
+ feedKey,
701
+ index
702
+ ]));
703
+ var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
704
+ feedKey,
705
+ index: index + 1
706
+ }));
707
+ var TimeframeClock = class {
708
+ constructor(_timeframe = new import_timeframe.Timeframe()) {
709
+ this._timeframe = _timeframe;
710
+ this.update = new import_async4.Event();
711
+ this._pendingTimeframe = _timeframe;
707
712
  }
708
- connect(peerId, peerMetadata) {
709
- this.peerId = peerId;
710
- this.peerMetadata = peerMetadata;
711
- this._connected.wake();
713
+ /**
714
+ * Timeframe that was processed by ECHO.
715
+ */
716
+ get timeframe() {
717
+ return this._timeframe;
712
718
  }
713
- send(message) {
714
- this._send(message);
719
+ /**
720
+ * Timeframe that is currently being processed by ECHO.
721
+ * Will be equal to `timeframe` after the processing is complete.
722
+ */
723
+ get pendingTimeframe() {
724
+ return this._pendingTimeframe;
715
725
  }
716
- disconnect() {
726
+ setTimeframe(timeframe) {
727
+ this._timeframe = timeframe;
728
+ this._pendingTimeframe = timeframe;
729
+ this.update.emit(this._timeframe);
717
730
  }
718
- async open() {
719
- if (this._lifecycleState === import_context6.LifecycleState.OPEN) {
720
- return;
721
- }
722
- this._lifecycleState = import_context6.LifecycleState.OPEN;
723
- (0, import_log3.log)("emit ready", void 0, {
724
- F: __dxlog_file3,
725
- L: 78,
726
- S: this,
727
- C: (f, a) => f(...a)
728
- });
729
- this.emit("ready", {
730
- network: this
731
- });
731
+ updatePendingTimeframe(key, seq) {
732
+ this._pendingTimeframe = import_timeframe.Timeframe.merge(this._pendingTimeframe, new import_timeframe.Timeframe([
733
+ [
734
+ key,
735
+ seq
736
+ ]
737
+ ]));
732
738
  }
733
- async close() {
734
- if (this._lifecycleState === import_context6.LifecycleState.CLOSED) {
735
- return this;
736
- }
737
- for (const replicator of this._replicators) {
738
- await replicator.disconnect();
739
- }
740
- this._replicators.clear();
741
- this._lifecycleState = import_context6.LifecycleState.CLOSED;
739
+ updateTimeframe() {
740
+ this._timeframe = this._pendingTimeframe;
741
+ this.update.emit(this._timeframe);
742
742
  }
743
- async whenConnected() {
744
- await this._connected.wait({
745
- timeout: 1e4
746
- });
743
+ hasGaps(timeframe) {
744
+ const gaps = import_timeframe.Timeframe.dependencies(timeframe, this._timeframe);
745
+ return !gaps.isEmpty();
747
746
  }
748
- async addReplicator(replicator) {
749
- (0, import_invariant4.invariant)(this._lifecycleState === import_context6.LifecycleState.OPEN, void 0, {
750
- F: __dxlog_file3,
751
- L: 104,
752
- S: this,
753
- A: [
754
- "this._lifecycleState === LifecycleState.OPEN",
755
- ""
756
- ]
757
- });
758
- (0, import_invariant4.invariant)(this.peerId, void 0, {
759
- F: __dxlog_file3,
760
- L: 105,
761
- S: this,
762
- A: [
763
- "this.peerId",
764
- ""
765
- ]
766
- });
767
- (0, import_invariant4.invariant)(!this._replicators.has(replicator), void 0, {
768
- F: __dxlog_file3,
769
- L: 106,
747
+ async waitUntilReached(target) {
748
+ (0, import_log3.log)("waitUntilReached", {
749
+ target,
750
+ current: this._timeframe
751
+ }, {
752
+ F: __dxlog_file4,
753
+ L: 70,
770
754
  S: this,
771
- A: [
772
- "!this._replicators.has(replicator)",
773
- ""
774
- ]
755
+ C: (f, a) => f(...a)
775
756
  });
776
- this._replicators.add(replicator);
777
- await replicator.connect({
778
- peerId: this.peerId,
779
- onConnectionOpen: this._onConnectionOpen.bind(this),
780
- onConnectionClosed: this._onConnectionClosed.bind(this),
781
- onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
782
- getContainingSpaceForDocument: this._params.getContainingSpaceForDocument
757
+ await this.update.waitForCondition(() => {
758
+ (0, import_log3.log)("check if reached", {
759
+ target,
760
+ current: this._timeframe,
761
+ deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
762
+ }, {
763
+ F: __dxlog_file4,
764
+ L: 72,
765
+ S: this,
766
+ C: (f, a) => f(...a)
767
+ });
768
+ return import_timeframe.Timeframe.dependencies(target, this._timeframe).isEmpty();
783
769
  });
784
770
  }
785
- async removeReplicator(replicator) {
786
- (0, import_invariant4.invariant)(this._lifecycleState === import_context6.LifecycleState.OPEN, void 0, {
787
- F: __dxlog_file3,
788
- L: 120,
789
- S: this,
790
- A: [
791
- "this._lifecycleState === LifecycleState.OPEN",
792
- ""
793
- ]
771
+ };
772
+ _ts_decorate([
773
+ (0, import_debug.timed)(5e3)
774
+ ], TimeframeClock.prototype, "waitUntilReached", null);
775
+ var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
776
+ var createMessageSelector = (timeframeClock) => {
777
+ return (messages) => {
778
+ for (let i = 0; i < messages.length; i++) {
779
+ const { data: { timeframe } } = messages[i];
780
+ (0, import_invariant5.invariant)(timeframe, void 0, {
781
+ F: __dxlog_file5,
782
+ L: 25,
783
+ S: void 0,
784
+ A: [
785
+ "timeframe",
786
+ ""
787
+ ]
788
+ });
789
+ if (!timeframeClock.hasGaps(timeframe)) {
790
+ return i;
791
+ }
792
+ }
793
+ (0, import_log5.log)("Skipping...", void 0, {
794
+ F: __dxlog_file5,
795
+ L: 33,
796
+ S: void 0,
797
+ C: (f, a) => f(...a)
794
798
  });
795
- (0, import_invariant4.invariant)(this._replicators.has(replicator), void 0, {
796
- F: __dxlog_file3,
797
- L: 121,
798
- S: this,
799
- A: [
800
- "this._replicators.has(replicator)",
801
- ""
802
- ]
799
+ };
800
+ };
801
+ function _ts_decorate2(decorators, target, key, desc) {
802
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
803
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
804
+ r = Reflect.decorate(decorators, target, key, desc);
805
+ else
806
+ for (var i = decorators.length - 1; i >= 0; i--)
807
+ if (d = decorators[i])
808
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
809
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
810
+ }
811
+ var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
812
+ var PipelineState = class {
813
+ constructor(_feeds, _timeframeClock) {
814
+ this._feeds = _feeds;
815
+ this._timeframeClock = _timeframeClock;
816
+ this._ctx = new import_context5.Context(void 0, {
817
+ F: __dxlog_file6,
818
+ L: 41
803
819
  });
804
- await replicator.disconnect();
805
- this._replicators.delete(replicator);
820
+ this.timeframeUpdate = this._timeframeClock.update;
821
+ this.stalled = new import_async5.Event();
822
+ this._startTimeframe = new import_timeframe2.Timeframe();
823
+ this._reachedTarget = false;
806
824
  }
807
- async shouldAdvertise(peerId, params) {
808
- const connection = this._connections.get(peerId);
809
- if (!connection) {
810
- return false;
811
- }
812
- return connection.connection.shouldAdvertise(params);
825
+ /**
826
+ * Latest theoretical timeframe based on the last mutation in each feed.
827
+ * NOTE: This might never be reached if the mutation dependencies
828
+ */
829
+ // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
830
+ get endTimeframe() {
831
+ return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
832
+ feedKey: feed.key,
833
+ index: feed.length - 1
834
+ })));
813
835
  }
814
- shouldSyncCollection(peerId, params) {
815
- const connection = this._connections.get(peerId);
816
- if (!connection) {
817
- return false;
818
- }
819
- return connection.connection.shouldSyncCollection(params);
836
+ get startTimeframe() {
837
+ return this._startTimeframe;
820
838
  }
821
- queryCollectionState(collectionId, targetId) {
822
- const message = {
823
- type: "collection-query",
824
- senderId: this.peerId,
825
- targetId,
826
- collectionId
827
- };
828
- this._send(message);
839
+ get timeframe() {
840
+ return this._timeframeClock.timeframe;
829
841
  }
830
- sendCollectionState(collectionId, targetId, state) {
831
- const message = {
832
- type: "collection-state",
833
- senderId: this.peerId,
834
- targetId,
835
- collectionId,
836
- state
837
- };
838
- this._send(message);
842
+ get pendingTimeframe() {
843
+ return this._timeframeClock.pendingTimeframe;
839
844
  }
840
- _send(message) {
841
- const connectionEntry = this._connections.get(message.targetId);
842
- if (!connectionEntry) {
843
- throw new Error("Connection not found.");
844
- }
845
- const writeStart = Date.now();
846
- connectionEntry.writer.write(message).then(() => {
847
- const durationMs = Date.now() - writeStart;
848
- this._params.monitor?.recordMessageSent(message, durationMs);
849
- }).catch((err) => {
850
- if (connectionEntry.isOpen) {
851
- import_log3.log.catch(err, void 0, {
852
- F: __dxlog_file3,
853
- L: 181,
854
- S: this,
855
- C: (f, a) => f(...a)
856
- });
857
- }
858
- this._params.monitor?.recordMessageSendingFailed(message);
859
- });
845
+ get targetTimeframe() {
846
+ return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
860
847
  }
861
- // TODO(dmaretskyi): Remove.
862
- getPeersInterestedInCollection(collectionId) {
863
- return Array.from(this._connections.values()).map((connection) => {
864
- return connection.connection.shouldSyncCollection({
865
- collectionId
866
- }) ? connection.connection.peerId : null;
867
- }).filter(import_util2.nonNullable);
848
+ get reachedTarget() {
849
+ return this._reachedTarget;
868
850
  }
869
- _onConnectionOpen(connection) {
870
- (0, import_log3.log)("Connection opened", {
871
- peerId: connection.peerId
851
+ get feeds() {
852
+ return Array.from(this._feeds.values());
853
+ }
854
+ async waitUntilTimeframe(target) {
855
+ await this._timeframeClock.waitUntilReached(target);
856
+ }
857
+ setTargetTimeframe(target) {
858
+ this._targetTimeframe = target;
859
+ }
860
+ /**
861
+ * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
862
+ *
863
+ * This function will resolve immediately if the pipeline is stalled.
864
+ *
865
+ * @param timeout Timeout in milliseconds to specify the maximum wait time.
866
+ */
867
+ async waitUntilReachedTargetTimeframe({ ctx = new import_context5.Context(void 0, {
868
+ F: __dxlog_file6,
869
+ L: 129
870
+ }), timeout, breakOnStall = true } = {}) {
871
+ (0, import_log4.log)("waitUntilReachedTargetTimeframe", {
872
+ timeout,
873
+ current: this.timeframe,
874
+ target: this.targetTimeframe
872
875
  }, {
873
- F: __dxlog_file3,
874
- L: 199,
876
+ F: __dxlog_file6,
877
+ L: 133,
875
878
  S: this,
876
879
  C: (f, a) => f(...a)
877
880
  });
878
- (0, import_invariant4.invariant)(!this._connections.has(connection.peerId), void 0, {
879
- F: __dxlog_file3,
880
- L: 200,
881
- S: this,
882
- A: [
883
- "!this._connections.has(connection.peerId as PeerId)",
884
- ""
885
- ]
886
- });
887
- const reader = connection.readable.getReader();
888
- const writer = connection.writable.getWriter();
889
- const connectionEntry = {
890
- connection,
891
- reader,
892
- writer,
893
- isOpen: true
894
- };
895
- this._connections.set(connection.peerId, connectionEntry);
896
- queueMicrotask(async () => {
897
- try {
898
- while (true) {
899
- const { done, value } = await reader.read();
881
+ this._reachedTargetPromise ??= Promise.race([
882
+ this._timeframeClock.update.waitForCondition(() => {
883
+ return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
884
+ }),
885
+ ...breakOnStall ? [
886
+ this.stalled.discardParameter().waitForCount(1)
887
+ ] : []
888
+ ]);
889
+ let done = false;
890
+ if (timeout) {
891
+ return Promise.race([
892
+ (0, import_context5.rejectOnDispose)(ctx),
893
+ (0, import_context5.rejectOnDispose)(this._ctx),
894
+ this._reachedTargetPromise.then(() => {
895
+ done = true;
896
+ this._reachedTarget = true;
897
+ }),
898
+ (0, import_async5.sleepWithContext)(this._ctx, timeout).then(() => {
900
899
  if (done) {
901
- break;
900
+ return;
902
901
  }
903
- this._onMessage(value);
904
- }
905
- } catch (err) {
906
- if (connectionEntry.isOpen) {
907
- import_log3.log.catch(err, void 0, {
908
- F: __dxlog_file3,
909
- L: 219,
902
+ import_log4.log.warn("waitUntilReachedTargetTimeframe timed out", {
903
+ timeout,
904
+ current: this.timeframe,
905
+ target: this.targetTimeframe,
906
+ dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
907
+ }, {
908
+ F: __dxlog_file6,
909
+ L: 161,
910
910
  S: this,
911
911
  C: (f, a) => f(...a)
912
912
  });
913
- }
914
- }
915
- });
916
- (0, import_log3.log)("emit peer-candidate", {
917
- peerId: connection.peerId
918
- }, {
919
- F: __dxlog_file3,
920
- L: 224,
913
+ })
914
+ ]);
915
+ } else {
916
+ return this._reachedTargetPromise;
917
+ }
918
+ }
919
+ };
920
+ var Pipeline = class {
921
+ constructor() {
922
+ this._timeframeClock = new TimeframeClock(new import_timeframe2.Timeframe());
923
+ this._feeds = new import_util2.ComplexMap(import_keys2.PublicKey.hash);
924
+ this._state = new PipelineState(this._feeds, this._timeframeClock);
925
+ this._processingTrigger = new import_async5.Trigger().wake();
926
+ this._pauseTrigger = new import_async5.Trigger().wake();
927
+ this._downloads = new import_util2.ComplexMap((value) => import_keys2.PublicKey.hash(value.key));
928
+ this._isStopping = false;
929
+ this._isStarted = false;
930
+ this._isBeingConsumed = false;
931
+ this._isPaused = false;
932
+ }
933
+ get state() {
934
+ return this._state;
935
+ }
936
+ get writer() {
937
+ (0, import_invariant4.invariant)(this._writer, "Writer not set.", {
938
+ F: __dxlog_file6,
939
+ L: 243,
921
940
  S: this,
922
- C: (f, a) => f(...a)
941
+ A: [
942
+ "this._writer",
943
+ "'Writer not set.'"
944
+ ]
923
945
  });
924
- this._emitPeerCandidate(connection);
925
- this._params.monitor?.recordPeerConnected(connection.peerId);
946
+ return this._writer;
926
947
  }
927
- _onMessage(message) {
928
- if (isCollectionQueryMessage(message)) {
929
- this._params.onCollectionStateQueried(message.collectionId, message.senderId);
930
- } else if (isCollectionStateMessage(message)) {
931
- this._params.onCollectionStateReceived(message.collectionId, message.senderId, message.state);
932
- } else {
933
- this.emit("message", message);
948
+ hasFeed(feedKey) {
949
+ return this._feeds.has(feedKey);
950
+ }
951
+ getFeeds() {
952
+ return this._feedSetIterator.feeds;
953
+ }
954
+ // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
955
+ // which might be opening feeds during the mutation processing, which w
956
+ async addFeed(feed) {
957
+ this._feeds.set(feed.key, feed);
958
+ if (this._feedSetIterator) {
959
+ await this._feedSetIterator.addFeed(feed);
960
+ }
961
+ if (this._isStarted && !this._isPaused) {
962
+ this._setFeedDownloadState(feed);
934
963
  }
935
- this._params.monitor?.recordMessageReceived(message);
936
964
  }
937
- /**
938
- * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
939
- * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
940
- */
941
- _onConnectionAuthScopeChanged(connection) {
942
- (0, import_log3.log)("Connection auth scope changed", {
943
- peerId: connection.peerId
944
- }, {
945
- F: __dxlog_file3,
946
- L: 245,
965
+ setWriteFeed(feed) {
966
+ (0, import_invariant4.invariant)(!this._writer, "Writer already set.", {
967
+ F: __dxlog_file6,
968
+ L: 270,
947
969
  S: this,
948
- C: (f, a) => f(...a)
970
+ A: [
971
+ "!this._writer",
972
+ "'Writer already set.'"
973
+ ]
949
974
  });
950
- const entry = this._connections.get(connection.peerId);
951
- (0, import_invariant4.invariant)(entry, void 0, {
952
- F: __dxlog_file3,
953
- L: 247,
975
+ (0, import_invariant4.invariant)(feed.properties.writable, "Feed must be writable.", {
976
+ F: __dxlog_file6,
977
+ L: 271,
954
978
  S: this,
955
979
  A: [
956
- "entry",
957
- ""
980
+ "feed.properties.writable",
981
+ "'Feed must be writable.'"
958
982
  ]
959
983
  });
960
- this.emit("peer-disconnected", {
961
- peerId: connection.peerId
962
- });
963
- this._emitPeerCandidate(connection);
984
+ this._writer = createMappedFeedWriter((payload) => ({
985
+ timeframe: this._timeframeClock.timeframe,
986
+ payload
987
+ }), feed.createFeedWriter());
964
988
  }
965
- _onConnectionClosed(connection) {
966
- (0, import_log3.log)("Connection closed", {
967
- peerId: connection.peerId
968
- }, {
969
- F: __dxlog_file3,
970
- L: 253,
971
- S: this,
972
- C: (f, a) => f(...a)
973
- });
974
- const entry = this._connections.get(connection.peerId);
975
- (0, import_invariant4.invariant)(entry, void 0, {
976
- F: __dxlog_file3,
977
- L: 255,
989
+ async start() {
990
+ (0, import_invariant4.invariant)(!this._isStarted, "Pipeline is already started.", {
991
+ F: __dxlog_file6,
992
+ L: 284,
978
993
  S: this,
979
994
  A: [
980
- "entry",
981
- ""
995
+ "!this._isStarted",
996
+ "'Pipeline is already started.'"
982
997
  ]
983
998
  });
984
- entry.isOpen = false;
985
- this.emit("peer-disconnected", {
986
- peerId: connection.peerId
987
- });
988
- this._params.monitor?.recordPeerDisconnected(connection.peerId);
989
- void entry.reader.cancel().catch((err) => import_log3.log.catch(err, void 0, {
990
- F: __dxlog_file3,
991
- L: 261,
999
+ (0, import_log4.log)("starting...", void 0, {
1000
+ F: __dxlog_file6,
1001
+ L: 285,
992
1002
  S: this,
993
1003
  C: (f, a) => f(...a)
994
- }));
995
- void entry.writer.abort().catch((err) => import_log3.log.catch(err, void 0, {
996
- F: __dxlog_file3,
997
- L: 262,
1004
+ });
1005
+ await this._initIterator();
1006
+ await this._feedSetIterator.open();
1007
+ this._isStarted = true;
1008
+ (0, import_log4.log)("started", void 0, {
1009
+ F: __dxlog_file6,
1010
+ L: 289,
998
1011
  S: this,
999
1012
  C: (f, a) => f(...a)
1000
- }));
1001
- this._connections.delete(connection.peerId);
1002
- }
1003
- _emitPeerCandidate(connection) {
1004
- this.emit("peer-candidate", {
1005
- peerId: connection.peerId,
1006
- peerMetadata: createEchoPeerMetadata()
1007
1013
  });
1014
+ if (!this._isPaused) {
1015
+ for (const feed of this._feeds.values()) {
1016
+ this._setFeedDownloadState(feed);
1017
+ }
1018
+ }
1008
1019
  }
1009
- };
1010
- _ts_decorate([
1011
- import_async4.synchronized
1012
- ], EchoNetworkAdapter.prototype, "open", null);
1013
- _ts_decorate([
1014
- import_async4.synchronized
1015
- ], EchoNetworkAdapter.prototype, "close", null);
1016
- _ts_decorate([
1017
- import_async4.synchronized
1018
- ], EchoNetworkAdapter.prototype, "addReplicator", null);
1019
- _ts_decorate([
1020
- import_async4.synchronized
1021
- ], EchoNetworkAdapter.prototype, "removeReplicator", null);
1022
- var createEchoPeerMetadata = () => ({
1023
- // TODO(dmaretskyi): Refactor this.
1024
- dxos_peerSource: "EchoNetworkAdapter"
1025
- });
1026
- var isEchoPeerMetadata = (metadata) => metadata?.dxos_peerSource === "EchoNetworkAdapter";
1027
- var HeadsStore = class {
1028
- constructor({ db }) {
1029
- this._db = db;
1030
- }
1031
- setHeads(documentId, heads, batch) {
1032
- batch.put(documentId, heads, {
1033
- sublevel: this._db,
1034
- keyEncoding: "utf8",
1035
- valueEncoding: import_indexing.headsEncoding
1036
- });
1037
- }
1038
- // TODO(dmaretskyi): Make batched.
1039
- async getHeads(documentIds) {
1040
- return this._db.getMany(documentIds, {
1041
- keyEncoding: "utf8",
1042
- valueEncoding: import_indexing.headsEncoding
1043
- });
1044
- }
1045
- };
1046
- function _ts_decorate2(decorators, target, key, desc) {
1047
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1048
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1049
- r = Reflect.decorate(decorators, target, key, desc);
1050
- else
1051
- for (var i = decorators.length - 1; i >= 0; i--)
1052
- if (d = decorators[i])
1053
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1054
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1055
- }
1056
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
1057
- var AutomergeHost = class extends import_context5.Resource {
1058
- constructor({ db, indexMetadataStore, dataMonitor }) {
1059
- super();
1060
- this._collectionSynchronizer = new CollectionSynchronizer({
1061
- queryCollectionState: this._queryCollectionState.bind(this),
1062
- sendCollectionState: this._sendCollectionState.bind(this),
1063
- shouldSyncCollection: this._shouldSyncCollection.bind(this)
1064
- });
1065
- this._db = db;
1066
- this._storage = new LevelDBStorageAdapter({
1067
- db: db.sublevel("automerge"),
1068
- callbacks: {
1069
- beforeSave: async (params) => this._beforeSave(params),
1070
- afterSave: async (key) => this._afterSave(key)
1071
- },
1072
- monitor: dataMonitor
1073
- });
1074
- this._echoNetworkAdapter = new EchoNetworkAdapter({
1075
- getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
1076
- onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
1077
- onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
1078
- monitor: dataMonitor
1079
- });
1080
- this._headsStore = new HeadsStore({
1081
- db: db.sublevel("heads")
1020
+ async stop() {
1021
+ (0, import_log4.log)("stopping...", void 0, {
1022
+ F: __dxlog_file6,
1023
+ L: 300,
1024
+ S: this,
1025
+ C: (f, a) => f(...a)
1082
1026
  });
1083
- this._indexMetadataStore = indexMetadataStore;
1084
- }
1085
- async _open() {
1086
- this._peerId = `host-${import_keys2.PublicKey.random().toHex()}`;
1087
- await this._storage.open?.();
1088
- this._repo = new import_automerge_repo.Repo({
1089
- peerId: this._peerId,
1090
- sharePolicy: this._sharePolicy.bind(this),
1091
- storage: this._storage,
1092
- network: [
1093
- // Upstream swarm.
1094
- this._echoNetworkAdapter
1095
- ]
1027
+ this._isStopping = true;
1028
+ for (const [feed, handle] of this._downloads.entries()) {
1029
+ feed.undownload(handle);
1030
+ }
1031
+ this._downloads.clear();
1032
+ await this._feedSetIterator?.close();
1033
+ await this._processingTrigger.wait();
1034
+ await this._state._ctx.dispose();
1035
+ this._state._ctx = new import_context5.Context(void 0, {
1036
+ F: __dxlog_file6,
1037
+ L: 309
1096
1038
  });
1097
- import_async3.Event.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => this._onPeerConnected(e.peerId));
1098
- import_async3.Event.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => this._onPeerDisconnected(e.peerId));
1099
- this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
1100
- this._onRemoteCollectionStateUpdated(collectionId, peerId);
1039
+ this._state._reachedTargetPromise = void 0;
1040
+ this._state._reachedTarget = false;
1041
+ this._isStarted = false;
1042
+ (0, import_log4.log)("stopped", void 0, {
1043
+ F: __dxlog_file6,
1044
+ L: 313,
1045
+ S: this,
1046
+ C: (f, a) => f(...a)
1101
1047
  });
1102
- await this._echoNetworkAdapter.open();
1103
- await this._collectionSynchronizer.open();
1104
- await this._echoNetworkAdapter.open();
1105
- await this._echoNetworkAdapter.whenConnected();
1106
- }
1107
- async _close() {
1108
- await this._collectionSynchronizer.close();
1109
- await this._storage.close?.();
1110
- await this._echoNetworkAdapter.close();
1111
- await this._ctx.dispose();
1112
1048
  }
1113
1049
  /**
1114
- * @deprecated To be abstracted away.
1050
+ * @param timeframe Timeframe of already processed messages.
1051
+ * The pipeline will start processing messages AFTER this timeframe.
1115
1052
  */
1116
- get repo() {
1117
- return this._repo;
1118
- }
1119
- get peerId() {
1120
- return this._peerId;
1121
- }
1122
- get loadedDocsCount() {
1123
- return Object.keys(this._repo.handles).length;
1124
- }
1125
- async addReplicator(replicator) {
1126
- await this._echoNetworkAdapter.addReplicator(replicator);
1127
- }
1128
- async removeReplicator(replicator) {
1129
- await this._echoNetworkAdapter.removeReplicator(replicator);
1053
+ async setCursor(timeframe) {
1054
+ (0, import_invariant4.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
1055
+ F: __dxlog_file6,
1056
+ L: 322,
1057
+ S: this,
1058
+ A: [
1059
+ "!this._isStarted || this._isPaused",
1060
+ "'Invalid state.'"
1061
+ ]
1062
+ });
1063
+ this._state._startTimeframe = timeframe;
1064
+ this._timeframeClock.setTimeframe(timeframe);
1065
+ if (this._feedSetIterator) {
1066
+ await this._feedSetIterator.close();
1067
+ await this._initIterator();
1068
+ await this._feedSetIterator.open();
1069
+ }
1130
1070
  }
1131
1071
  /**
1132
- * Loads the document handle from the repo and waits for it to be ready.
1072
+ * Calling pause while processing will cause a deadlock.
1133
1073
  */
1134
- async loadDoc(ctx, documentId, opts) {
1135
- let handle;
1136
- if (typeof documentId === "string") {
1137
- handle = this._repo.handles[documentId];
1138
- }
1139
- if (!handle) {
1140
- handle = this._repo.find(documentId);
1074
+ async pause() {
1075
+ if (this._isPaused) {
1076
+ return;
1141
1077
  }
1142
- if (!handle.isReady()) {
1143
- if (!opts?.timeout) {
1144
- await (0, import_context5.cancelWithContext)(ctx, handle.whenReady());
1145
- } else {
1146
- await (0, import_context5.cancelWithContext)(ctx, (0, import_async3.asyncTimeout)(handle.whenReady(), opts.timeout));
1147
- }
1078
+ this._pauseTrigger.reset();
1079
+ await this._processingTrigger.wait();
1080
+ this._isPaused = true;
1081
+ }
1082
+ async unpause() {
1083
+ (0, import_invariant4.invariant)(this._isPaused, "Pipeline is not paused.", {
1084
+ F: __dxlog_file6,
1085
+ L: 351,
1086
+ S: this,
1087
+ A: [
1088
+ "this._isPaused",
1089
+ "'Pipeline is not paused.'"
1090
+ ]
1091
+ });
1092
+ this._pauseTrigger.wake();
1093
+ this._isPaused = false;
1094
+ for (const feed of this._feeds.values()) {
1095
+ this._setFeedDownloadState(feed);
1148
1096
  }
1149
- return handle;
1150
1097
  }
1151
1098
  /**
1152
- * Create new persisted document.
1099
+ * Starts to iterate over the ordered messages from the added feeds.
1100
+ * Updates the timeframe clock after the message has bee processed.
1153
1101
  */
1154
- createDoc(initialValue, opts) {
1155
- if (opts?.preserveHistory) {
1156
- if (!(0, import_automerge3.isAutomerge)(initialValue)) {
1157
- throw new TypeError("Initial value must be an Automerge document");
1102
+ async *consume() {
1103
+ (0, import_invariant4.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
1104
+ F: __dxlog_file6,
1105
+ L: 366,
1106
+ S: this,
1107
+ A: [
1108
+ "!this._isBeingConsumed",
1109
+ "'Pipeline is already being consumed.'"
1110
+ ]
1111
+ });
1112
+ this._isBeingConsumed = true;
1113
+ (0, import_invariant4.invariant)(this._feedSetIterator, "Iterator not initialized.", {
1114
+ F: __dxlog_file6,
1115
+ L: 369,
1116
+ S: this,
1117
+ A: [
1118
+ "this._feedSetIterator",
1119
+ "'Iterator not initialized.'"
1120
+ ]
1121
+ });
1122
+ let lastFeedSetIterator = this._feedSetIterator;
1123
+ let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1124
+ while (!this._isStopping) {
1125
+ await this._pauseTrigger.wait();
1126
+ if (lastFeedSetIterator !== this._feedSetIterator) {
1127
+ (0, import_invariant4.invariant)(this._feedSetIterator, "Iterator not initialized.", {
1128
+ F: __dxlog_file6,
1129
+ L: 378,
1130
+ S: this,
1131
+ A: [
1132
+ "this._feedSetIterator",
1133
+ "'Iterator not initialized.'"
1134
+ ]
1135
+ });
1136
+ lastFeedSetIterator = this._feedSetIterator;
1137
+ iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1138
+ }
1139
+ const { done, value } = await iterable.next();
1140
+ if (!done) {
1141
+ const block = value ?? (0, import_debug2.failUndefined)();
1142
+ this._processingTrigger.reset();
1143
+ this._timeframeClock.updatePendingTimeframe(import_keys2.PublicKey.from(block.feedKey), block.seq);
1144
+ yield block;
1145
+ this._processingTrigger.wake();
1146
+ this._timeframeClock.updateTimeframe();
1158
1147
  }
1159
- return this._repo.import((0, import_automerge3.save)(initialValue));
1160
- } else {
1161
- return this._repo.create(initialValue);
1162
1148
  }
1149
+ this._isBeingConsumed = false;
1163
1150
  }
1164
- async waitUntilHeadsReplicated(heads) {
1165
- const entries = heads.entries;
1166
- if (!entries?.length) {
1167
- return;
1151
+ _setFeedDownloadState(feed) {
1152
+ let handle = this._downloads.get(feed);
1153
+ if (handle) {
1154
+ feed.undownload(handle);
1168
1155
  }
1169
- const documentIds = entries.map((entry) => entry.documentId);
1170
- const documentHeads = await this.getHeads(documentIds);
1171
- const headsToWait = entries.filter((entry, index) => {
1172
- const targetHeads = entry.heads;
1173
- if (!targetHeads || targetHeads.length === 0) {
1174
- return false;
1156
+ const timeframe = this._state._startTimeframe;
1157
+ const seq = timeframe.get(feed.key) ?? -1;
1158
+ (0, import_log4.log)("download", {
1159
+ feed: feed.key.truncate(),
1160
+ seq,
1161
+ length: feed.length
1162
+ }, {
1163
+ F: __dxlog_file6,
1164
+ L: 407,
1165
+ S: this,
1166
+ C: (f, a) => f(...a)
1167
+ });
1168
+ handle = feed.download({
1169
+ start: seq + 1,
1170
+ linear: true
1171
+ }, (err, data) => {
1172
+ if (err) {
1173
+ } else {
1174
+ import_log4.log.info("downloaded", {
1175
+ data
1176
+ }, {
1177
+ F: __dxlog_file6,
1178
+ L: 412,
1179
+ S: this,
1180
+ C: (f, a) => f(...a)
1181
+ });
1175
1182
  }
1176
- const currentHeads = documentHeads[index];
1177
- return !(currentHeads !== null && (0, import_automerge3.equals)(currentHeads, targetHeads));
1178
1183
  });
1179
- if (headsToWait.length > 0) {
1180
- await Promise.all(headsToWait.map(async (entry, index) => {
1181
- const handle = await this.loadDoc(import_context5.Context.default(void 0, {
1182
- F: __dxlog_file4,
1183
- L: 226
1184
- }), entry.documentId);
1185
- await waitForHeads(handle, entry.heads);
1186
- }));
1184
+ this._downloads.set(feed, handle);
1185
+ }
1186
+ async _initIterator() {
1187
+ this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
1188
+ start: startAfter(this._timeframeClock.timeframe),
1189
+ stallTimeout: 1e3
1190
+ });
1191
+ this._feedSetIterator.stalled.on((iterator) => {
1192
+ import_log4.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
1193
+ F: __dxlog_file6,
1194
+ L: 426,
1195
+ S: this,
1196
+ C: (f, a) => f(...a)
1197
+ });
1198
+ this._state.stalled.emit();
1199
+ });
1200
+ for (const feed of this._feeds.values()) {
1201
+ await this._feedSetIterator.addFeed(feed);
1187
1202
  }
1188
- await this._repo.flush(documentIds.filter((documentId) => !!this._repo.handles[documentId]));
1189
1203
  }
1190
- async reIndexHeads(documentIds) {
1191
- for (const documentId of documentIds) {
1192
- import_log2.log.info("re-indexing heads for document", {
1193
- documentId
1204
+ };
1205
+ _ts_decorate2([
1206
+ import_async5.synchronized
1207
+ ], Pipeline.prototype, "start", null);
1208
+ _ts_decorate2([
1209
+ import_async5.synchronized
1210
+ ], Pipeline.prototype, "stop", null);
1211
+ _ts_decorate2([
1212
+ import_async5.synchronized
1213
+ ], Pipeline.prototype, "setCursor", null);
1214
+ _ts_decorate2([
1215
+ import_async5.synchronized
1216
+ ], Pipeline.prototype, "pause", null);
1217
+ _ts_decorate2([
1218
+ import_async5.synchronized
1219
+ ], Pipeline.prototype, "unpause", null);
1220
+ function _ts_decorate3(decorators, target, key, desc) {
1221
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1222
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1223
+ r = Reflect.decorate(decorators, target, key, desc);
1224
+ else
1225
+ for (var i = decorators.length - 1; i >= 0; i--)
1226
+ if (d = decorators[i])
1227
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1228
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1229
+ }
1230
+ var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
1231
+ var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
1232
+ var CONTROL_PIPELINE_SNAPSHOT_DELAY = 1e4;
1233
+ var USE_SNAPSHOTS = true;
1234
+ var ControlPipeline = class {
1235
+ constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
1236
+ this._ctx = new import_context7.Context(void 0, {
1237
+ F: __dxlog_file7,
1238
+ L: 47
1239
+ });
1240
+ this._lastTimeframeSaveTime = Date.now();
1241
+ this.onFeedAdmitted = new import_util4.Callback();
1242
+ this._usage = new import_tracing2.TimeUsageCounter();
1243
+ this._mutations = new import_tracing2.TimeSeriesCounter();
1244
+ this._snapshotTask = new import_async7.DeferredTask(this._ctx, async () => {
1245
+ await (0, import_async7.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
1246
+ await this._saveSnapshot();
1247
+ });
1248
+ this._spaceKey = spaceKey;
1249
+ this._metadata = metadataStore;
1250
+ this._pipeline = new Pipeline();
1251
+ void this._pipeline.addFeed(genesisFeed);
1252
+ this._spaceStateMachine = new import_credentials2.SpaceStateMachine(spaceKey);
1253
+ this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
1254
+ (0, import_log7.log)("feed admitted", {
1255
+ key: info.key
1194
1256
  }, {
1195
- F: __dxlog_file4,
1196
- L: 238,
1257
+ F: __dxlog_file7,
1258
+ L: 82,
1197
1259
  S: this,
1198
1260
  C: (f, a) => f(...a)
1199
1261
  });
1200
- const handle = this._repo.find(documentId);
1201
- await handle.whenReady([
1202
- "ready",
1203
- "requesting"
1204
- ]);
1205
- if (handle.inState([
1206
- "requesting"
1207
- ])) {
1208
- import_log2.log.warn("document is not available locally, skipping", {
1209
- documentId
1262
+ if (info.assertion.designation === import_credentials3.AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
1263
+ queueMicrotask(async () => {
1264
+ try {
1265
+ const feed = await feedProvider(info.key);
1266
+ if (!this._pipeline.hasFeed(feed.key)) {
1267
+ await this._pipeline.addFeed(feed);
1268
+ }
1269
+ } catch (err) {
1270
+ import_log7.log.catch(err, void 0, {
1271
+ F: __dxlog_file7,
1272
+ L: 93,
1273
+ S: this,
1274
+ C: (f, a) => f(...a)
1275
+ });
1276
+ }
1277
+ });
1278
+ }
1279
+ await this.onFeedAdmitted.callIfSet(info);
1280
+ });
1281
+ this.onMemberRoleChanged = this._spaceStateMachine.onMemberRoleChanged;
1282
+ this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
1283
+ this.onDelegatedInvitation = this._spaceStateMachine.onDelegatedInvitation;
1284
+ this.onDelegatedInvitationRemoved = this._spaceStateMachine.onDelegatedInvitationRemoved;
1285
+ }
1286
+ get spaceState() {
1287
+ return this._spaceStateMachine;
1288
+ }
1289
+ get pipeline() {
1290
+ return this._pipeline;
1291
+ }
1292
+ async setWriteFeed(feed) {
1293
+ await this._pipeline.addFeed(feed);
1294
+ this._pipeline.setWriteFeed(feed);
1295
+ }
1296
+ async start() {
1297
+ const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
1298
+ (0, import_log7.log)("load snapshot", {
1299
+ key: this._spaceKey,
1300
+ present: !!snapshot,
1301
+ tf: snapshot?.timeframe
1302
+ }, {
1303
+ F: __dxlog_file7,
1304
+ L: 123,
1305
+ S: this,
1306
+ C: (f, a) => f(...a)
1307
+ });
1308
+ if (USE_SNAPSHOTS && snapshot) {
1309
+ await this._processSnapshot(snapshot);
1310
+ }
1311
+ (0, import_log7.log)("starting...", void 0, {
1312
+ F: __dxlog_file7,
1313
+ L: 128,
1314
+ S: this,
1315
+ C: (f, a) => f(...a)
1316
+ });
1317
+ setTimeout(async () => {
1318
+ void this._consumePipeline(new import_context7.Context(void 0, {
1319
+ F: __dxlog_file7,
1320
+ L: 130
1321
+ }));
1322
+ });
1323
+ await this._pipeline.start();
1324
+ (0, import_log7.log)("started", void 0, {
1325
+ F: __dxlog_file7,
1326
+ L: 134,
1327
+ S: this,
1328
+ C: (f, a) => f(...a)
1329
+ });
1330
+ }
1331
+ async _processSnapshot(snapshot) {
1332
+ await this._pipeline.setCursor(snapshot.timeframe);
1333
+ for (const message of snapshot.messages ?? []) {
1334
+ const result = await this._spaceStateMachine.process(message.credential, {
1335
+ sourceFeed: message.feedKey,
1336
+ skipVerification: true
1337
+ });
1338
+ if (!result) {
1339
+ import_log7.log.warn("credential processing failed from snapshot", {
1340
+ message
1210
1341
  }, {
1211
- F: __dxlog_file4,
1212
- L: 242,
1342
+ F: __dxlog_file7,
1343
+ L: 147,
1213
1344
  S: this,
1214
1345
  C: (f, a) => f(...a)
1215
1346
  });
1216
- continue;
1217
1347
  }
1218
- const doc = handle.docSync();
1219
- (0, import_invariant3.invariant)(doc, void 0, {
1220
- F: __dxlog_file4,
1221
- L: 247,
1222
- S: this,
1223
- A: [
1224
- "doc",
1225
- ""
1226
- ]
1227
- });
1228
- const heads = (0, import_automerge3.getHeads)(doc);
1229
- const batch = this._db.batch();
1230
- this._headsStore.setHeads(documentId, heads, batch);
1231
- await batch.write();
1232
1348
  }
1233
- import_log2.log.info("done re-indexing heads", void 0, {
1234
- F: __dxlog_file4,
1235
- L: 254,
1349
+ }
1350
+ async _saveSnapshot() {
1351
+ await this._pipeline.pause();
1352
+ const snapshot = {
1353
+ timeframe: this._pipeline.state.timeframe,
1354
+ messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
1355
+ feedKey: entry.sourceFeed,
1356
+ credential: entry.credential
1357
+ }))
1358
+ };
1359
+ await this._pipeline.unpause();
1360
+ (0, import_log7.log)("save snapshot", {
1361
+ key: this._spaceKey,
1362
+ snapshot
1363
+ }, {
1364
+ F: __dxlog_file7,
1365
+ L: 163,
1236
1366
  S: this,
1237
1367
  C: (f, a) => f(...a)
1238
1368
  });
1369
+ await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
1239
1370
  }
1240
- // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
1241
- // Hosts, running in the worker, don't share documents unless requested by other peers.
1242
- // NOTE: If both peers return sharePolicy=false the replication will not happen
1243
- // https://github.com/automerge/automerge-repo/pull/292
1244
- async _sharePolicy(peerId, documentId) {
1245
- if (peerId.startsWith("client-")) {
1246
- return false;
1247
- }
1248
- if (!documentId) {
1249
- return false;
1371
+ async _consumePipeline(ctx) {
1372
+ for await (const msg of this._pipeline.consume()) {
1373
+ const span = this._usage.beginRecording();
1374
+ this._mutations.inc();
1375
+ try {
1376
+ await this._processMessage(ctx, msg);
1377
+ } catch (err) {
1378
+ import_log7.log.catch(err, void 0, {
1379
+ F: __dxlog_file7,
1380
+ L: 176,
1381
+ S: this,
1382
+ C: (f, a) => f(...a)
1383
+ });
1384
+ }
1385
+ span.end();
1250
1386
  }
1251
- const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
1252
- if (isEchoPeerMetadata(peerMetadata)) {
1253
- return this._echoNetworkAdapter.shouldAdvertise(peerId, {
1254
- documentId
1387
+ }
1388
+ async _processMessage(ctx, msg) {
1389
+ (0, import_log7.log)("processing", {
1390
+ key: msg.feedKey,
1391
+ seq: msg.seq
1392
+ }, {
1393
+ F: __dxlog_file7,
1394
+ L: 186,
1395
+ S: this,
1396
+ C: (f, a) => f(...a)
1397
+ });
1398
+ if (msg.data.payload.credential) {
1399
+ const timer = import_util4.tracer.mark("dxos.echo.pipeline.control");
1400
+ const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
1401
+ sourceFeed: import_keys4.PublicKey.from(msg.feedKey)
1255
1402
  });
1403
+ timer.end();
1404
+ if (!result) {
1405
+ import_log7.log.warn("processing failed", {
1406
+ msg
1407
+ }, {
1408
+ F: __dxlog_file7,
1409
+ L: 195,
1410
+ S: this,
1411
+ C: (f, a) => f(...a)
1412
+ });
1413
+ } else {
1414
+ await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
1415
+ }
1416
+ this._snapshotTask.schedule();
1256
1417
  }
1257
- return false;
1258
1418
  }
1259
- async _beforeSave({ path, batch }) {
1260
- const handle = this._repo.handles[path[0]];
1261
- if (!handle) {
1262
- return;
1263
- }
1264
- const doc = handle.docSync();
1265
- if (!doc) {
1266
- return;
1419
+ async _noteTargetStateIfNeeded(timeframe) {
1420
+ if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
1421
+ this._lastTimeframeSaveTime = Date.now();
1422
+ await this._saveTargetTimeframe(timeframe);
1267
1423
  }
1268
- const heads = (0, import_automerge3.getHeads)(doc);
1269
- this._headsStore.setHeads(handle.documentId, heads, batch);
1270
- const spaceKey = getSpaceKeyFromDoc(doc) ?? void 0;
1271
- const objectIds = Object.keys(doc.objects ?? {});
1272
- const encodedIds = objectIds.map((objectId) => import_protocols4.objectPointerCodec.encode({
1273
- documentId: handle.documentId,
1274
- objectId,
1275
- spaceKey
1276
- }));
1277
- const idToLastHash = new Map(encodedIds.map((id) => [
1278
- id,
1279
- heads
1280
- ]));
1281
- this._indexMetadataStore.markDirty(idToLastHash, batch);
1282
1424
  }
1283
- _shouldSyncCollection(collectionId, peerId) {
1284
- const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
1285
- if (isEchoPeerMetadata(peerMetadata)) {
1286
- return this._echoNetworkAdapter.shouldSyncCollection(peerId, {
1287
- collectionId
1288
- });
1289
- }
1290
- return false;
1291
- }
1292
- /**
1293
- * Called by AutomergeStorageAdapter after levelDB batch commit.
1294
- */
1295
- async _afterSave(path) {
1296
- this._indexMetadataStore.notifyMarkedDirty();
1297
- const documentId = path[0];
1298
- const document = this._repo.handles[documentId]?.docSync();
1299
- if (document) {
1300
- const heads = (0, import_automerge3.getHeads)(document);
1301
- this._onHeadsChanged(documentId, heads);
1302
- }
1303
- }
1304
- _automergePeers() {
1305
- return this._repo.peers;
1306
- }
1307
- async _getContainingSpaceForDocument(documentId) {
1308
- const doc = this._repo.handles[documentId]?.docSync();
1309
- if (!doc) {
1310
- return null;
1311
- }
1312
- const spaceKeyHex = getSpaceKeyFromDoc(doc);
1313
- if (!spaceKeyHex) {
1314
- return null;
1315
- }
1316
- return import_keys2.PublicKey.from(spaceKeyHex);
1317
- }
1318
- /**
1319
- * Flush documents to disk.
1320
- */
1321
- async flush({ documentIds } = {}) {
1322
- await this._repo.flush(documentIds);
1323
- }
1324
- async getHeads(documentIds) {
1325
- const result = [];
1326
- const storeRequestIds = [];
1327
- const storeResultIndices = [];
1328
- for (const documentId of documentIds) {
1329
- const doc = this._repo.handles[documentId]?.docSync();
1330
- if (doc) {
1331
- result.push((0, import_automerge3.getHeads)(doc));
1332
- } else {
1333
- storeRequestIds.push(documentId);
1334
- storeResultIndices.push(result.length);
1335
- result.push(void 0);
1336
- }
1337
- }
1338
- if (storeRequestIds.length > 0) {
1339
- const storedHeads = await this._headsStore.getHeads(storeRequestIds);
1340
- for (let i = 0; i < storedHeads.length; i++) {
1341
- result[storeResultIndices[i]] = storedHeads[i];
1342
- }
1343
- }
1344
- return result;
1345
- }
1346
- //
1347
- // Collection sync.
1348
- //
1349
- getLocalCollectionState(collectionId) {
1350
- return this._collectionSynchronizer.getLocalCollectionState(collectionId);
1351
- }
1352
- getRemoteCollectionStates(collectionId) {
1353
- return this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
1354
- }
1355
- refreshCollection(collectionId) {
1356
- this._collectionSynchronizer.refreshCollection(collectionId);
1357
- }
1358
- async getCollectionSyncState(collectionId) {
1359
- const result = {
1360
- peers: []
1361
- };
1362
- const localState = this.getLocalCollectionState(collectionId);
1363
- const remoteState = this.getRemoteCollectionStates(collectionId);
1364
- if (!localState) {
1365
- return result;
1366
- }
1367
- for (const [peerId, state] of remoteState) {
1368
- const diff = diffCollectionState(localState, state);
1369
- result.peers.push({
1370
- peerId,
1371
- differentDocuments: diff.different.length
1372
- });
1373
- }
1374
- return result;
1375
- }
1376
- /**
1377
- * Update the local collection state based on the locally stored document heads.
1378
- */
1379
- async updateLocalCollectionState(collectionId, documentIds) {
1380
- const heads = await this.getHeads(documentIds);
1381
- const documents = Object.fromEntries(heads.map((heads2, index) => [
1382
- documentIds[index],
1383
- heads2 ?? []
1384
- ]));
1385
- this._collectionSynchronizer.setLocalCollectionState(collectionId, {
1386
- documents
1425
+ async stop() {
1426
+ (0, import_log7.log)("stopping...", void 0, {
1427
+ F: __dxlog_file7,
1428
+ L: 215,
1429
+ S: this,
1430
+ C: (f, a) => f(...a)
1387
1431
  });
1388
- }
1389
- _onCollectionStateQueried(collectionId, peerId) {
1390
- this._collectionSynchronizer.onCollectionStateQueried(collectionId, peerId);
1391
- }
1392
- _onCollectionStateReceived(collectionId, peerId, state) {
1393
- this._collectionSynchronizer.onRemoteStateReceived(collectionId, peerId, decodeCollectionState(state));
1394
- }
1395
- _queryCollectionState(collectionId, peerId) {
1396
- this._echoNetworkAdapter.queryCollectionState(collectionId, peerId);
1397
- }
1398
- _sendCollectionState(collectionId, peerId, state) {
1399
- this._echoNetworkAdapter.sendCollectionState(collectionId, peerId, encodeCollectionState(state));
1400
- }
1401
- _onPeerConnected(peerId) {
1402
- this._collectionSynchronizer.onConnectionOpen(peerId);
1403
- }
1404
- _onPeerDisconnected(peerId) {
1405
- this._collectionSynchronizer.onConnectionClosed(peerId);
1406
- }
1407
- _onRemoteCollectionStateUpdated(collectionId, peerId) {
1408
- const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
1409
- const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
1410
- if (!localState || !remoteState) {
1411
- return;
1412
- }
1413
- const { different } = diffCollectionState(localState, remoteState);
1414
- if (different.length === 0) {
1415
- return;
1416
- }
1417
- import_log2.log.info("replication documents after collection sync", {
1418
- count: different.length
1419
- }, {
1420
- F: __dxlog_file4,
1421
- L: 463,
1432
+ await this._ctx.dispose();
1433
+ await this._pipeline.stop();
1434
+ await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1435
+ (0, import_log7.log)("stopped", void 0, {
1436
+ F: __dxlog_file7,
1437
+ L: 219,
1422
1438
  S: this,
1423
1439
  C: (f, a) => f(...a)
1424
1440
  });
1425
- for (const documentId of different) {
1426
- this._repo.find(documentId);
1427
- }
1428
1441
  }
1429
- _onHeadsChanged(documentId, heads) {
1430
- for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
1431
- const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
1432
- if (state?.documents[documentId]) {
1433
- const newState = structuredClone(state);
1434
- newState.documents[documentId] = heads;
1435
- this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
1436
- }
1442
+ async _saveTargetTimeframe(timeframe) {
1443
+ try {
1444
+ const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
1445
+ await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
1446
+ this._targetTimeframe = newTimeframe;
1447
+ } catch (err) {
1448
+ (0, import_log7.log)(err, void 0, {
1449
+ F: __dxlog_file7,
1450
+ L: 228,
1451
+ S: this,
1452
+ C: (f, a) => f(...a)
1453
+ });
1437
1454
  }
1438
1455
  }
1439
1456
  };
1440
- _ts_decorate2([
1441
- import_tracing.trace.info()
1442
- ], AutomergeHost.prototype, "_peerId", void 0);
1443
- _ts_decorate2([
1444
- import_tracing.trace.info({
1445
- depth: null
1446
- })
1447
- ], AutomergeHost.prototype, "_automergePeers", null);
1448
- _ts_decorate2([
1449
- import_tracing.trace.span({
1457
+ _ts_decorate3([
1458
+ import_tracing2.trace.metricsCounter()
1459
+ ], ControlPipeline.prototype, "_usage", void 0);
1460
+ _ts_decorate3([
1461
+ import_tracing2.trace.metricsCounter()
1462
+ ], ControlPipeline.prototype, "_mutations", void 0);
1463
+ _ts_decorate3([
1464
+ import_tracing2.trace.span({
1450
1465
  showInBrowserTimeline: true
1451
1466
  })
1452
- ], AutomergeHost.prototype, "flush", null);
1453
- AutomergeHost = _ts_decorate2([
1454
- import_tracing.trace.resource()
1455
- ], AutomergeHost);
1456
- var getSpaceKeyFromDoc = (doc) => {
1457
- const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
1458
- if (rawSpaceKey == null) {
1459
- return null;
1460
- }
1461
- return String(rawSpaceKey);
1462
- };
1463
- var waitForHeads = async (handle, heads) => {
1464
- const unavailableHeads = new Set(heads);
1465
- await handle.whenReady();
1466
- await import_async3.Event.wrap(handle, "change").waitForCondition(() => {
1467
- for (const changeHash of unavailableHeads.values()) {
1468
- if (changeIsPresentInDoc(handle.docSync(), changeHash)) {
1469
- unavailableHeads.delete(changeHash);
1470
- }
1471
- }
1472
- return unavailableHeads.size === 0;
1473
- });
1474
- };
1475
- var changeIsPresentInDoc = (doc, changeHash) => {
1476
- return !!(0, import_automerge3.getBackend)(doc).getChangeByHash(changeHash);
1477
- };
1478
- var decodeCollectionState = (state) => {
1479
- (0, import_invariant3.invariant)(typeof state === "object" && state !== null, "Invalid state", {
1480
- F: __dxlog_file4,
1481
- L: 516,
1482
- S: void 0,
1483
- A: [
1484
- "typeof state === 'object' && state !== null",
1485
- "'Invalid state'"
1486
- ]
1487
- });
1488
- return state;
1489
- };
1490
- var encodeCollectionState = (state) => {
1491
- return state;
1492
- };
1493
- var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
1494
- var deriveCollectionIdFromSpaceId = (spaceId) => `space:${spaceId}`;
1495
- var getSpaceIdFromCollectionId = (collectionId) => {
1496
- const spaceId = collectionId.replace(/^space:/, "");
1497
- (0, import_invariant5.invariant)(import_keys3.SpaceId.isValid(spaceId), void 0, {
1498
- F: __dxlog_file5,
1499
- L: 12,
1500
- S: void 0,
1501
- A: [
1502
- "SpaceId.isValid(spaceId)",
1503
- ""
1504
- ]
1505
- });
1506
- return spaceId;
1507
- };
1508
- var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/auth.ts";
1509
- var AuthExtension = class extends import_teleport.RpcExtension {
1510
- constructor(_authParams) {
1511
- super({
1512
- requested: {
1513
- AuthService: import_protocols5.schema.getService("dxos.mesh.teleport.auth.AuthService")
1514
- },
1515
- exposed: {
1516
- AuthService: import_protocols5.schema.getService("dxos.mesh.teleport.auth.AuthService")
1517
- },
1518
- timeout: 60 * 1e3
1467
+ ], ControlPipeline.prototype, "start", null);
1468
+ _ts_decorate3([
1469
+ import_tracing2.trace.span()
1470
+ ], ControlPipeline.prototype, "_consumePipeline", null);
1471
+ _ts_decorate3([
1472
+ import_tracing2.trace.span()
1473
+ ], ControlPipeline.prototype, "_processMessage", null);
1474
+ ControlPipeline = _ts_decorate3([
1475
+ import_tracing2.trace.resource(),
1476
+ (0, import_async7.trackLeaks)("start", "stop")
1477
+ ], ControlPipeline);
1478
+ function _ts_decorate4(decorators, target, key, desc) {
1479
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1480
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1481
+ r = Reflect.decorate(decorators, target, key, desc);
1482
+ else
1483
+ for (var i = decorators.length - 1; i >= 0; i--)
1484
+ if (d = decorators[i])
1485
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1486
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1487
+ }
1488
+ var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
1489
+ var Space = class extends import_context6.Resource {
1490
+ constructor(params) {
1491
+ super();
1492
+ this.onCredentialProcessed = new import_util3.Callback();
1493
+ this.stateUpdate = new import_async6.Event();
1494
+ (0, import_invariant6.invariant)(params.spaceKey && params.feedProvider, void 0, {
1495
+ F: __dxlog_file8,
1496
+ L: 76,
1497
+ S: this,
1498
+ A: [
1499
+ "params.spaceKey && params.feedProvider",
1500
+ ""
1501
+ ]
1519
1502
  });
1520
- this._authParams = _authParams;
1521
- this._ctx = new import_context7.Context({
1522
- onError: (err) => {
1523
- import_log4.log.catch(err, void 0, {
1524
- F: __dxlog_file6,
1525
- L: 28,
1526
- S: this,
1527
- C: (f, a) => f(...a)
1528
- });
1529
- }
1530
- }, {
1531
- F: __dxlog_file6,
1532
- L: 26
1503
+ this._id = params.id;
1504
+ this._key = params.spaceKey;
1505
+ this._genesisFeedKey = params.genesisFeed.key;
1506
+ this._feedProvider = params.feedProvider;
1507
+ this._snapshotManager = params.snapshotManager;
1508
+ this._controlPipeline = new ControlPipeline({
1509
+ spaceKey: params.spaceKey,
1510
+ genesisFeed: params.genesisFeed,
1511
+ feedProvider: params.feedProvider,
1512
+ metadataStore: params.metadataStore
1533
1513
  });
1534
- }
1535
- async getHandlers() {
1536
- return {
1537
- AuthService: {
1538
- authenticate: async ({ challenge }) => {
1539
- try {
1540
- const credential = await this._authParams.provider(challenge);
1541
- if (!credential) {
1542
- throw new Error("auth rejected");
1543
- }
1544
- return {
1545
- credential
1546
- };
1547
- } catch (err) {
1548
- import_log4.log.error("failed to generate auth credentials", err, {
1549
- F: __dxlog_file6,
1550
- L: 55,
1551
- S: this,
1552
- C: (f, a) => f(...a)
1553
- });
1554
- throw new Error("auth rejected");
1555
- }
1556
- }
1557
- }
1558
- };
1559
- }
1560
- async onOpen(context) {
1561
- await super.onOpen(context);
1562
- (0, import_async5.scheduleTask)(this._ctx, async () => {
1563
- try {
1564
- const challenge = (0, import_crypto2.randomBytes)(32);
1565
- const { credential } = await this.rpc.AuthService.authenticate({
1566
- challenge
1567
- });
1568
- (0, import_invariant6.invariant)(credential?.length > 0, "invalid credential", {
1569
- F: __dxlog_file6,
1570
- L: 69,
1571
- S: this,
1572
- A: [
1573
- "credential?.length > 0",
1574
- "'invalid credential'"
1575
- ]
1576
- });
1577
- const success = await this._authParams.verifier(challenge, credential);
1578
- (0, import_invariant6.invariant)(success, "credential not verified", {
1579
- F: __dxlog_file6,
1580
- L: 71,
1581
- S: this,
1582
- A: [
1583
- "success",
1584
- "'credential not verified'"
1585
- ]
1586
- });
1587
- (0, import_async5.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
1588
- } catch (err) {
1589
- (0, import_log4.log)("auth failed", err, {
1590
- F: __dxlog_file6,
1591
- L: 74,
1592
- S: this,
1593
- C: (f, a) => f(...a)
1514
+ this._controlPipeline.onFeedAdmitted.set(async (info) => {
1515
+ const sparse = info.assertion.designation === import_credentials.AdmittedFeed.Designation.DATA;
1516
+ if (!info.key.equals(params.genesisFeed.key)) {
1517
+ (0, import_async6.scheduleMicroTask)(this._ctx, async () => {
1518
+ await this.protocol.addFeed(await params.feedProvider(info.key, {
1519
+ sparse
1520
+ }));
1594
1521
  });
1595
- this.close();
1596
- this._authParams.onAuthFailure();
1597
1522
  }
1598
1523
  });
1524
+ this._controlPipeline.onCredentialProcessed.set(async (credential) => {
1525
+ await this.onCredentialProcessed.callIfSet(credential);
1526
+ (0, import_log6.log)("onCredentialProcessed", {
1527
+ credential
1528
+ }, {
1529
+ F: __dxlog_file8,
1530
+ L: 104,
1531
+ S: this,
1532
+ C: (f, a) => f(...a)
1533
+ });
1534
+ this.stateUpdate.emit();
1535
+ });
1536
+ this._controlPipeline.onDelegatedInvitation.set(async (invitation) => {
1537
+ (0, import_log6.log)("onDelegatedInvitation", {
1538
+ invitation
1539
+ }, {
1540
+ F: __dxlog_file8,
1541
+ L: 108,
1542
+ S: this,
1543
+ C: (f, a) => f(...a)
1544
+ });
1545
+ await params.onDelegatedInvitationStatusChange(invitation, true);
1546
+ });
1547
+ this._controlPipeline.onDelegatedInvitationRemoved.set(async (invitation) => {
1548
+ (0, import_log6.log)("onDelegatedInvitationRemoved", {
1549
+ invitation
1550
+ }, {
1551
+ F: __dxlog_file8,
1552
+ L: 112,
1553
+ S: this,
1554
+ C: (f, a) => f(...a)
1555
+ });
1556
+ await params.onDelegatedInvitationStatusChange(invitation, false);
1557
+ });
1558
+ this._controlPipeline.onMemberRoleChanged.set(async (changedMembers) => {
1559
+ (0, import_log6.log)("onMemberRoleChanged", () => ({
1560
+ changedMembers: changedMembers.map((m) => [
1561
+ m.key,
1562
+ m.role
1563
+ ])
1564
+ }), {
1565
+ F: __dxlog_file8,
1566
+ L: 116,
1567
+ S: this,
1568
+ C: (f, a) => f(...a)
1569
+ });
1570
+ await params.onMemberRolesChanged(changedMembers);
1571
+ });
1572
+ this.protocol = params.protocol;
1599
1573
  }
1600
- async onClose() {
1601
- await this._ctx.dispose();
1602
- await super.onClose();
1574
+ get id() {
1575
+ return this._id;
1603
1576
  }
1604
- async onAbort() {
1605
- await this._ctx.dispose();
1606
- await super.onAbort();
1577
+ get key() {
1578
+ return this._key;
1607
1579
  }
1608
- };
1609
- function _ts_decorate3(decorators, target, key, desc) {
1610
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1611
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1612
- r = Reflect.decorate(decorators, target, key, desc);
1613
- else
1614
- for (var i = decorators.length - 1; i >= 0; i--)
1615
- if (d = decorators[i])
1616
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1617
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1618
- }
1619
- var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
1620
- var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
1621
- feedKey,
1622
- index
1623
- }));
1624
- var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
1625
- feedKey,
1626
- index
1627
- ]));
1628
- var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
1629
- feedKey,
1630
- index: index + 1
1631
- }));
1632
- var TimeframeClock = class {
1633
- constructor(_timeframe = new import_timeframe.Timeframe()) {
1634
- this._timeframe = _timeframe;
1635
- this.update = new import_async6.Event();
1636
- this._pendingTimeframe = _timeframe;
1580
+ get isOpen() {
1581
+ return this._lifecycleState === import_context6.LifecycleState.OPEN;
1637
1582
  }
1638
- /**
1639
- * Timeframe that was processed by ECHO.
1640
- */
1641
- get timeframe() {
1642
- return this._timeframe;
1583
+ get genesisFeedKey() {
1584
+ return this._genesisFeedKey;
1585
+ }
1586
+ get controlFeedKey() {
1587
+ return this._controlFeed?.key;
1588
+ }
1589
+ get dataFeedKey() {
1590
+ return this._dataFeed?.key;
1591
+ }
1592
+ get spaceState() {
1593
+ return this._controlPipeline.spaceState;
1643
1594
  }
1644
1595
  /**
1645
- * Timeframe that is currently being processed by ECHO.
1646
- * Will be equal to `timeframe` after the processing is complete.
1596
+ * @test-only
1647
1597
  */
1648
- get pendingTimeframe() {
1649
- return this._pendingTimeframe;
1598
+ get controlPipeline() {
1599
+ return this._controlPipeline.pipeline;
1650
1600
  }
1651
- setTimeframe(timeframe) {
1652
- this._timeframe = timeframe;
1653
- this._pendingTimeframe = timeframe;
1654
- this.update.emit(this._timeframe);
1601
+ get snapshotManager() {
1602
+ return this._snapshotManager;
1655
1603
  }
1656
- updatePendingTimeframe(key, seq) {
1657
- this._pendingTimeframe = import_timeframe.Timeframe.merge(this._pendingTimeframe, new import_timeframe.Timeframe([
1658
- [
1659
- key,
1660
- seq
1604
+ async setControlFeed(feed) {
1605
+ (0, import_invariant6.invariant)(!this._controlFeed, "Control feed already set.", {
1606
+ F: __dxlog_file8,
1607
+ L: 168,
1608
+ S: this,
1609
+ A: [
1610
+ "!this._controlFeed",
1611
+ "'Control feed already set.'"
1661
1612
  ]
1662
- ]));
1613
+ });
1614
+ this._controlFeed = feed;
1615
+ await this._controlPipeline.setWriteFeed(feed);
1616
+ return this;
1663
1617
  }
1664
- updateTimeframe() {
1665
- this._timeframe = this._pendingTimeframe;
1666
- this.update.emit(this._timeframe);
1618
+ async setDataFeed(feed) {
1619
+ (0, import_invariant6.invariant)(!this._dataFeed, "Data feed already set.", {
1620
+ F: __dxlog_file8,
1621
+ L: 175,
1622
+ S: this,
1623
+ A: [
1624
+ "!this._dataFeed",
1625
+ "'Data feed already set.'"
1626
+ ]
1627
+ });
1628
+ this._dataFeed = feed;
1629
+ return this;
1667
1630
  }
1668
- hasGaps(timeframe) {
1669
- const gaps = import_timeframe.Timeframe.dependencies(timeframe, this._timeframe);
1670
- return !gaps.isEmpty();
1631
+ /**
1632
+ * Use for diagnostics.
1633
+ */
1634
+ getControlFeeds() {
1635
+ return Array.from(this._controlPipeline.spaceState.feeds.values());
1671
1636
  }
1672
- async waitUntilReached(target) {
1673
- (0, import_log5.log)("waitUntilReached", {
1674
- target,
1675
- current: this._timeframe
1637
+ async _open(ctx) {
1638
+ (0, import_log6.log)("opening...", void 0, {
1639
+ F: __dxlog_file8,
1640
+ L: 189,
1641
+ S: this,
1642
+ C: (f, a) => f(...a)
1643
+ });
1644
+ await this._controlPipeline.start();
1645
+ await this.protocol.start();
1646
+ await this.protocol.addFeed(await this._feedProvider(this._genesisFeedKey));
1647
+ (0, import_log6.log)("opened", void 0, {
1648
+ F: __dxlog_file8,
1649
+ L: 196,
1650
+ S: this,
1651
+ C: (f, a) => f(...a)
1652
+ });
1653
+ }
1654
+ async _close() {
1655
+ (0, import_log6.log)("closing...", {
1656
+ key: this._key
1676
1657
  }, {
1677
- F: __dxlog_file7,
1678
- L: 70,
1658
+ F: __dxlog_file8,
1659
+ L: 201,
1679
1660
  S: this,
1680
1661
  C: (f, a) => f(...a)
1681
1662
  });
1682
- await this.update.waitForCondition(() => {
1683
- (0, import_log5.log)("check if reached", {
1684
- target,
1685
- current: this._timeframe,
1686
- deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
1687
- }, {
1688
- F: __dxlog_file7,
1689
- L: 72,
1690
- S: this,
1691
- C: (f, a) => f(...a)
1692
- });
1693
- return import_timeframe.Timeframe.dependencies(target, this._timeframe).isEmpty();
1663
+ await this.protocol.stop();
1664
+ await this._controlPipeline.stop();
1665
+ (0, import_log6.log)("closed", void 0, {
1666
+ F: __dxlog_file8,
1667
+ L: 207,
1668
+ S: this,
1669
+ C: (f, a) => f(...a)
1694
1670
  });
1695
1671
  }
1696
1672
  };
1697
- _ts_decorate3([
1698
- (0, import_debug.timed)(5e3)
1699
- ], TimeframeClock.prototype, "waitUntilReached", null);
1700
- var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
1701
- var createMessageSelector = (timeframeClock) => {
1702
- return (messages) => {
1703
- for (let i = 0; i < messages.length; i++) {
1704
- const { data: { timeframe } } = messages[i];
1705
- (0, import_invariant8.invariant)(timeframe, void 0, {
1706
- F: __dxlog_file8,
1707
- L: 25,
1708
- S: void 0,
1709
- A: [
1710
- "timeframe",
1711
- ""
1712
- ]
1713
- });
1714
- if (!timeframeClock.hasGaps(timeframe)) {
1715
- return i;
1673
+ _ts_decorate4([
1674
+ import_tracing.trace.info()
1675
+ ], Space.prototype, "protocol", void 0);
1676
+ _ts_decorate4([
1677
+ import_tracing.trace.info()
1678
+ ], Space.prototype, "_controlPipeline", void 0);
1679
+ _ts_decorate4([
1680
+ import_log6.logInfo,
1681
+ import_tracing.trace.info()
1682
+ ], Space.prototype, "id", null);
1683
+ _ts_decorate4([
1684
+ import_log6.logInfo,
1685
+ import_tracing.trace.info()
1686
+ ], Space.prototype, "key", null);
1687
+ _ts_decorate4([
1688
+ import_tracing.trace.span()
1689
+ ], Space.prototype, "_open", null);
1690
+ _ts_decorate4([
1691
+ import_async6.synchronized
1692
+ ], Space.prototype, "_close", null);
1693
+ Space = _ts_decorate4([
1694
+ (0, import_async6.trackLeaks)("open", "close"),
1695
+ import_tracing.trace.resource()
1696
+ ], Space);
1697
+ var SPACE_IDS_CACHE = new import_util3.ComplexMap(import_keys3.PublicKey.hash);
1698
+ var createIdFromSpaceKey = async (spaceKey) => {
1699
+ const cachedValue = SPACE_IDS_CACHE.get(spaceKey);
1700
+ if (cachedValue !== void 0) {
1701
+ return cachedValue;
1702
+ }
1703
+ const digest = await import_crypto3.subtleCrypto.digest("SHA-256", spaceKey.asUint8Array());
1704
+ const bytes = new Uint8Array(digest).slice(0, import_keys3.SpaceId.byteLength);
1705
+ const spaceId = import_keys3.SpaceId.encode(bytes);
1706
+ SPACE_IDS_CACHE.set(spaceKey, spaceId);
1707
+ return spaceId;
1708
+ };
1709
+ var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/admission-discovery-extension.ts";
1710
+ var CredentialRetrieverExtension = class extends import_teleport2.RpcExtension {
1711
+ constructor(_request, _onResult) {
1712
+ super({
1713
+ requested: {
1714
+ AdmissionDiscoveryService: import_protocols5.schema.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
1715
+ }
1716
+ });
1717
+ this._request = _request;
1718
+ this._onResult = _onResult;
1719
+ this._ctx = new import_context8.Context(void 0, {
1720
+ F: __dxlog_file9,
1721
+ L: 25
1722
+ });
1723
+ }
1724
+ async getHandlers() {
1725
+ return {};
1726
+ }
1727
+ async onOpen(context) {
1728
+ await super.onOpen(context);
1729
+ (0, import_async8.scheduleTask)(this._ctx, async () => {
1730
+ try {
1731
+ const result = await this.rpc.AdmissionDiscoveryService.getAdmissionCredential(this._request);
1732
+ this._onResult.wake(result.admissionCredential);
1733
+ } catch (err) {
1734
+ context.close(err);
1716
1735
  }
1717
- }
1718
- (0, import_log7.log)("Skipping...", void 0, {
1719
- F: __dxlog_file8,
1720
- L: 33,
1721
- S: void 0,
1722
- C: (f, a) => f(...a)
1723
1736
  });
1724
- };
1737
+ }
1738
+ async onClose() {
1739
+ await this._ctx.dispose();
1740
+ }
1741
+ async onAbort() {
1742
+ await this._ctx.dispose();
1743
+ }
1725
1744
  };
1726
- function _ts_decorate4(decorators, target, key, desc) {
1745
+ var CredentialServerExtension = class extends import_teleport2.RpcExtension {
1746
+ constructor(_space) {
1747
+ super({
1748
+ exposed: {
1749
+ AdmissionDiscoveryService: import_protocols5.schema.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
1750
+ }
1751
+ });
1752
+ this._space = _space;
1753
+ }
1754
+ async getHandlers() {
1755
+ return {
1756
+ AdmissionDiscoveryService: {
1757
+ getAdmissionCredential: async (request) => {
1758
+ const memberInfo = this._space.spaceState.members.get(request.memberKey);
1759
+ if (!memberInfo?.credential) {
1760
+ throw new import_protocols5.ProtocolError("Space member not found.", request);
1761
+ }
1762
+ return {
1763
+ admissionCredential: memberInfo.credential
1764
+ };
1765
+ }
1766
+ }
1767
+ };
1768
+ }
1769
+ };
1770
+ function _ts_decorate5(decorators, target, key, desc) {
1727
1771
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1728
1772
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1729
1773
  r = Reflect.decorate(decorators, target, key, desc);
@@ -1733,674 +1777,678 @@ function _ts_decorate4(decorators, target, key, desc) {
1733
1777
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1734
1778
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1735
1779
  }
1736
- var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
1737
- var PipelineState = class {
1738
- constructor(_feeds, _timeframeClock) {
1739
- this._feeds = _feeds;
1740
- this._timeframeClock = _timeframeClock;
1741
- this._ctx = new import_context8.Context(void 0, {
1742
- F: __dxlog_file9,
1743
- L: 41
1780
+ var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
1781
+ var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
1782
+ var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
1783
+ var SpaceProtocol = class {
1784
+ constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore, disableP2pReplication }) {
1785
+ this._feeds = /* @__PURE__ */ new Set();
1786
+ this._sessions = new import_util5.ComplexMap(import_keys5.PublicKey.hash);
1787
+ this._topology = new import_network_manager.MMSTTopology({
1788
+ originateConnections: 4,
1789
+ maxPeers: 10,
1790
+ sampleSize: 20
1744
1791
  });
1745
- this.timeframeUpdate = this._timeframeClock.update;
1746
- this.stalled = new import_async7.Event();
1747
- this._startTimeframe = new import_timeframe2.Timeframe();
1748
- this._reachedTarget = false;
1749
- }
1750
- /**
1751
- * Latest theoretical timeframe based on the last mutation in each feed.
1752
- * NOTE: This might never be reached if the mutation dependencies
1753
- */
1754
- // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
1755
- get endTimeframe() {
1756
- return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
1757
- feedKey: feed.key,
1758
- index: feed.length - 1
1759
- })));
1792
+ this.feedAdded = new import_util5.CallbackCollection();
1793
+ this._spaceKey = topic;
1794
+ this._networkManager = networkManager;
1795
+ this._swarmIdentity = swarmIdentity;
1796
+ this._onSessionAuth = onSessionAuth;
1797
+ this._onAuthFailure = onAuthFailure;
1798
+ this.blobSync = new import_teleport_extension_object_sync.BlobSync({
1799
+ blobStore
1800
+ });
1801
+ this._topic = import_crypto4.subtleCrypto.digest("SHA-256", topic.asBuffer()).then(import_crypto4.discoveryKey).then(import_keys5.PublicKey.from);
1802
+ this._disableP2pReplication = disableP2pReplication ?? false;
1760
1803
  }
1761
- get startTimeframe() {
1762
- return this._startTimeframe;
1804
+ get sessions() {
1805
+ return this._sessions;
1763
1806
  }
1764
- get timeframe() {
1765
- return this._timeframeClock.timeframe;
1807
+ get feeds() {
1808
+ return this._feeds;
1766
1809
  }
1767
- get pendingTimeframe() {
1768
- return this._timeframeClock.pendingTimeframe;
1810
+ get _ownPeerKey() {
1811
+ return this._swarmIdentity.peerKey;
1769
1812
  }
1770
- get targetTimeframe() {
1771
- return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
1813
+ // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
1814
+ async addFeed(feed) {
1815
+ (0, import_log8.log)("addFeed", {
1816
+ key: feed.key
1817
+ }, {
1818
+ F: __dxlog_file10,
1819
+ L: 127,
1820
+ S: this,
1821
+ C: (f, a) => f(...a)
1822
+ });
1823
+ this._feeds.add(feed);
1824
+ for (const session of this._sessions.values()) {
1825
+ session.replicator.addFeed(feed);
1826
+ }
1827
+ await this.feedAdded.callSerial(feed);
1772
1828
  }
1773
- get reachedTarget() {
1774
- return this._reachedTarget;
1829
+ // TODO(burdon): Rename open? Common open/close interfaces for all services?
1830
+ async start() {
1831
+ if (this._connection) {
1832
+ return;
1833
+ }
1834
+ const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(""));
1835
+ await this.blobSync.open();
1836
+ (0, import_log8.log)("starting...", void 0, {
1837
+ F: __dxlog_file10,
1838
+ L: 148,
1839
+ S: this,
1840
+ C: (f, a) => f(...a)
1841
+ });
1842
+ const topic = await this._topic;
1843
+ this._connection = await this._networkManager.joinSwarm({
1844
+ protocolProvider: this._createProtocolProvider(credentials),
1845
+ peerId: this._swarmIdentity.peerKey,
1846
+ topic,
1847
+ topology: this._topology,
1848
+ label: `swarm ${topic.truncate()} for space ${this._spaceKey.truncate()}`
1849
+ });
1850
+ (0, import_log8.log)("started", void 0, {
1851
+ F: __dxlog_file10,
1852
+ L: 158,
1853
+ S: this,
1854
+ C: (f, a) => f(...a)
1855
+ });
1775
1856
  }
1776
- get feeds() {
1777
- return Array.from(this._feeds.values());
1857
+ updateTopology() {
1858
+ this._topology.forceUpdate();
1778
1859
  }
1779
- async waitUntilTimeframe(target) {
1780
- await this._timeframeClock.waitUntilReached(target);
1860
+ async stop() {
1861
+ await this.blobSync.close();
1862
+ if (this._connection) {
1863
+ (0, import_log8.log)("stopping...", void 0, {
1864
+ F: __dxlog_file10,
1865
+ L: 169,
1866
+ S: this,
1867
+ C: (f, a) => f(...a)
1868
+ });
1869
+ await this._connection.close();
1870
+ (0, import_log8.log)("stopped", void 0, {
1871
+ F: __dxlog_file10,
1872
+ L: 171,
1873
+ S: this,
1874
+ C: (f, a) => f(...a)
1875
+ });
1876
+ }
1781
1877
  }
1782
- setTargetTimeframe(target) {
1783
- this._targetTimeframe = target;
1878
+ _createProtocolProvider(credentials) {
1879
+ return (wireParams) => {
1880
+ const session = new SpaceProtocolSession({
1881
+ wireParams,
1882
+ swarmIdentity: this._swarmIdentity,
1883
+ onSessionAuth: this._onSessionAuth,
1884
+ onAuthFailure: this._onAuthFailure,
1885
+ blobSync: this.blobSync,
1886
+ disableP2pReplication: this._disableP2pReplication
1887
+ });
1888
+ this._sessions.set(wireParams.remotePeerId, session);
1889
+ for (const feed of this._feeds) {
1890
+ session.replicator.addFeed(feed);
1891
+ }
1892
+ return session;
1893
+ };
1784
1894
  }
1785
- /**
1786
- * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
1787
- *
1788
- * This function will resolve immediately if the pipeline is stalled.
1789
- *
1790
- * @param timeout Timeout in milliseconds to specify the maximum wait time.
1791
- */
1792
- async waitUntilReachedTargetTimeframe({ ctx = new import_context8.Context(void 0, {
1793
- F: __dxlog_file9,
1794
- L: 129
1795
- }), timeout, breakOnStall = true } = {}) {
1796
- (0, import_log6.log)("waitUntilReachedTargetTimeframe", {
1797
- timeout,
1798
- current: this.timeframe,
1799
- target: this.targetTimeframe
1800
- }, {
1801
- F: __dxlog_file9,
1802
- L: 133,
1803
- S: this,
1804
- C: (f, a) => f(...a)
1895
+ };
1896
+ _ts_decorate5([
1897
+ import_log8.logInfo,
1898
+ import_tracing3.trace.info()
1899
+ ], SpaceProtocol.prototype, "_topic", void 0);
1900
+ _ts_decorate5([
1901
+ import_tracing3.trace.info()
1902
+ ], SpaceProtocol.prototype, "_spaceKey", void 0);
1903
+ _ts_decorate5([
1904
+ import_log8.logInfo
1905
+ ], SpaceProtocol.prototype, "_ownPeerKey", null);
1906
+ SpaceProtocol = _ts_decorate5([
1907
+ import_tracing3.trace.resource()
1908
+ ], SpaceProtocol);
1909
+ var AuthStatus;
1910
+ (function(AuthStatus2) {
1911
+ AuthStatus2["INITIAL"] = "INITIAL";
1912
+ AuthStatus2["SUCCESS"] = "SUCCESS";
1913
+ AuthStatus2["FAILURE"] = "FAILURE";
1914
+ })(AuthStatus || (AuthStatus = {}));
1915
+ var SpaceProtocolSession = class {
1916
+ // TODO(dmaretskyi): Allow to pass in extra extensions.
1917
+ constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync, disableP2pReplication }) {
1918
+ this.replicator = new import_teleport_extension_replicator.ReplicatorExtension().setOptions({
1919
+ upload: true
1805
1920
  });
1806
- this._reachedTargetPromise ??= Promise.race([
1807
- this._timeframeClock.update.waitForCondition(() => {
1808
- return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
1809
- }),
1810
- ...breakOnStall ? [
1811
- this.stalled.discardParameter().waitForCount(1)
1812
- ] : []
1813
- ]);
1814
- let done = false;
1815
- if (timeout) {
1816
- return Promise.race([
1817
- (0, import_context8.rejectOnDispose)(ctx),
1818
- (0, import_context8.rejectOnDispose)(this._ctx),
1819
- this._reachedTargetPromise.then(() => {
1820
- done = true;
1821
- this._reachedTarget = true;
1822
- }),
1823
- (0, import_async7.sleepWithContext)(this._ctx, timeout).then(() => {
1824
- if (done) {
1825
- return;
1826
- }
1827
- import_log6.log.warn("waitUntilReachedTargetTimeframe timed out", {
1828
- timeout,
1829
- current: this.timeframe,
1830
- target: this.targetTimeframe,
1831
- dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
1832
- }, {
1833
- F: __dxlog_file9,
1834
- L: 161,
1835
- S: this,
1836
- C: (f, a) => f(...a)
1837
- });
1838
- })
1839
- ]);
1840
- } else {
1841
- return this._reachedTargetPromise;
1921
+ this._authStatus = "INITIAL";
1922
+ this._wireParams = wireParams;
1923
+ this._swarmIdentity = swarmIdentity;
1924
+ this._onSessionAuth = onSessionAuth;
1925
+ this._onAuthFailure = onAuthFailure;
1926
+ this._blobSync = blobSync;
1927
+ this._teleport = new import_teleport3.Teleport(wireParams);
1928
+ this._disableP2pReplication = disableP2pReplication ?? false;
1929
+ }
1930
+ get authStatus() {
1931
+ return this._authStatus;
1932
+ }
1933
+ get stats() {
1934
+ return this._teleport.stats;
1935
+ }
1936
+ get stream() {
1937
+ return this._teleport.stream;
1938
+ }
1939
+ async open(sessionId) {
1940
+ await this._teleport.open(sessionId);
1941
+ this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
1942
+ provider: this._swarmIdentity.credentialProvider,
1943
+ verifier: this._swarmIdentity.credentialAuthenticator,
1944
+ onAuthSuccess: () => {
1945
+ (0, import_log8.log)("Peer authenticated", void 0, {
1946
+ F: __dxlog_file10,
1947
+ L: 282,
1948
+ S: this,
1949
+ C: (f, a) => f(...a)
1950
+ });
1951
+ this._authStatus = "SUCCESS";
1952
+ this._onSessionAuth?.(this._teleport);
1953
+ },
1954
+ onAuthFailure: () => {
1955
+ this._authStatus = "FAILURE";
1956
+ this._onAuthFailure?.(this._teleport);
1957
+ }
1958
+ }));
1959
+ if (!this._disableP2pReplication) {
1960
+ this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
1842
1961
  }
1962
+ this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
1963
+ }
1964
+ async close() {
1965
+ (0, import_log8.log)("close", void 0, {
1966
+ F: __dxlog_file10,
1967
+ L: 301,
1968
+ S: this,
1969
+ C: (f, a) => f(...a)
1970
+ });
1971
+ await this._teleport.close();
1972
+ }
1973
+ async abort() {
1974
+ await this._teleport.abort();
1843
1975
  }
1844
1976
  };
1845
- var Pipeline = class {
1846
- constructor() {
1847
- this._timeframeClock = new TimeframeClock(new import_timeframe2.Timeframe());
1848
- this._feeds = new import_util3.ComplexMap(import_keys4.PublicKey.hash);
1849
- this._state = new PipelineState(this._feeds, this._timeframeClock);
1850
- this._processingTrigger = new import_async7.Trigger().wake();
1851
- this._pauseTrigger = new import_async7.Trigger().wake();
1852
- this._downloads = new import_util3.ComplexMap((value) => import_keys4.PublicKey.hash(value.key));
1853
- this._isStopping = false;
1854
- this._isStarted = false;
1855
- this._isBeingConsumed = false;
1856
- this._isPaused = false;
1977
+ _ts_decorate5([
1978
+ import_log8.logInfo
1979
+ ], SpaceProtocolSession.prototype, "_wireParams", void 0);
1980
+ _ts_decorate5([
1981
+ import_log8.logInfo
1982
+ ], SpaceProtocolSession.prototype, "authStatus", null);
1983
+ var isCollectionQueryMessage = (message) => message.type === import_protocols8.MESSAGE_TYPE_COLLECTION_QUERY;
1984
+ var isCollectionStateMessage = (message) => message.type === import_protocols8.MESSAGE_TYPE_COLLECTION_STATE;
1985
+ function _ts_decorate6(decorators, target, key, desc) {
1986
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1987
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1988
+ r = Reflect.decorate(decorators, target, key, desc);
1989
+ else
1990
+ for (var i = decorators.length - 1; i >= 0; i--)
1991
+ if (d = decorators[i])
1992
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1993
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1994
+ }
1995
+ var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
1996
+ var EchoNetworkAdapter = class extends import_automerge_repo2.NetworkAdapter {
1997
+ constructor(_params) {
1998
+ super();
1999
+ this._params = _params;
2000
+ this._replicators = /* @__PURE__ */ new Set();
2001
+ this._connections = /* @__PURE__ */ new Map();
2002
+ this._lifecycleState = import_context10.LifecycleState.CLOSED;
2003
+ this._connected = new import_async11.Trigger();
1857
2004
  }
1858
- get state() {
1859
- return this._state;
2005
+ connect(peerId, peerMetadata) {
2006
+ this.peerId = peerId;
2007
+ this.peerMetadata = peerMetadata;
2008
+ this._connected.wake();
1860
2009
  }
1861
- get writer() {
1862
- (0, import_invariant7.invariant)(this._writer, "Writer not set.", {
1863
- F: __dxlog_file9,
1864
- L: 243,
1865
- S: this,
1866
- A: [
1867
- "this._writer",
1868
- "'Writer not set.'"
1869
- ]
1870
- });
1871
- return this._writer;
2010
+ send(message) {
2011
+ this._send(message);
1872
2012
  }
1873
- hasFeed(feedKey) {
1874
- return this._feeds.has(feedKey);
2013
+ disconnect() {
1875
2014
  }
1876
- getFeeds() {
1877
- return this._feedSetIterator.feeds;
2015
+ async open() {
2016
+ if (this._lifecycleState === import_context10.LifecycleState.OPEN) {
2017
+ return;
2018
+ }
2019
+ this._lifecycleState = import_context10.LifecycleState.OPEN;
2020
+ (0, import_log12.log)("emit ready", void 0, {
2021
+ F: __dxlog_file11,
2022
+ L: 81,
2023
+ S: this,
2024
+ C: (f, a) => f(...a)
2025
+ });
2026
+ this.emit("ready", {
2027
+ network: this
2028
+ });
1878
2029
  }
1879
- // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
1880
- // which might be opening feeds during the mutation processing, which w
1881
- async addFeed(feed) {
1882
- this._feeds.set(feed.key, feed);
1883
- if (this._feedSetIterator) {
1884
- await this._feedSetIterator.addFeed(feed);
2030
+ async close() {
2031
+ if (this._lifecycleState === import_context10.LifecycleState.CLOSED) {
2032
+ return this;
1885
2033
  }
1886
- if (this._isStarted && !this._isPaused) {
1887
- this._setFeedDownloadState(feed);
2034
+ for (const replicator of this._replicators) {
2035
+ await replicator.disconnect();
1888
2036
  }
2037
+ this._replicators.clear();
2038
+ this._lifecycleState = import_context10.LifecycleState.CLOSED;
1889
2039
  }
1890
- setWriteFeed(feed) {
1891
- (0, import_invariant7.invariant)(!this._writer, "Writer already set.", {
1892
- F: __dxlog_file9,
1893
- L: 270,
2040
+ async whenConnected() {
2041
+ await this._connected.wait({
2042
+ timeout: 1e4
2043
+ });
2044
+ }
2045
+ async addReplicator(replicator) {
2046
+ (0, import_invariant9.invariant)(this._lifecycleState === import_context10.LifecycleState.OPEN, void 0, {
2047
+ F: __dxlog_file11,
2048
+ L: 107,
1894
2049
  S: this,
1895
2050
  A: [
1896
- "!this._writer",
1897
- "'Writer already set.'"
2051
+ "this._lifecycleState === LifecycleState.OPEN",
2052
+ ""
1898
2053
  ]
1899
2054
  });
1900
- (0, import_invariant7.invariant)(feed.properties.writable, "Feed must be writable.", {
1901
- F: __dxlog_file9,
1902
- L: 271,
2055
+ (0, import_invariant9.invariant)(this.peerId, void 0, {
2056
+ F: __dxlog_file11,
2057
+ L: 108,
1903
2058
  S: this,
1904
2059
  A: [
1905
- "feed.properties.writable",
1906
- "'Feed must be writable.'"
2060
+ "this.peerId",
2061
+ ""
1907
2062
  ]
1908
2063
  });
1909
- this._writer = createMappedFeedWriter((payload) => ({
1910
- timeframe: this._timeframeClock.timeframe,
1911
- payload
1912
- }), feed.createFeedWriter());
1913
- }
1914
- async start() {
1915
- (0, import_invariant7.invariant)(!this._isStarted, "Pipeline is already started.", {
1916
- F: __dxlog_file9,
1917
- L: 284,
2064
+ (0, import_invariant9.invariant)(!this._replicators.has(replicator), void 0, {
2065
+ F: __dxlog_file11,
2066
+ L: 109,
1918
2067
  S: this,
1919
2068
  A: [
1920
- "!this._isStarted",
1921
- "'Pipeline is already started.'"
2069
+ "!this._replicators.has(replicator)",
2070
+ ""
1922
2071
  ]
1923
2072
  });
1924
- (0, import_log6.log)("starting...", void 0, {
1925
- F: __dxlog_file9,
1926
- L: 285,
1927
- S: this,
1928
- C: (f, a) => f(...a)
1929
- });
1930
- await this._initIterator();
1931
- await this._feedSetIterator.open();
1932
- this._isStarted = true;
1933
- (0, import_log6.log)("started", void 0, {
1934
- F: __dxlog_file9,
1935
- L: 289,
1936
- S: this,
1937
- C: (f, a) => f(...a)
1938
- });
1939
- if (!this._isPaused) {
1940
- for (const feed of this._feeds.values()) {
1941
- this._setFeedDownloadState(feed);
2073
+ this._replicators.add(replicator);
2074
+ await replicator.connect({
2075
+ peerId: this.peerId,
2076
+ onConnectionOpen: this._onConnectionOpen.bind(this),
2077
+ onConnectionClosed: this._onConnectionClosed.bind(this),
2078
+ onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
2079
+ isDocumentInRemoteCollection: this._params.isDocumentInRemoteCollection,
2080
+ getContainingSpaceForDocument: this._params.getContainingSpaceForDocument,
2081
+ getContainingSpaceIdForDocument: async (documentId) => {
2082
+ const key = await this._params.getContainingSpaceForDocument(documentId);
2083
+ return key ? createIdFromSpaceKey(key) : null;
1942
2084
  }
1943
- }
1944
- }
1945
- async stop() {
1946
- (0, import_log6.log)("stopping...", void 0, {
1947
- F: __dxlog_file9,
1948
- L: 300,
1949
- S: this,
1950
- C: (f, a) => f(...a)
1951
- });
1952
- this._isStopping = true;
1953
- for (const [feed, handle] of this._downloads.entries()) {
1954
- feed.undownload(handle);
1955
- }
1956
- this._downloads.clear();
1957
- await this._feedSetIterator?.close();
1958
- await this._processingTrigger.wait();
1959
- await this._state._ctx.dispose();
1960
- this._state._ctx = new import_context8.Context(void 0, {
1961
- F: __dxlog_file9,
1962
- L: 309
1963
2085
  });
1964
- this._state._reachedTargetPromise = void 0;
1965
- this._state._reachedTarget = false;
1966
- this._isStarted = false;
1967
- (0, import_log6.log)("stopped", void 0, {
1968
- F: __dxlog_file9,
1969
- L: 313,
2086
+ }
2087
+ async removeReplicator(replicator) {
2088
+ (0, import_invariant9.invariant)(this._lifecycleState === import_context10.LifecycleState.OPEN, void 0, {
2089
+ F: __dxlog_file11,
2090
+ L: 128,
1970
2091
  S: this,
1971
- C: (f, a) => f(...a)
2092
+ A: [
2093
+ "this._lifecycleState === LifecycleState.OPEN",
2094
+ ""
2095
+ ]
1972
2096
  });
1973
- }
1974
- /**
1975
- * @param timeframe Timeframe of already processed messages.
1976
- * The pipeline will start processing messages AFTER this timeframe.
1977
- */
1978
- async setCursor(timeframe) {
1979
- (0, import_invariant7.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
1980
- F: __dxlog_file9,
1981
- L: 322,
2097
+ (0, import_invariant9.invariant)(this._replicators.has(replicator), void 0, {
2098
+ F: __dxlog_file11,
2099
+ L: 129,
1982
2100
  S: this,
1983
2101
  A: [
1984
- "!this._isStarted || this._isPaused",
1985
- "'Invalid state.'"
2102
+ "this._replicators.has(replicator)",
2103
+ ""
1986
2104
  ]
1987
2105
  });
1988
- this._state._startTimeframe = timeframe;
1989
- this._timeframeClock.setTimeframe(timeframe);
1990
- if (this._feedSetIterator) {
1991
- await this._feedSetIterator.close();
1992
- await this._initIterator();
1993
- await this._feedSetIterator.open();
2106
+ await replicator.disconnect();
2107
+ this._replicators.delete(replicator);
2108
+ }
2109
+ async shouldAdvertise(peerId, params) {
2110
+ const connection = this._connections.get(peerId);
2111
+ if (!connection) {
2112
+ return false;
1994
2113
  }
2114
+ return connection.connection.shouldAdvertise(params);
1995
2115
  }
1996
- /**
1997
- * Calling pause while processing will cause a deadlock.
1998
- */
1999
- async pause() {
2000
- if (this._isPaused) {
2001
- return;
2116
+ shouldSyncCollection(peerId, params) {
2117
+ const connection = this._connections.get(peerId);
2118
+ if (!connection) {
2119
+ return false;
2002
2120
  }
2003
- this._pauseTrigger.reset();
2004
- await this._processingTrigger.wait();
2005
- this._isPaused = true;
2121
+ return connection.connection.shouldSyncCollection(params);
2006
2122
  }
2007
- async unpause() {
2008
- (0, import_invariant7.invariant)(this._isPaused, "Pipeline is not paused.", {
2009
- F: __dxlog_file9,
2010
- L: 351,
2011
- S: this,
2012
- A: [
2013
- "this._isPaused",
2014
- "'Pipeline is not paused.'"
2015
- ]
2016
- });
2017
- this._pauseTrigger.wake();
2018
- this._isPaused = false;
2019
- for (const feed of this._feeds.values()) {
2020
- this._setFeedDownloadState(feed);
2123
+ queryCollectionState(collectionId, targetId) {
2124
+ const message = {
2125
+ type: "collection-query",
2126
+ senderId: this.peerId,
2127
+ targetId,
2128
+ collectionId
2129
+ };
2130
+ this._send(message);
2131
+ }
2132
+ sendCollectionState(collectionId, targetId, state) {
2133
+ const message = {
2134
+ type: "collection-state",
2135
+ senderId: this.peerId,
2136
+ targetId,
2137
+ collectionId,
2138
+ state
2139
+ };
2140
+ this._send(message);
2141
+ }
2142
+ _send(message) {
2143
+ const connectionEntry = this._connections.get(message.targetId);
2144
+ if (!connectionEntry) {
2145
+ throw new Error("Connection not found.");
2021
2146
  }
2147
+ const writeStart = Date.now();
2148
+ connectionEntry.writer.write(message).then(() => {
2149
+ const durationMs = Date.now() - writeStart;
2150
+ this._params.monitor?.recordMessageSent(message, durationMs);
2151
+ }).catch((err) => {
2152
+ if (connectionEntry.isOpen) {
2153
+ import_log12.log.catch(err, void 0, {
2154
+ F: __dxlog_file11,
2155
+ L: 189,
2156
+ S: this,
2157
+ C: (f, a) => f(...a)
2158
+ });
2159
+ }
2160
+ this._params.monitor?.recordMessageSendingFailed(message);
2161
+ });
2022
2162
  }
2023
- /**
2024
- * Starts to iterate over the ordered messages from the added feeds.
2025
- * Updates the timeframe clock after the message has bee processed.
2026
- */
2027
- async *consume() {
2028
- (0, import_invariant7.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
2029
- F: __dxlog_file9,
2030
- L: 366,
2163
+ // TODO(dmaretskyi): Remove.
2164
+ getPeersInterestedInCollection(collectionId) {
2165
+ return Array.from(this._connections.values()).map((connection) => {
2166
+ return connection.connection.shouldSyncCollection({
2167
+ collectionId
2168
+ }) ? connection.connection.peerId : null;
2169
+ }).filter(import_util7.nonNullable);
2170
+ }
2171
+ _onConnectionOpen(connection) {
2172
+ (0, import_log12.log)("Connection opened", {
2173
+ peerId: connection.peerId
2174
+ }, {
2175
+ F: __dxlog_file11,
2176
+ L: 207,
2031
2177
  S: this,
2032
- A: [
2033
- "!this._isBeingConsumed",
2034
- "'Pipeline is already being consumed.'"
2035
- ]
2178
+ C: (f, a) => f(...a)
2036
2179
  });
2037
- this._isBeingConsumed = true;
2038
- (0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
2039
- F: __dxlog_file9,
2040
- L: 369,
2180
+ (0, import_invariant9.invariant)(!this._connections.has(connection.peerId), void 0, {
2181
+ F: __dxlog_file11,
2182
+ L: 208,
2041
2183
  S: this,
2042
2184
  A: [
2043
- "this._feedSetIterator",
2044
- "'Iterator not initialized.'"
2185
+ "!this._connections.has(connection.peerId as PeerId)",
2186
+ ""
2045
2187
  ]
2046
2188
  });
2047
- let lastFeedSetIterator = this._feedSetIterator;
2048
- let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
2049
- while (!this._isStopping) {
2050
- await this._pauseTrigger.wait();
2051
- if (lastFeedSetIterator !== this._feedSetIterator) {
2052
- (0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
2053
- F: __dxlog_file9,
2054
- L: 378,
2055
- S: this,
2056
- A: [
2057
- "this._feedSetIterator",
2058
- "'Iterator not initialized.'"
2059
- ]
2060
- });
2061
- lastFeedSetIterator = this._feedSetIterator;
2062
- iterable = lastFeedSetIterator[Symbol.asyncIterator]();
2063
- }
2064
- const { done, value } = await iterable.next();
2065
- if (!done) {
2066
- const block = value ?? (0, import_debug2.failUndefined)();
2067
- this._processingTrigger.reset();
2068
- this._timeframeClock.updatePendingTimeframe(import_keys4.PublicKey.from(block.feedKey), block.seq);
2069
- yield block;
2070
- this._processingTrigger.wake();
2071
- this._timeframeClock.updateTimeframe();
2189
+ const reader = connection.readable.getReader();
2190
+ const writer = connection.writable.getWriter();
2191
+ const connectionEntry = {
2192
+ connection,
2193
+ reader,
2194
+ writer,
2195
+ isOpen: true
2196
+ };
2197
+ this._connections.set(connection.peerId, connectionEntry);
2198
+ queueMicrotask(async () => {
2199
+ try {
2200
+ while (true) {
2201
+ const { done, value } = await reader.read();
2202
+ if (done) {
2203
+ break;
2204
+ }
2205
+ this._onMessage(value);
2206
+ }
2207
+ } catch (err) {
2208
+ if (connectionEntry.isOpen) {
2209
+ import_log12.log.catch(err, void 0, {
2210
+ F: __dxlog_file11,
2211
+ L: 227,
2212
+ S: this,
2213
+ C: (f, a) => f(...a)
2214
+ });
2215
+ }
2072
2216
  }
2073
- }
2074
- this._isBeingConsumed = false;
2075
- }
2076
- _setFeedDownloadState(feed) {
2077
- let handle = this._downloads.get(feed);
2078
- if (handle) {
2079
- feed.undownload(handle);
2080
- }
2081
- const timeframe = this._state._startTimeframe;
2082
- const seq = timeframe.get(feed.key) ?? -1;
2083
- (0, import_log6.log)("download", {
2084
- feed: feed.key.truncate(),
2085
- seq,
2086
- length: feed.length
2217
+ });
2218
+ (0, import_log12.log)("emit peer-candidate", {
2219
+ peerId: connection.peerId
2087
2220
  }, {
2088
- F: __dxlog_file9,
2089
- L: 407,
2221
+ F: __dxlog_file11,
2222
+ L: 232,
2090
2223
  S: this,
2091
2224
  C: (f, a) => f(...a)
2092
2225
  });
2093
- handle = feed.download({
2094
- start: seq + 1,
2095
- linear: true
2096
- }, (err, data) => {
2097
- if (err) {
2098
- } else {
2099
- import_log6.log.info("downloaded", {
2100
- data
2101
- }, {
2102
- F: __dxlog_file9,
2103
- L: 412,
2104
- S: this,
2105
- C: (f, a) => f(...a)
2106
- });
2107
- }
2108
- });
2109
- this._downloads.set(feed, handle);
2226
+ this._emitPeerCandidate(connection);
2227
+ this._params.monitor?.recordPeerConnected(connection.peerId);
2110
2228
  }
2111
- async _initIterator() {
2112
- this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
2113
- start: startAfter(this._timeframeClock.timeframe),
2114
- stallTimeout: 1e3
2115
- });
2116
- this._feedSetIterator.stalled.on((iterator) => {
2117
- import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
2118
- F: __dxlog_file9,
2119
- L: 426,
2120
- S: this,
2121
- C: (f, a) => f(...a)
2122
- });
2123
- this._state.stalled.emit();
2124
- });
2125
- for (const feed of this._feeds.values()) {
2126
- await this._feedSetIterator.addFeed(feed);
2229
+ _onMessage(message) {
2230
+ if (isCollectionQueryMessage(message)) {
2231
+ this._params.onCollectionStateQueried(message.collectionId, message.senderId);
2232
+ } else if (isCollectionStateMessage(message)) {
2233
+ this._params.onCollectionStateReceived(message.collectionId, message.senderId, message.state);
2234
+ } else {
2235
+ this.emit("message", message);
2127
2236
  }
2237
+ this._params.monitor?.recordMessageReceived(message);
2128
2238
  }
2129
- };
2130
- _ts_decorate4([
2131
- import_async7.synchronized
2132
- ], Pipeline.prototype, "start", null);
2133
- _ts_decorate4([
2134
- import_async7.synchronized
2135
- ], Pipeline.prototype, "stop", null);
2136
- _ts_decorate4([
2137
- import_async7.synchronized
2138
- ], Pipeline.prototype, "setCursor", null);
2139
- _ts_decorate4([
2140
- import_async7.synchronized
2141
- ], Pipeline.prototype, "pause", null);
2142
- _ts_decorate4([
2143
- import_async7.synchronized
2144
- ], Pipeline.prototype, "unpause", null);
2145
- function _ts_decorate5(decorators, target, key, desc) {
2146
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2147
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2148
- r = Reflect.decorate(decorators, target, key, desc);
2149
- else
2150
- for (var i = decorators.length - 1; i >= 0; i--)
2151
- if (d = decorators[i])
2152
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2153
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2154
- }
2155
- var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
2156
- var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
2157
- var CONTROL_PIPELINE_SNAPSHOT_DELAY = 1e4;
2158
- var USE_SNAPSHOTS = true;
2159
- var ControlPipeline = class {
2160
- constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
2161
- this._ctx = new import_context10.Context(void 0, {
2162
- F: __dxlog_file10,
2163
- L: 47
2164
- });
2165
- this._lastTimeframeSaveTime = Date.now();
2166
- this.onFeedAdmitted = new import_util5.Callback();
2167
- this._usage = new import_tracing3.TimeUsageCounter();
2168
- this._mutations = new import_tracing3.TimeSeriesCounter();
2169
- this._snapshotTask = new import_async9.DeferredTask(this._ctx, async () => {
2170
- await (0, import_async9.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
2171
- await this._saveSnapshot();
2239
+ /**
2240
+ * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
2241
+ * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
2242
+ */
2243
+ _onConnectionAuthScopeChanged(connection) {
2244
+ (0, import_log12.log)("Connection auth scope changed", {
2245
+ peerId: connection.peerId
2246
+ }, {
2247
+ F: __dxlog_file11,
2248
+ L: 253,
2249
+ S: this,
2250
+ C: (f, a) => f(...a)
2172
2251
  });
2173
- this._spaceKey = spaceKey;
2174
- this._metadata = metadataStore;
2175
- this._pipeline = new Pipeline();
2176
- void this._pipeline.addFeed(genesisFeed);
2177
- this._spaceStateMachine = new import_credentials2.SpaceStateMachine(spaceKey);
2178
- this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
2179
- (0, import_log9.log)("feed admitted", {
2180
- key: info.key
2181
- }, {
2182
- F: __dxlog_file10,
2183
- L: 82,
2184
- S: this,
2185
- C: (f, a) => f(...a)
2186
- });
2187
- if (info.assertion.designation === import_credentials3.AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
2188
- queueMicrotask(async () => {
2189
- try {
2190
- const feed = await feedProvider(info.key);
2191
- if (!this._pipeline.hasFeed(feed.key)) {
2192
- await this._pipeline.addFeed(feed);
2193
- }
2194
- } catch (err) {
2195
- import_log9.log.catch(err, void 0, {
2196
- F: __dxlog_file10,
2197
- L: 93,
2198
- S: this,
2199
- C: (f, a) => f(...a)
2200
- });
2201
- }
2202
- });
2203
- }
2204
- await this.onFeedAdmitted.callIfSet(info);
2252
+ const entry = this._connections.get(connection.peerId);
2253
+ (0, import_invariant9.invariant)(entry, void 0, {
2254
+ F: __dxlog_file11,
2255
+ L: 255,
2256
+ S: this,
2257
+ A: [
2258
+ "entry",
2259
+ ""
2260
+ ]
2205
2261
  });
2206
- this.onMemberRoleChanged = this._spaceStateMachine.onMemberRoleChanged;
2207
- this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
2208
- this.onDelegatedInvitation = this._spaceStateMachine.onDelegatedInvitation;
2209
- this.onDelegatedInvitationRemoved = this._spaceStateMachine.onDelegatedInvitationRemoved;
2210
- }
2211
- get spaceState() {
2212
- return this._spaceStateMachine;
2213
- }
2214
- get pipeline() {
2215
- return this._pipeline;
2216
- }
2217
- async setWriteFeed(feed) {
2218
- await this._pipeline.addFeed(feed);
2219
- this._pipeline.setWriteFeed(feed);
2220
- }
2221
- async start() {
2222
- const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
2223
- (0, import_log9.log)("load snapshot", {
2224
- key: this._spaceKey,
2225
- present: !!snapshot,
2226
- tf: snapshot?.timeframe
2262
+ this.emit("peer-disconnected", {
2263
+ peerId: connection.peerId
2264
+ });
2265
+ this._emitPeerCandidate(connection);
2266
+ }
2267
+ _onConnectionClosed(connection) {
2268
+ (0, import_log12.log)("Connection closed", {
2269
+ peerId: connection.peerId
2227
2270
  }, {
2228
- F: __dxlog_file10,
2229
- L: 123,
2271
+ F: __dxlog_file11,
2272
+ L: 261,
2230
2273
  S: this,
2231
2274
  C: (f, a) => f(...a)
2232
2275
  });
2233
- if (USE_SNAPSHOTS && snapshot) {
2234
- await this._processSnapshot(snapshot);
2235
- }
2236
- (0, import_log9.log)("starting...", void 0, {
2237
- F: __dxlog_file10,
2238
- L: 128,
2276
+ const entry = this._connections.get(connection.peerId);
2277
+ (0, import_invariant9.invariant)(entry, void 0, {
2278
+ F: __dxlog_file11,
2279
+ L: 263,
2239
2280
  S: this,
2240
- C: (f, a) => f(...a)
2281
+ A: [
2282
+ "entry",
2283
+ ""
2284
+ ]
2241
2285
  });
2242
- setTimeout(async () => {
2243
- void this._consumePipeline(new import_context10.Context(void 0, {
2244
- F: __dxlog_file10,
2245
- L: 130
2246
- }));
2286
+ entry.isOpen = false;
2287
+ this.emit("peer-disconnected", {
2288
+ peerId: connection.peerId
2247
2289
  });
2248
- await this._pipeline.start();
2249
- (0, import_log9.log)("started", void 0, {
2250
- F: __dxlog_file10,
2251
- L: 134,
2290
+ this._params.monitor?.recordPeerDisconnected(connection.peerId);
2291
+ void entry.reader.cancel().catch((err) => import_log12.log.catch(err, void 0, {
2292
+ F: __dxlog_file11,
2293
+ L: 269,
2252
2294
  S: this,
2253
2295
  C: (f, a) => f(...a)
2254
- });
2255
- }
2256
- async _processSnapshot(snapshot) {
2257
- await this._pipeline.setCursor(snapshot.timeframe);
2258
- for (const message of snapshot.messages ?? []) {
2259
- const result = await this._spaceStateMachine.process(message.credential, {
2260
- sourceFeed: message.feedKey,
2261
- skipVerification: true
2262
- });
2263
- if (!result) {
2264
- import_log9.log.warn("credential processing failed from snapshot", {
2265
- message
2266
- }, {
2267
- F: __dxlog_file10,
2268
- L: 147,
2269
- S: this,
2270
- C: (f, a) => f(...a)
2271
- });
2272
- }
2273
- }
2274
- }
2275
- async _saveSnapshot() {
2276
- await this._pipeline.pause();
2277
- const snapshot = {
2278
- timeframe: this._pipeline.state.timeframe,
2279
- messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
2280
- feedKey: entry.sourceFeed,
2281
- credential: entry.credential
2282
- }))
2283
- };
2284
- await this._pipeline.unpause();
2285
- (0, import_log9.log)("save snapshot", {
2286
- key: this._spaceKey,
2287
- snapshot
2288
- }, {
2289
- F: __dxlog_file10,
2290
- L: 163,
2296
+ }));
2297
+ void entry.writer.abort().catch((err) => import_log12.log.catch(err, void 0, {
2298
+ F: __dxlog_file11,
2299
+ L: 270,
2291
2300
  S: this,
2292
2301
  C: (f, a) => f(...a)
2302
+ }));
2303
+ this._connections.delete(connection.peerId);
2304
+ }
2305
+ _emitPeerCandidate(connection) {
2306
+ this.emit("peer-candidate", {
2307
+ peerId: connection.peerId,
2308
+ peerMetadata: createEchoPeerMetadata()
2293
2309
  });
2294
- await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
2295
2310
  }
2296
- async _consumePipeline(ctx) {
2297
- for await (const msg of this._pipeline.consume()) {
2298
- const span = this._usage.beginRecording();
2299
- this._mutations.inc();
2300
- try {
2301
- await this._processMessage(ctx, msg);
2302
- } catch (err) {
2303
- import_log9.log.catch(err, void 0, {
2304
- F: __dxlog_file10,
2305
- L: 176,
2306
- S: this,
2307
- C: (f, a) => f(...a)
2308
- });
2309
- }
2310
- span.end();
2311
- }
2311
+ };
2312
+ _ts_decorate6([
2313
+ import_async11.synchronized
2314
+ ], EchoNetworkAdapter.prototype, "open", null);
2315
+ _ts_decorate6([
2316
+ import_async11.synchronized
2317
+ ], EchoNetworkAdapter.prototype, "close", null);
2318
+ _ts_decorate6([
2319
+ import_async11.synchronized
2320
+ ], EchoNetworkAdapter.prototype, "addReplicator", null);
2321
+ _ts_decorate6([
2322
+ import_async11.synchronized
2323
+ ], EchoNetworkAdapter.prototype, "removeReplicator", null);
2324
+ var createEchoPeerMetadata = () => ({
2325
+ // TODO(dmaretskyi): Refactor this.
2326
+ dxos_peerSource: "EchoNetworkAdapter"
2327
+ });
2328
+ var isEchoPeerMetadata = (metadata) => metadata?.dxos_peerSource === "EchoNetworkAdapter";
2329
+ var HeadsStore = class {
2330
+ constructor({ db }) {
2331
+ this._db = db;
2312
2332
  }
2313
- async _processMessage(ctx, msg) {
2314
- (0, import_log9.log)("processing", {
2315
- key: msg.feedKey,
2316
- seq: msg.seq
2317
- }, {
2318
- F: __dxlog_file10,
2319
- L: 186,
2320
- S: this,
2321
- C: (f, a) => f(...a)
2333
+ setHeads(documentId, heads, batch) {
2334
+ batch.put(documentId, heads, {
2335
+ sublevel: this._db,
2336
+ keyEncoding: "utf8",
2337
+ valueEncoding: import_indexing.headsEncoding
2322
2338
  });
2323
- if (msg.data.payload.credential) {
2324
- const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
2325
- const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
2326
- sourceFeed: import_keys6.PublicKey.from(msg.feedKey)
2339
+ }
2340
+ // TODO(dmaretskyi): Make batched.
2341
+ async getHeads(documentIds) {
2342
+ return this._db.getMany(documentIds, {
2343
+ keyEncoding: "utf8",
2344
+ valueEncoding: import_indexing.headsEncoding
2345
+ });
2346
+ }
2347
+ };
2348
+ var LevelDBStorageAdapter = class extends import_context11.Resource {
2349
+ constructor(_params) {
2350
+ super();
2351
+ this._params = _params;
2352
+ }
2353
+ async load(keyArray) {
2354
+ try {
2355
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2356
+ return void 0;
2357
+ }
2358
+ const startMs = Date.now();
2359
+ const chunk = await this._params.db.get(keyArray, {
2360
+ ...encodingOptions
2327
2361
  });
2328
- timer.end();
2329
- if (!result) {
2330
- import_log9.log.warn("processing failed", {
2331
- msg
2332
- }, {
2333
- F: __dxlog_file10,
2334
- L: 195,
2335
- S: this,
2336
- C: (f, a) => f(...a)
2337
- });
2338
- } else {
2339
- await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
2362
+ this._params.monitor?.recordBytesLoaded(chunk.byteLength);
2363
+ this._params.monitor?.recordLoadDuration(Date.now() - startMs);
2364
+ return chunk;
2365
+ } catch (err) {
2366
+ if (isLevelDbNotFoundError(err)) {
2367
+ return void 0;
2340
2368
  }
2341
- this._snapshotTask.schedule();
2369
+ throw err;
2342
2370
  }
2343
2371
  }
2344
- async _noteTargetStateIfNeeded(timeframe) {
2345
- if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
2346
- this._lastTimeframeSaveTime = Date.now();
2347
- await this._saveTargetTimeframe(timeframe);
2372
+ async save(keyArray, binary) {
2373
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2374
+ return void 0;
2348
2375
  }
2349
- }
2350
- async stop() {
2351
- (0, import_log9.log)("stopping...", void 0, {
2352
- F: __dxlog_file10,
2353
- L: 215,
2354
- S: this,
2355
- C: (f, a) => f(...a)
2376
+ const startMs = Date.now();
2377
+ const batch = this._params.db.batch();
2378
+ await this._params.callbacks?.beforeSave?.({
2379
+ path: keyArray,
2380
+ batch
2356
2381
  });
2357
- await this._ctx.dispose();
2358
- await this._pipeline.stop();
2359
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
2360
- (0, import_log9.log)("stopped", void 0, {
2361
- F: __dxlog_file10,
2362
- L: 219,
2363
- S: this,
2364
- C: (f, a) => f(...a)
2382
+ batch.put(keyArray, Buffer.from(binary), {
2383
+ ...encodingOptions
2365
2384
  });
2385
+ await batch.write();
2386
+ this._params.monitor?.recordBytesStored(binary.byteLength);
2387
+ await this._params.callbacks?.afterSave?.(keyArray);
2388
+ this._params.monitor?.recordStoreDuration(Date.now() - startMs);
2366
2389
  }
2367
- async _saveTargetTimeframe(timeframe) {
2368
- try {
2369
- const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
2370
- await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
2371
- this._targetTimeframe = newTimeframe;
2372
- } catch (err) {
2373
- (0, import_log9.log)(err, void 0, {
2374
- F: __dxlog_file10,
2375
- L: 228,
2376
- S: this,
2377
- C: (f, a) => f(...a)
2390
+ async remove(keyArray) {
2391
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2392
+ return void 0;
2393
+ }
2394
+ await this._params.db.del(keyArray, {
2395
+ ...encodingOptions
2396
+ });
2397
+ }
2398
+ async loadRange(keyPrefix) {
2399
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2400
+ return [];
2401
+ }
2402
+ const startMs = Date.now();
2403
+ const result = [];
2404
+ for await (const [key, value] of this._params.db.iterator({
2405
+ gte: keyPrefix,
2406
+ lte: [
2407
+ ...keyPrefix,
2408
+ "\uFFFF"
2409
+ ],
2410
+ ...encodingOptions
2411
+ })) {
2412
+ result.push({
2413
+ key,
2414
+ data: value
2415
+ });
2416
+ this._params.monitor?.recordBytesLoaded(value.byteLength);
2417
+ }
2418
+ this._params.monitor?.recordLoadDuration(Date.now() - startMs);
2419
+ return result;
2420
+ }
2421
+ async removeRange(keyPrefix) {
2422
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2423
+ return void 0;
2424
+ }
2425
+ const batch = this._params.db.batch();
2426
+ for await (const [key] of this._params.db.iterator({
2427
+ gte: keyPrefix,
2428
+ lte: [
2429
+ ...keyPrefix,
2430
+ "\uFFFF"
2431
+ ],
2432
+ ...encodingOptions
2433
+ })) {
2434
+ batch.del(key, {
2435
+ ...encodingOptions
2378
2436
  });
2379
2437
  }
2438
+ await batch.write();
2380
2439
  }
2381
2440
  };
2382
- _ts_decorate5([
2383
- import_tracing3.trace.metricsCounter()
2384
- ], ControlPipeline.prototype, "_usage", void 0);
2385
- _ts_decorate5([
2386
- import_tracing3.trace.metricsCounter()
2387
- ], ControlPipeline.prototype, "_mutations", void 0);
2388
- _ts_decorate5([
2389
- import_tracing3.trace.span({
2390
- showInBrowserTimeline: true
2391
- })
2392
- ], ControlPipeline.prototype, "start", null);
2393
- _ts_decorate5([
2394
- import_tracing3.trace.span()
2395
- ], ControlPipeline.prototype, "_consumePipeline", null);
2396
- _ts_decorate5([
2397
- import_tracing3.trace.span()
2398
- ], ControlPipeline.prototype, "_processMessage", null);
2399
- ControlPipeline = _ts_decorate5([
2400
- import_tracing3.trace.resource(),
2401
- (0, import_async9.trackLeaks)("start", "stop")
2402
- ], ControlPipeline);
2403
- function _ts_decorate6(decorators, target, key, desc) {
2441
+ var keyEncoder = {
2442
+ encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
2443
+ decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
2444
+ format: "buffer"
2445
+ };
2446
+ var encodingOptions = {
2447
+ keyEncoding: keyEncoder,
2448
+ valueEncoding: "buffer"
2449
+ };
2450
+ var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
2451
+ function _ts_decorate7(decorators, target, key, desc) {
2404
2452
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2405
2453
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2406
2454
  r = Reflect.decorate(decorators, target, key, desc);
@@ -2410,495 +2458,455 @@ function _ts_decorate6(decorators, target, key, desc) {
2410
2458
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2411
2459
  return c > 3 && r && Object.defineProperty(target, key, r), r;
2412
2460
  }
2413
- var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
2414
- var Space = class extends import_context9.Resource {
2415
- constructor(params) {
2461
+ var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
2462
+ var AutomergeHost = class extends import_context9.Resource {
2463
+ constructor({ db, indexMetadataStore, dataMonitor }) {
2416
2464
  super();
2417
- this.onCredentialProcessed = new import_util4.Callback();
2418
- this.stateUpdate = new import_async8.Event();
2419
- (0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
2420
- F: __dxlog_file11,
2421
- L: 76,
2422
- S: this,
2423
- A: [
2424
- "params.spaceKey && params.feedProvider",
2425
- ""
2426
- ]
2465
+ this._collectionSynchronizer = new CollectionSynchronizer({
2466
+ queryCollectionState: this._queryCollectionState.bind(this),
2467
+ sendCollectionState: this._sendCollectionState.bind(this),
2468
+ shouldSyncCollection: this._shouldSyncCollection.bind(this)
2427
2469
  });
2428
- this._id = params.id;
2429
- this._key = params.spaceKey;
2430
- this._genesisFeedKey = params.genesisFeed.key;
2431
- this._feedProvider = params.feedProvider;
2432
- this._snapshotManager = params.snapshotManager;
2433
- this._controlPipeline = new ControlPipeline({
2434
- spaceKey: params.spaceKey,
2435
- genesisFeed: params.genesisFeed,
2436
- feedProvider: params.feedProvider,
2437
- metadataStore: params.metadataStore
2470
+ this._db = db;
2471
+ this._storage = new LevelDBStorageAdapter({
2472
+ db: db.sublevel("automerge"),
2473
+ callbacks: {
2474
+ beforeSave: async (params) => this._beforeSave(params),
2475
+ afterSave: async (key) => this._afterSave(key)
2476
+ },
2477
+ monitor: dataMonitor
2438
2478
  });
2439
- this._controlPipeline.onFeedAdmitted.set(async (info) => {
2440
- const sparse = info.assertion.designation === import_credentials.AdmittedFeed.Designation.DATA;
2441
- if (!info.key.equals(params.genesisFeed.key)) {
2442
- queueMicrotask(async () => {
2443
- this.protocol.addFeed(await params.feedProvider(info.key, {
2444
- sparse
2445
- }));
2446
- });
2447
- }
2479
+ this._echoNetworkAdapter = new EchoNetworkAdapter({
2480
+ getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
2481
+ isDocumentInRemoteCollection: this._isDocumentInRemoteCollection.bind(this),
2482
+ onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
2483
+ onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
2484
+ monitor: dataMonitor
2448
2485
  });
2449
- this._controlPipeline.onCredentialProcessed.set(async (credential) => {
2450
- await this.onCredentialProcessed.callIfSet(credential);
2451
- (0, import_log8.log)("onCredentialProcessed", {
2452
- credential
2453
- }, {
2454
- F: __dxlog_file11,
2455
- L: 104,
2456
- S: this,
2457
- C: (f, a) => f(...a)
2458
- });
2459
- this.stateUpdate.emit();
2486
+ this._headsStore = new HeadsStore({
2487
+ db: db.sublevel("heads")
2460
2488
  });
2461
- this._controlPipeline.onDelegatedInvitation.set(async (invitation) => {
2462
- (0, import_log8.log)("onDelegatedInvitation", {
2463
- invitation
2464
- }, {
2465
- F: __dxlog_file11,
2466
- L: 108,
2467
- S: this,
2468
- C: (f, a) => f(...a)
2469
- });
2470
- await params.onDelegatedInvitationStatusChange(invitation, true);
2489
+ this._indexMetadataStore = indexMetadataStore;
2490
+ }
2491
+ async _open() {
2492
+ this._peerId = `host-${import_keys8.PublicKey.random().toHex()}`;
2493
+ await this._storage.open?.();
2494
+ this._repo = new import_automerge_repo.Repo({
2495
+ peerId: this._peerId,
2496
+ sharePolicy: this._sharePolicy.bind(this),
2497
+ storage: this._storage,
2498
+ network: [
2499
+ // Upstream swarm.
2500
+ this._echoNetworkAdapter
2501
+ ]
2471
2502
  });
2472
- this._controlPipeline.onDelegatedInvitationRemoved.set(async (invitation) => {
2473
- (0, import_log8.log)("onDelegatedInvitationRemoved", {
2474
- invitation
2503
+ import_async10.Event.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => this._onPeerConnected(e.peerId));
2504
+ import_async10.Event.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => this._onPeerDisconnected(e.peerId));
2505
+ this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
2506
+ this._onRemoteCollectionStateUpdated(collectionId, peerId);
2507
+ });
2508
+ await this._echoNetworkAdapter.open();
2509
+ await this._collectionSynchronizer.open();
2510
+ await this._echoNetworkAdapter.open();
2511
+ await this._echoNetworkAdapter.whenConnected();
2512
+ }
2513
+ async _close() {
2514
+ await this._collectionSynchronizer.close();
2515
+ await this._storage.close?.();
2516
+ await this._echoNetworkAdapter.close();
2517
+ await this._ctx.dispose();
2518
+ }
2519
+ /**
2520
+ * @deprecated To be abstracted away.
2521
+ */
2522
+ get repo() {
2523
+ return this._repo;
2524
+ }
2525
+ get peerId() {
2526
+ return this._peerId;
2527
+ }
2528
+ get loadedDocsCount() {
2529
+ return Object.keys(this._repo.handles).length;
2530
+ }
2531
+ async addReplicator(replicator) {
2532
+ await this._echoNetworkAdapter.addReplicator(replicator);
2533
+ }
2534
+ async removeReplicator(replicator) {
2535
+ await this._echoNetworkAdapter.removeReplicator(replicator);
2536
+ }
2537
+ /**
2538
+ * Loads the document handle from the repo and waits for it to be ready.
2539
+ */
2540
+ async loadDoc(ctx, documentId, opts) {
2541
+ let handle;
2542
+ if (typeof documentId === "string") {
2543
+ handle = this._repo.handles[documentId];
2544
+ }
2545
+ if (!handle) {
2546
+ handle = this._repo.find(documentId);
2547
+ }
2548
+ if (!handle.isReady()) {
2549
+ if (!opts?.timeout) {
2550
+ await (0, import_context9.cancelWithContext)(ctx, handle.whenReady());
2551
+ } else {
2552
+ await (0, import_context9.cancelWithContext)(ctx, (0, import_async10.asyncTimeout)(handle.whenReady(), opts.timeout));
2553
+ }
2554
+ }
2555
+ return handle;
2556
+ }
2557
+ /**
2558
+ * Create new persisted document.
2559
+ */
2560
+ createDoc(initialValue, opts) {
2561
+ if (opts?.preserveHistory) {
2562
+ if (!(0, import_automerge3.isAutomerge)(initialValue)) {
2563
+ throw new TypeError("Initial value must be an Automerge document");
2564
+ }
2565
+ return this._repo.import((0, import_automerge3.save)(initialValue));
2566
+ } else {
2567
+ return this._repo.create(initialValue);
2568
+ }
2569
+ }
2570
+ async waitUntilHeadsReplicated(heads) {
2571
+ const entries = heads.entries;
2572
+ if (!entries?.length) {
2573
+ return;
2574
+ }
2575
+ const documentIds = entries.map((entry) => entry.documentId);
2576
+ const documentHeads = await this.getHeads(documentIds);
2577
+ const headsToWait = entries.filter((entry, index) => {
2578
+ const targetHeads = entry.heads;
2579
+ if (!targetHeads || targetHeads.length === 0) {
2580
+ return false;
2581
+ }
2582
+ const currentHeads = documentHeads[index];
2583
+ return !(currentHeads !== null && (0, import_automerge3.equals)(currentHeads, targetHeads));
2584
+ });
2585
+ if (headsToWait.length > 0) {
2586
+ await Promise.all(headsToWait.map(async (entry, index) => {
2587
+ const handle = await this.loadDoc(import_context9.Context.default(void 0, {
2588
+ F: __dxlog_file12,
2589
+ L: 227
2590
+ }), entry.documentId);
2591
+ await waitForHeads(handle, entry.heads);
2592
+ }));
2593
+ }
2594
+ await this._repo.flush(documentIds.filter((documentId) => !!this._repo.handles[documentId]));
2595
+ }
2596
+ async reIndexHeads(documentIds) {
2597
+ for (const documentId of documentIds) {
2598
+ import_log11.log.info("re-indexing heads for document", {
2599
+ documentId
2475
2600
  }, {
2476
- F: __dxlog_file11,
2477
- L: 112,
2601
+ F: __dxlog_file12,
2602
+ L: 239,
2478
2603
  S: this,
2479
2604
  C: (f, a) => f(...a)
2480
2605
  });
2481
- await params.onDelegatedInvitationStatusChange(invitation, false);
2482
- });
2483
- this._controlPipeline.onMemberRoleChanged.set(async (changedMembers) => {
2484
- (0, import_log8.log)("onMemberRoleChanged", () => ({
2485
- changedMembers: changedMembers.map((m) => [
2486
- m.key,
2487
- m.role
2488
- ])
2489
- }), {
2490
- F: __dxlog_file11,
2491
- L: 116,
2606
+ const handle = this._repo.find(documentId);
2607
+ await handle.whenReady([
2608
+ "ready",
2609
+ "requesting"
2610
+ ]);
2611
+ if (handle.inState([
2612
+ "requesting"
2613
+ ])) {
2614
+ import_log11.log.warn("document is not available locally, skipping", {
2615
+ documentId
2616
+ }, {
2617
+ F: __dxlog_file12,
2618
+ L: 243,
2619
+ S: this,
2620
+ C: (f, a) => f(...a)
2621
+ });
2622
+ continue;
2623
+ }
2624
+ const doc = handle.docSync();
2625
+ (0, import_invariant8.invariant)(doc, void 0, {
2626
+ F: __dxlog_file12,
2627
+ L: 248,
2492
2628
  S: this,
2493
- C: (f, a) => f(...a)
2629
+ A: [
2630
+ "doc",
2631
+ ""
2632
+ ]
2494
2633
  });
2495
- await params.onMemberRolesChanged(changedMembers);
2634
+ const heads = (0, import_automerge3.getHeads)(doc);
2635
+ const batch = this._db.batch();
2636
+ this._headsStore.setHeads(documentId, heads, batch);
2637
+ await batch.write();
2638
+ }
2639
+ import_log11.log.info("done re-indexing heads", void 0, {
2640
+ F: __dxlog_file12,
2641
+ L: 255,
2642
+ S: this,
2643
+ C: (f, a) => f(...a)
2496
2644
  });
2497
- this.protocol = params.protocol;
2498
- this.protocol.addFeed(params.genesisFeed);
2499
- }
2500
- get id() {
2501
- return this._id;
2502
- }
2503
- get key() {
2504
- return this._key;
2505
- }
2506
- get isOpen() {
2507
- return this._lifecycleState === import_context9.LifecycleState.OPEN;
2508
- }
2509
- get genesisFeedKey() {
2510
- return this._genesisFeedKey;
2511
2645
  }
2512
- get controlFeedKey() {
2513
- return this._controlFeed?.key;
2646
+ // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
2647
+ // Hosts, running in the worker, don't share documents unless requested by other peers.
2648
+ // NOTE: If both peers return sharePolicy=false the replication will not happen
2649
+ // https://github.com/automerge/automerge-repo/pull/292
2650
+ async _sharePolicy(peerId, documentId) {
2651
+ if (peerId.startsWith("client-")) {
2652
+ return false;
2653
+ }
2654
+ if (!documentId) {
2655
+ return false;
2656
+ }
2657
+ const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
2658
+ if (isEchoPeerMetadata(peerMetadata)) {
2659
+ return this._echoNetworkAdapter.shouldAdvertise(peerId, {
2660
+ documentId
2661
+ });
2662
+ }
2663
+ return false;
2514
2664
  }
2515
- get dataFeedKey() {
2516
- return this._dataFeed?.key;
2665
+ async _beforeSave({ path, batch }) {
2666
+ const handle = this._repo.handles[path[0]];
2667
+ if (!handle) {
2668
+ return;
2669
+ }
2670
+ const doc = handle.docSync();
2671
+ if (!doc) {
2672
+ return;
2673
+ }
2674
+ const heads = (0, import_automerge3.getHeads)(doc);
2675
+ this._headsStore.setHeads(handle.documentId, heads, batch);
2676
+ const spaceKey = getSpaceKeyFromDoc(doc) ?? void 0;
2677
+ const objectIds = Object.keys(doc.objects ?? {});
2678
+ const encodedIds = objectIds.map((objectId) => import_protocols7.objectPointerCodec.encode({
2679
+ documentId: handle.documentId,
2680
+ objectId,
2681
+ spaceKey
2682
+ }));
2683
+ const idToLastHash = new Map(encodedIds.map((id) => [
2684
+ id,
2685
+ heads
2686
+ ]));
2687
+ this._indexMetadataStore.markDirty(idToLastHash, batch);
2517
2688
  }
2518
- get spaceState() {
2519
- return this._controlPipeline.spaceState;
2689
+ _shouldSyncCollection(collectionId, peerId) {
2690
+ const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
2691
+ if (isEchoPeerMetadata(peerMetadata)) {
2692
+ return this._echoNetworkAdapter.shouldSyncCollection(peerId, {
2693
+ collectionId
2694
+ });
2695
+ }
2696
+ return false;
2520
2697
  }
2521
2698
  /**
2522
- * @test-only
2699
+ * Called by AutomergeStorageAdapter after levelDB batch commit.
2523
2700
  */
2524
- get controlPipeline() {
2525
- return this._controlPipeline.pipeline;
2701
+ async _afterSave(path) {
2702
+ this._indexMetadataStore.notifyMarkedDirty();
2703
+ const documentId = path[0];
2704
+ const document = this._repo.handles[documentId]?.docSync();
2705
+ if (document) {
2706
+ const heads = (0, import_automerge3.getHeads)(document);
2707
+ this._onHeadsChanged(documentId, heads);
2708
+ }
2526
2709
  }
2527
- get snapshotManager() {
2528
- return this._snapshotManager;
2710
+ _automergePeers() {
2711
+ return this._repo.peers;
2529
2712
  }
2530
- async setControlFeed(feed) {
2531
- (0, import_invariant9.invariant)(!this._controlFeed, "Control feed already set.", {
2532
- F: __dxlog_file11,
2533
- L: 169,
2534
- S: this,
2535
- A: [
2536
- "!this._controlFeed",
2537
- "'Control feed already set.'"
2538
- ]
2539
- });
2540
- this._controlFeed = feed;
2541
- await this._controlPipeline.setWriteFeed(feed);
2542
- return this;
2713
+ async _isDocumentInRemoteCollection(params) {
2714
+ for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
2715
+ const remoteCollections = this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
2716
+ const remotePeerDocs = remoteCollections.get(params.peerId)?.documents;
2717
+ if (remotePeerDocs && params.documentId in remotePeerDocs) {
2718
+ return true;
2719
+ }
2720
+ }
2721
+ return false;
2543
2722
  }
2544
- async setDataFeed(feed) {
2545
- (0, import_invariant9.invariant)(!this._dataFeed, "Data feed already set.", {
2546
- F: __dxlog_file11,
2547
- L: 176,
2548
- S: this,
2549
- A: [
2550
- "!this._dataFeed",
2551
- "'Data feed already set.'"
2552
- ]
2553
- });
2554
- this._dataFeed = feed;
2555
- return this;
2723
+ async _getContainingSpaceForDocument(documentId) {
2724
+ const doc = this._repo.handles[documentId]?.docSync();
2725
+ if (!doc) {
2726
+ return null;
2727
+ }
2728
+ const spaceKeyHex = getSpaceKeyFromDoc(doc);
2729
+ if (!spaceKeyHex) {
2730
+ return null;
2731
+ }
2732
+ return import_keys8.PublicKey.from(spaceKeyHex);
2556
2733
  }
2557
2734
  /**
2558
- * Use for diagnostics.
2735
+ * Flush documents to disk.
2559
2736
  */
2560
- getControlFeeds() {
2561
- return Array.from(this._controlPipeline.spaceState.feeds.values());
2562
- }
2563
- async _open(ctx) {
2564
- (0, import_log8.log)("opening...", void 0, {
2565
- F: __dxlog_file11,
2566
- L: 190,
2567
- S: this,
2568
- C: (f, a) => f(...a)
2569
- });
2570
- await this._controlPipeline.start();
2571
- await this.protocol.start();
2572
- (0, import_log8.log)("opened", void 0, {
2573
- F: __dxlog_file11,
2574
- L: 196,
2575
- S: this,
2576
- C: (f, a) => f(...a)
2577
- });
2578
- }
2579
- async _close() {
2580
- (0, import_log8.log)("closing...", {
2581
- key: this._key
2582
- }, {
2583
- F: __dxlog_file11,
2584
- L: 201,
2585
- S: this,
2586
- C: (f, a) => f(...a)
2587
- });
2588
- await this.protocol.stop();
2589
- await this._controlPipeline.stop();
2590
- (0, import_log8.log)("closed", void 0, {
2591
- F: __dxlog_file11,
2592
- L: 207,
2593
- S: this,
2594
- C: (f, a) => f(...a)
2595
- });
2596
- }
2597
- };
2598
- _ts_decorate6([
2599
- import_tracing2.trace.info()
2600
- ], Space.prototype, "protocol", void 0);
2601
- _ts_decorate6([
2602
- import_tracing2.trace.info()
2603
- ], Space.prototype, "_controlPipeline", void 0);
2604
- _ts_decorate6([
2605
- import_log8.logInfo,
2606
- import_tracing2.trace.info()
2607
- ], Space.prototype, "id", null);
2608
- _ts_decorate6([
2609
- import_log8.logInfo,
2610
- import_tracing2.trace.info()
2611
- ], Space.prototype, "key", null);
2612
- _ts_decorate6([
2613
- import_tracing2.trace.span()
2614
- ], Space.prototype, "_open", null);
2615
- _ts_decorate6([
2616
- import_async8.synchronized
2617
- ], Space.prototype, "_close", null);
2618
- Space = _ts_decorate6([
2619
- (0, import_async8.trackLeaks)("open", "close"),
2620
- import_tracing2.trace.resource()
2621
- ], Space);
2622
- var SPACE_IDS_CACHE = new import_util4.ComplexMap(import_keys5.PublicKey.hash);
2623
- var createIdFromSpaceKey = async (spaceKey) => {
2624
- const cachedValue = SPACE_IDS_CACHE.get(spaceKey);
2625
- if (cachedValue !== void 0) {
2626
- return cachedValue;
2737
+ async flush({ documentIds } = {}) {
2738
+ await this._repo.flush(documentIds);
2627
2739
  }
2628
- const digest = await import_crypto3.subtleCrypto.digest("SHA-256", spaceKey.asUint8Array());
2629
- const bytes = new Uint8Array(digest).slice(0, import_keys5.SpaceId.byteLength);
2630
- const spaceId = import_keys5.SpaceId.encode(bytes);
2631
- SPACE_IDS_CACHE.set(spaceKey, spaceId);
2632
- return spaceId;
2633
- };
2634
- var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/admission-discovery-extension.ts";
2635
- var CredentialRetrieverExtension = class extends import_teleport2.RpcExtension {
2636
- constructor(_request, _onResult) {
2637
- super({
2638
- requested: {
2639
- AdmissionDiscoveryService: import_protocols6.schema.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
2740
+ async getHeads(documentIds) {
2741
+ const result = [];
2742
+ const storeRequestIds = [];
2743
+ const storeResultIndices = [];
2744
+ for (const documentId of documentIds) {
2745
+ const doc = this._repo.handles[documentId]?.docSync();
2746
+ if (doc) {
2747
+ result.push((0, import_automerge3.getHeads)(doc));
2748
+ } else {
2749
+ storeRequestIds.push(documentId);
2750
+ storeResultIndices.push(result.length);
2751
+ result.push(void 0);
2640
2752
  }
2641
- });
2642
- this._request = _request;
2643
- this._onResult = _onResult;
2644
- this._ctx = new import_context11.Context(void 0, {
2645
- F: __dxlog_file12,
2646
- L: 25
2647
- });
2648
- }
2649
- async getHandlers() {
2650
- return {};
2651
- }
2652
- async onOpen(context) {
2653
- await super.onOpen(context);
2654
- (0, import_async10.scheduleTask)(this._ctx, async () => {
2655
- try {
2656
- const result = await this.rpc.AdmissionDiscoveryService.getAdmissionCredential(this._request);
2657
- this._onResult.wake(result.admissionCredential);
2658
- } catch (err) {
2659
- context.close(err);
2753
+ }
2754
+ if (storeRequestIds.length > 0) {
2755
+ const storedHeads = await this._headsStore.getHeads(storeRequestIds);
2756
+ for (let i = 0; i < storedHeads.length; i++) {
2757
+ result[storeResultIndices[i]] = storedHeads[i];
2660
2758
  }
2661
- });
2759
+ }
2760
+ return result;
2662
2761
  }
2663
- async onClose() {
2664
- await this._ctx.dispose();
2762
+ //
2763
+ // Collection sync.
2764
+ //
2765
+ getLocalCollectionState(collectionId) {
2766
+ return this._collectionSynchronizer.getLocalCollectionState(collectionId);
2665
2767
  }
2666
- async onAbort() {
2667
- await this._ctx.dispose();
2768
+ getRemoteCollectionStates(collectionId) {
2769
+ return this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
2668
2770
  }
2669
- };
2670
- var CredentialServerExtension = class extends import_teleport2.RpcExtension {
2671
- constructor(_space) {
2672
- super({
2673
- exposed: {
2674
- AdmissionDiscoveryService: import_protocols6.schema.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
2675
- }
2676
- });
2677
- this._space = _space;
2771
+ refreshCollection(collectionId) {
2772
+ this._collectionSynchronizer.refreshCollection(collectionId);
2678
2773
  }
2679
- async getHandlers() {
2680
- return {
2681
- AdmissionDiscoveryService: {
2682
- getAdmissionCredential: async (request) => {
2683
- const memberInfo = this._space.spaceState.members.get(request.memberKey);
2684
- if (!memberInfo?.credential) {
2685
- throw new import_protocols6.ProtocolError("Space member not found.", request);
2686
- }
2687
- return {
2688
- admissionCredential: memberInfo.credential
2689
- };
2690
- }
2691
- }
2774
+ async getCollectionSyncState(collectionId) {
2775
+ const result = {
2776
+ peers: []
2692
2777
  };
2693
- }
2694
- };
2695
- function _ts_decorate7(decorators, target, key, desc) {
2696
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2697
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2698
- r = Reflect.decorate(decorators, target, key, desc);
2699
- else
2700
- for (var i = decorators.length - 1; i >= 0; i--)
2701
- if (d = decorators[i])
2702
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2703
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2704
- }
2705
- var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
2706
- var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
2707
- var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
2708
- var SpaceProtocol = class {
2709
- constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore }) {
2710
- this._feeds = /* @__PURE__ */ new Set();
2711
- this._sessions = new import_util6.ComplexMap(import_keys7.PublicKey.hash);
2712
- this._topology = new import_network_manager.MMSTTopology({
2713
- originateConnections: 4,
2714
- maxPeers: 10,
2715
- sampleSize: 20
2716
- });
2717
- this._spaceKey = topic;
2718
- this._networkManager = networkManager;
2719
- this._swarmIdentity = swarmIdentity;
2720
- this._onSessionAuth = onSessionAuth;
2721
- this._onAuthFailure = onAuthFailure;
2722
- this.blobSync = new import_teleport_extension_object_sync.BlobSync({
2723
- blobStore
2778
+ const localState = this.getLocalCollectionState(collectionId);
2779
+ const remoteState = this.getRemoteCollectionStates(collectionId);
2780
+ if (!localState) {
2781
+ return result;
2782
+ }
2783
+ for (const [peerId, state] of remoteState) {
2784
+ const diff = diffCollectionState(localState, state);
2785
+ result.peers.push({
2786
+ peerId,
2787
+ differentDocuments: diff.different.length
2788
+ });
2789
+ }
2790
+ return result;
2791
+ }
2792
+ /**
2793
+ * Update the local collection state based on the locally stored document heads.
2794
+ */
2795
+ async updateLocalCollectionState(collectionId, documentIds) {
2796
+ const heads = await this.getHeads(documentIds);
2797
+ const documents = Object.fromEntries(heads.map((heads2, index) => [
2798
+ documentIds[index],
2799
+ heads2 ?? []
2800
+ ]));
2801
+ this._collectionSynchronizer.setLocalCollectionState(collectionId, {
2802
+ documents
2724
2803
  });
2725
- this._topic = import_crypto4.subtleCrypto.digest("SHA-256", topic.asBuffer()).then(import_crypto4.discoveryKey).then(import_keys7.PublicKey.from);
2726
2804
  }
2727
- get sessions() {
2728
- return this._sessions;
2805
+ _onCollectionStateQueried(collectionId, peerId) {
2806
+ this._collectionSynchronizer.onCollectionStateQueried(collectionId, peerId);
2729
2807
  }
2730
- get feeds() {
2731
- return this._feeds;
2808
+ _onCollectionStateReceived(collectionId, peerId, state) {
2809
+ this._collectionSynchronizer.onRemoteStateReceived(collectionId, peerId, decodeCollectionState(state));
2732
2810
  }
2733
- get _ownPeerKey() {
2734
- return this._swarmIdentity.peerKey;
2811
+ _queryCollectionState(collectionId, peerId) {
2812
+ this._echoNetworkAdapter.queryCollectionState(collectionId, peerId);
2735
2813
  }
2736
- // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
2737
- addFeed(feed) {
2738
- (0, import_log10.log)("addFeed", {
2739
- key: feed.key
2740
- }, {
2741
- F: __dxlog_file13,
2742
- L: 109,
2743
- S: this,
2744
- C: (f, a) => f(...a)
2745
- });
2746
- this._feeds.add(feed);
2747
- for (const session of this._sessions.values()) {
2748
- session.replicator.addFeed(feed);
2749
- }
2814
+ _sendCollectionState(collectionId, peerId, state) {
2815
+ this._echoNetworkAdapter.sendCollectionState(collectionId, peerId, encodeCollectionState(state));
2750
2816
  }
2751
- // TODO(burdon): Rename open? Common open/close interfaces for all services?
2752
- async start() {
2753
- if (this._connection) {
2817
+ _onPeerConnected(peerId) {
2818
+ this._collectionSynchronizer.onConnectionOpen(peerId);
2819
+ }
2820
+ _onPeerDisconnected(peerId) {
2821
+ this._collectionSynchronizer.onConnectionClosed(peerId);
2822
+ }
2823
+ _onRemoteCollectionStateUpdated(collectionId, peerId) {
2824
+ const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
2825
+ const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
2826
+ if (!localState || !remoteState) {
2754
2827
  return;
2755
2828
  }
2756
- const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(""));
2757
- await this.blobSync.open();
2758
- (0, import_log10.log)("starting...", void 0, {
2759
- F: __dxlog_file13,
2760
- L: 128,
2761
- S: this,
2762
- C: (f, a) => f(...a)
2763
- });
2764
- const topic = await this._topic;
2765
- this._connection = await this._networkManager.joinSwarm({
2766
- protocolProvider: this._createProtocolProvider(credentials),
2767
- peerId: this._swarmIdentity.peerKey,
2768
- topic,
2769
- topology: this._topology,
2770
- label: `swarm ${topic.truncate()} for space ${this._spaceKey.truncate()}`
2771
- });
2772
- (0, import_log10.log)("started", void 0, {
2773
- F: __dxlog_file13,
2774
- L: 138,
2829
+ const { different } = diffCollectionState(localState, remoteState);
2830
+ if (different.length === 0) {
2831
+ return;
2832
+ }
2833
+ import_log11.log.info("replication documents after collection sync", {
2834
+ count: different.length
2835
+ }, {
2836
+ F: __dxlog_file12,
2837
+ L: 475,
2775
2838
  S: this,
2776
2839
  C: (f, a) => f(...a)
2777
2840
  });
2778
- }
2779
- updateTopology() {
2780
- this._topology.forceUpdate();
2781
- }
2782
- async stop() {
2783
- await this.blobSync.close();
2784
- if (this._connection) {
2785
- (0, import_log10.log)("stopping...", void 0, {
2786
- F: __dxlog_file13,
2787
- L: 149,
2788
- S: this,
2789
- C: (f, a) => f(...a)
2790
- });
2791
- await this._connection.close();
2792
- (0, import_log10.log)("stopped", void 0, {
2793
- F: __dxlog_file13,
2794
- L: 151,
2795
- S: this,
2796
- C: (f, a) => f(...a)
2797
- });
2841
+ for (const documentId of different) {
2842
+ this._repo.find(documentId);
2798
2843
  }
2799
2844
  }
2800
- _createProtocolProvider(credentials) {
2801
- return (wireParams) => {
2802
- const session = new SpaceProtocolSession({
2803
- wireParams,
2804
- swarmIdentity: this._swarmIdentity,
2805
- onSessionAuth: this._onSessionAuth,
2806
- onAuthFailure: this._onAuthFailure,
2807
- blobSync: this.blobSync
2808
- });
2809
- this._sessions.set(wireParams.remotePeerId, session);
2810
- for (const feed of this._feeds) {
2811
- session.replicator.addFeed(feed);
2845
+ _onHeadsChanged(documentId, heads) {
2846
+ for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
2847
+ const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
2848
+ if (state?.documents[documentId]) {
2849
+ const newState = structuredClone(state);
2850
+ newState.documents[documentId] = heads;
2851
+ this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
2812
2852
  }
2813
- return session;
2814
- };
2853
+ }
2815
2854
  }
2816
2855
  };
2817
2856
  _ts_decorate7([
2818
- import_log10.logInfo,
2819
2857
  import_tracing4.trace.info()
2820
- ], SpaceProtocol.prototype, "_topic", void 0);
2858
+ ], AutomergeHost.prototype, "_peerId", void 0);
2821
2859
  _ts_decorate7([
2822
- import_tracing4.trace.info()
2823
- ], SpaceProtocol.prototype, "_spaceKey", void 0);
2860
+ import_tracing4.trace.info({
2861
+ depth: null
2862
+ })
2863
+ ], AutomergeHost.prototype, "_automergePeers", null);
2824
2864
  _ts_decorate7([
2825
- import_log10.logInfo
2826
- ], SpaceProtocol.prototype, "_ownPeerKey", null);
2827
- SpaceProtocol = _ts_decorate7([
2865
+ import_tracing4.trace.span({
2866
+ showInBrowserTimeline: true
2867
+ })
2868
+ ], AutomergeHost.prototype, "flush", null);
2869
+ AutomergeHost = _ts_decorate7([
2828
2870
  import_tracing4.trace.resource()
2829
- ], SpaceProtocol);
2830
- var AuthStatus;
2831
- (function(AuthStatus2) {
2832
- AuthStatus2["INITIAL"] = "INITIAL";
2833
- AuthStatus2["SUCCESS"] = "SUCCESS";
2834
- AuthStatus2["FAILURE"] = "FAILURE";
2835
- })(AuthStatus || (AuthStatus = {}));
2836
- var SpaceProtocolSession = class {
2837
- // TODO(dmaretskyi): Allow to pass in extra extensions.
2838
- constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync }) {
2839
- this.replicator = new import_teleport_extension_replicator.ReplicatorExtension().setOptions({
2840
- upload: true
2841
- });
2842
- this._authStatus = "INITIAL";
2843
- this._wireParams = wireParams;
2844
- this._swarmIdentity = swarmIdentity;
2845
- this._onSessionAuth = onSessionAuth;
2846
- this._onAuthFailure = onAuthFailure;
2847
- this._blobSync = blobSync;
2848
- this._teleport = new import_teleport3.Teleport(wireParams);
2849
- }
2850
- get authStatus() {
2851
- return this._authStatus;
2852
- }
2853
- get stats() {
2854
- return this._teleport.stats;
2855
- }
2856
- get stream() {
2857
- return this._teleport.stream;
2871
+ ], AutomergeHost);
2872
+ var getSpaceKeyFromDoc = (doc) => {
2873
+ const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
2874
+ if (rawSpaceKey == null) {
2875
+ return null;
2858
2876
  }
2859
- async open(sessionId) {
2860
- await this._teleport.open(sessionId);
2861
- this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
2862
- provider: this._swarmIdentity.credentialProvider,
2863
- verifier: this._swarmIdentity.credentialAuthenticator,
2864
- onAuthSuccess: () => {
2865
- (0, import_log10.log)("Peer authenticated", void 0, {
2866
- F: __dxlog_file13,
2867
- L: 248,
2868
- S: this,
2869
- C: (f, a) => f(...a)
2870
- });
2871
- this._authStatus = "SUCCESS";
2872
- this._onSessionAuth?.(this._teleport);
2873
- },
2874
- onAuthFailure: () => {
2875
- this._authStatus = "FAILURE";
2876
- this._onAuthFailure?.(this._teleport);
2877
+ return String(rawSpaceKey);
2878
+ };
2879
+ var waitForHeads = async (handle, heads) => {
2880
+ const unavailableHeads = new Set(heads);
2881
+ await handle.whenReady();
2882
+ await import_async10.Event.wrap(handle, "change").waitForCondition(() => {
2883
+ for (const changeHash of unavailableHeads.values()) {
2884
+ if (changeIsPresentInDoc(handle.docSync(), changeHash)) {
2885
+ unavailableHeads.delete(changeHash);
2877
2886
  }
2878
- }));
2879
- this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
2880
- this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
2881
- }
2882
- async close() {
2883
- (0, import_log10.log)("close", void 0, {
2884
- F: __dxlog_file13,
2885
- L: 264,
2886
- S: this,
2887
- C: (f, a) => f(...a)
2888
- });
2889
- await this._teleport.close();
2890
- }
2891
- async abort() {
2892
- await this._teleport.abort();
2893
- }
2887
+ }
2888
+ return unavailableHeads.size === 0;
2889
+ });
2894
2890
  };
2895
- _ts_decorate7([
2896
- import_log10.logInfo
2897
- ], SpaceProtocolSession.prototype, "_wireParams", void 0);
2898
- _ts_decorate7([
2899
- import_log10.logInfo
2900
- ], SpaceProtocolSession.prototype, "authStatus", null);
2901
- var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
2891
+ var changeIsPresentInDoc = (doc, changeHash) => {
2892
+ return !!(0, import_automerge3.getBackend)(doc).getChangeByHash(changeHash);
2893
+ };
2894
+ var decodeCollectionState = (state) => {
2895
+ (0, import_invariant8.invariant)(typeof state === "object" && state !== null, "Invalid state", {
2896
+ F: __dxlog_file12,
2897
+ L: 528,
2898
+ S: void 0,
2899
+ A: [
2900
+ "typeof state === 'object' && state !== null",
2901
+ "'Invalid state'"
2902
+ ]
2903
+ });
2904
+ return state;
2905
+ };
2906
+ var encodeCollectionState = (state) => {
2907
+ return state;
2908
+ };
2909
+ var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
2902
2910
  var DEFAULT_FACTORY = (params) => new import_teleport_extension_automerge_replicator.AutomergeReplicator(...params);
2903
2911
  var MeshReplicatorConnection = class extends import_context12.Resource {
2904
2912
  constructor(_params) {
@@ -2916,9 +2924,9 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2916
2924
  });
2917
2925
  this.writable = new WritableStream({
2918
2926
  write: async (message, controller) => {
2919
- (0, import_invariant12.invariant)(this._isEnabled, "Writing to a disabled connection", {
2920
- F: __dxlog_file14,
2921
- L: 48,
2927
+ (0, import_invariant11.invariant)(this._isEnabled, "Writing to a disabled connection", {
2928
+ F: __dxlog_file13,
2929
+ L: 49,
2922
2930
  S: this,
2923
2931
  A: [
2924
2932
  "this._isEnabled",
@@ -2926,6 +2934,7 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2926
2934
  ]
2927
2935
  });
2928
2936
  try {
2937
+ logSendSync(message);
2929
2938
  await this.replicatorExtension.sendSyncMessage({
2930
2939
  payload: import_automerge_repo3.cbor.encode(message)
2931
2940
  });
@@ -2949,8 +2958,8 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2949
2958
  thisPeerId: this.peerId,
2950
2959
  remotePeerId: remotePeerId.toHex()
2951
2960
  }, {
2952
- F: __dxlog_file14,
2953
- L: 82,
2961
+ F: __dxlog_file13,
2962
+ L: 84,
2954
2963
  S: this,
2955
2964
  C: (f, a) => f(...a)
2956
2965
  });
@@ -2975,9 +2984,9 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2975
2984
  }
2976
2985
  }
2977
2986
  get peerId() {
2978
- (0, import_invariant12.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
2979
- F: __dxlog_file14,
2980
- L: 108,
2987
+ (0, import_invariant11.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
2988
+ F: __dxlog_file13,
2989
+ L: 110,
2981
2990
  S: this,
2982
2991
  A: [
2983
2992
  "this._remotePeerId != null",
@@ -2997,9 +3006,9 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2997
3006
  * Call after the remote peer has connected.
2998
3007
  */
2999
3008
  enable() {
3000
- (0, import_invariant12.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
3001
- F: __dxlog_file14,
3002
- L: 125,
3009
+ (0, import_invariant11.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
3010
+ F: __dxlog_file13,
3011
+ L: 127,
3003
3012
  S: this,
3004
3013
  A: [
3005
3014
  "this._remotePeerId != null",
@@ -3015,6 +3024,41 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
3015
3024
  this._isEnabled = false;
3016
3025
  }
3017
3026
  };
3027
+ var logSendSync = (message) => {
3028
+ (0, import_log14.log)("sendSyncMessage", () => {
3029
+ const decodedSyncMessage = message.type === "sync" && message.data ? A2.decodeSyncMessage(message.data) : void 0;
3030
+ return {
3031
+ sync: decodedSyncMessage && {
3032
+ headsLength: decodedSyncMessage.heads.length,
3033
+ requesting: decodedSyncMessage.need.length > 0,
3034
+ sendingChanges: decodedSyncMessage.changes.length > 0
3035
+ },
3036
+ type: message.type,
3037
+ from: message.senderId,
3038
+ to: message.targetId
3039
+ };
3040
+ }, {
3041
+ F: __dxlog_file13,
3042
+ L: 140,
3043
+ S: void 0,
3044
+ C: (f, a) => f(...a)
3045
+ });
3046
+ };
3047
+ var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
3048
+ var deriveCollectionIdFromSpaceId = (spaceId) => `space:${spaceId}`;
3049
+ var getSpaceIdFromCollectionId = (collectionId) => {
3050
+ const spaceId = collectionId.replace(/^space:/, "");
3051
+ (0, import_invariant12.invariant)(import_keys10.SpaceId.isValid(spaceId), void 0, {
3052
+ F: __dxlog_file14,
3053
+ L: 12,
3054
+ S: void 0,
3055
+ A: [
3056
+ "SpaceId.isValid(spaceId)",
3057
+ ""
3058
+ ]
3059
+ });
3060
+ return spaceId;
3061
+ };
3018
3062
  var __dxlog_file15 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts";
3019
3063
  var MeshEchoReplicator = class {
3020
3064
  constructor() {
@@ -3038,7 +3082,7 @@ var MeshEchoReplicator = class {
3038
3082
  this._context = null;
3039
3083
  }
3040
3084
  createExtension(extensionFactory) {
3041
- (0, import_invariant11.invariant)(this._context, void 0, {
3085
+ (0, import_invariant10.invariant)(this._context, void 0, {
3042
3086
  F: __dxlog_file15,
3043
3087
  L: 56,
3044
3088
  S: this,
@@ -3059,7 +3103,7 @@ var MeshEchoReplicator = class {
3059
3103
  S: this,
3060
3104
  C: (f, a) => f(...a)
3061
3105
  });
3062
- (0, import_invariant11.invariant)(this._context, void 0, {
3106
+ (0, import_invariant10.invariant)(this._context, void 0, {
3063
3107
  F: __dxlog_file15,
3064
3108
  L: 63,
3065
3109
  S: this,
@@ -3100,7 +3144,7 @@ var MeshEchoReplicator = class {
3100
3144
  S: this,
3101
3145
  C: (f, a) => f(...a)
3102
3146
  });
3103
- (0, import_invariant11.invariant)(this._context, void 0, {
3147
+ (0, import_invariant10.invariant)(this._context, void 0, {
3104
3148
  F: __dxlog_file15,
3105
3149
  L: 82,
3106
3150
  S: this,
@@ -3112,16 +3156,21 @@ var MeshEchoReplicator = class {
3112
3156
  try {
3113
3157
  const spaceKey = await this._context.getContainingSpaceForDocument(params.documentId);
3114
3158
  if (!spaceKey) {
3115
- (0, import_log13.log)("space key not found for share policy check", {
3159
+ const remoteDocumentExists = await this._context.isDocumentInRemoteCollection({
3160
+ documentId: params.documentId,
3161
+ peerId: connection.peerId
3162
+ });
3163
+ (0, import_log13.log)("document not found locally for share policy check, accepting the remote document", {
3116
3164
  peerId: connection.peerId,
3117
- documentId: params.documentId
3165
+ documentId: params.documentId,
3166
+ remoteDocumentExists
3118
3167
  }, {
3119
3168
  F: __dxlog_file15,
3120
- L: 86,
3169
+ L: 90,
3121
3170
  S: this,
3122
3171
  C: (f, a) => f(...a)
3123
3172
  });
3124
- return false;
3173
+ return remoteDocumentExists;
3125
3174
  }
3126
3175
  const spaceId = await createIdFromSpaceKey(spaceKey);
3127
3176
  const authorizedDevices = this._authorizedDevices.get(spaceId);
@@ -3131,7 +3180,7 @@ var MeshEchoReplicator = class {
3131
3180
  documentId: params.documentId
3132
3181
  }, {
3133
3182
  F: __dxlog_file15,
3134
- L: 98,
3183
+ L: 106,
3135
3184
  S: this,
3136
3185
  C: (f, a) => f(...a)
3137
3186
  });
@@ -3147,7 +3196,7 @@ var MeshEchoReplicator = class {
3147
3196
  isAuthorized
3148
3197
  }, {
3149
3198
  F: __dxlog_file15,
3150
- L: 106,
3199
+ L: 114,
3151
3200
  S: this,
3152
3201
  C: (f, a) => f(...a)
3153
3202
  });
@@ -3155,7 +3204,7 @@ var MeshEchoReplicator = class {
3155
3204
  } catch (err) {
3156
3205
  import_log13.log.catch(err, void 0, {
3157
3206
  F: __dxlog_file15,
3158
- L: 116,
3207
+ L: 124,
3159
3208
  S: this,
3160
3209
  C: (f, a) => f(...a)
3161
3210
  });
@@ -3171,7 +3220,7 @@ var MeshEchoReplicator = class {
3171
3220
  collectionId
3172
3221
  }, {
3173
3222
  F: __dxlog_file15,
3174
- L: 126,
3223
+ L: 134,
3175
3224
  S: this,
3176
3225
  C: (f, a) => f(...a)
3177
3226
  });
@@ -3190,12 +3239,12 @@ var MeshEchoReplicator = class {
3190
3239
  deviceKey
3191
3240
  }, {
3192
3241
  F: __dxlog_file15,
3193
- L: 143,
3242
+ L: 151,
3194
3243
  S: this,
3195
3244
  C: (f, a) => f(...a)
3196
3245
  });
3197
3246
  const spaceId = await createIdFromSpaceKey(spaceKey);
3198
- (0, import_util8.defaultMap)(this._authorizedDevices, spaceId, () => new import_util8.ComplexSet(import_keys10.PublicKey.hash)).add(deviceKey);
3247
+ (0, import_util8.defaultMap)(this._authorizedDevices, spaceId, () => new import_util8.ComplexSet(import_keys9.PublicKey.hash)).add(deviceKey);
3199
3248
  for (const connection of this._connections) {
3200
3249
  if (connection.remoteDeviceKey && connection.remoteDeviceKey.equals(deviceKey)) {
3201
3250
  if (this._connectionsPerPeer.has(connection.peerId)) {
@@ -3582,7 +3631,7 @@ var DataServiceImpl = class {
3582
3631
  synchronizer.open().then(() => {
3583
3632
  this._subscriptions.set(request.subscriptionId, synchronizer);
3584
3633
  ready();
3585
- }).catch((err) => import_log12.log.catch(err, void 0, {
3634
+ }).catch((err) => import_log10.log.catch(err, void 0, {
3586
3635
  F: __dxlog_file16,
3587
3636
  L: 64,
3588
3637
  S: this,
@@ -3593,7 +3642,7 @@ var DataServiceImpl = class {
3593
3642
  }
3594
3643
  async updateSubscription(request) {
3595
3644
  const synchronizer = this._subscriptions.get(request.subscriptionId);
3596
- (0, import_invariant10.invariant)(synchronizer, "Subscription not found", {
3645
+ (0, import_invariant7.invariant)(synchronizer, "Subscription not found", {
3597
3646
  F: __dxlog_file16,
3598
3647
  L: 71,
3599
3648
  S: this,
@@ -3614,7 +3663,7 @@ var DataServiceImpl = class {
3614
3663
  return;
3615
3664
  }
3616
3665
  const synchronizer = this._subscriptions.get(request.subscriptionId);
3617
- (0, import_invariant10.invariant)(synchronizer, "Subscription not found", {
3666
+ (0, import_invariant7.invariant)(synchronizer, "Subscription not found", {
3618
3667
  F: __dxlog_file16,
3619
3668
  L: 86,
3620
3669
  S: this,
@@ -3657,7 +3706,7 @@ var DataServiceImpl = class {
3657
3706
  await this._updateIndexes();
3658
3707
  }
3659
3708
  async getSpaceSyncState(request, options) {
3660
- (0, import_invariant10.invariant)(import_keys9.SpaceId.isValid(request.spaceId), void 0, {
3709
+ (0, import_invariant7.invariant)(import_keys7.SpaceId.isValid(request.spaceId), void 0, {
3661
3710
  F: __dxlog_file16,
3662
3711
  L: 127,
3663
3712
  S: this,
@@ -3688,14 +3737,15 @@ function _ts_decorate9(decorators, target, key, desc) {
3688
3737
  }
3689
3738
  var __dxlog_file17 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
3690
3739
  var SpaceManager = class {
3691
- constructor({ feedStore, networkManager, metadataStore, snapshotStore, blobStore }) {
3692
- this._spaces = new import_util7.ComplexMap(import_keys8.PublicKey.hash);
3693
- this._instanceId = import_keys8.PublicKey.random().toHex();
3740
+ constructor({ feedStore, networkManager, metadataStore, snapshotStore, blobStore, disableP2pReplication }) {
3741
+ this._spaces = new import_util6.ComplexMap(import_keys6.PublicKey.hash);
3742
+ this._instanceId = import_keys6.PublicKey.random().toHex();
3694
3743
  this._feedStore = feedStore;
3695
3744
  this._networkManager = networkManager;
3696
3745
  this._metadataStore = metadataStore;
3697
3746
  this._snapshotStore = snapshotStore;
3698
3747
  this._blobStore = blobStore;
3748
+ this._disableP2pReplication = disableP2pReplication ?? false;
3699
3749
  }
3700
3750
  // TODO(burdon): Remove.
3701
3751
  get spaces() {
@@ -3709,19 +3759,19 @@ var SpaceManager = class {
3709
3759
  ].map((space) => space.close()));
3710
3760
  }
3711
3761
  async constructSpace({ metadata, swarmIdentity, onAuthorizedConnection, onAuthFailure, onDelegatedInvitationStatusChange, onMemberRolesChanged, memberKey }) {
3712
- import_log11.log.trace("dxos.echo.space-manager.construct-space", import_protocols7.trace.begin({
3762
+ import_log9.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.begin({
3713
3763
  id: this._instanceId
3714
3764
  }), {
3715
3765
  F: __dxlog_file17,
3716
- L: 103,
3766
+ L: 114,
3717
3767
  S: this,
3718
3768
  C: (f, a) => f(...a)
3719
3769
  });
3720
- (0, import_log11.log)("constructing space...", {
3770
+ (0, import_log9.log)("constructing space...", {
3721
3771
  spaceKey: metadata.genesisFeedKey
3722
3772
  }, {
3723
3773
  F: __dxlog_file17,
3724
- L: 104,
3774
+ L: 115,
3725
3775
  S: this,
3726
3776
  C: (f, a) => f(...a)
3727
3777
  });
@@ -3734,7 +3784,8 @@ var SpaceManager = class {
3734
3784
  networkManager: this._networkManager,
3735
3785
  onSessionAuth: onAuthorizedConnection,
3736
3786
  onAuthFailure,
3737
- blobStore: this._blobStore
3787
+ blobStore: this._blobStore,
3788
+ disableP2pReplication: this._disableP2pReplication
3738
3789
  });
3739
3790
  const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
3740
3791
  const space = new Space({
@@ -3750,11 +3801,11 @@ var SpaceManager = class {
3750
3801
  onMemberRolesChanged
3751
3802
  });
3752
3803
  this._spaces.set(space.key, space);
3753
- import_log11.log.trace("dxos.echo.space-manager.construct-space", import_protocols7.trace.end({
3804
+ import_log9.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.end({
3754
3805
  id: this._instanceId
3755
3806
  }), {
3756
3807
  F: __dxlog_file17,
3757
- L: 135,
3808
+ L: 147,
3758
3809
  S: this,
3759
3810
  C: (f, a) => f(...a)
3760
3811
  });
@@ -3762,23 +3813,23 @@ var SpaceManager = class {
3762
3813
  }
3763
3814
  async requestSpaceAdmissionCredential(params) {
3764
3815
  const traceKey = "dxos.echo.space-manager.request-space-admission";
3765
- import_log11.log.trace(traceKey, import_protocols7.trace.begin({
3816
+ import_log9.log.trace(traceKey, import_protocols6.trace.begin({
3766
3817
  id: this._instanceId
3767
3818
  }), {
3768
3819
  F: __dxlog_file17,
3769
- L: 141,
3820
+ L: 153,
3770
3821
  S: this,
3771
3822
  C: (f, a) => f(...a)
3772
3823
  });
3773
- (0, import_log11.log)("requesting space admission credential...", {
3824
+ (0, import_log9.log)("requesting space admission credential...", {
3774
3825
  spaceKey: params.spaceKey
3775
3826
  }, {
3776
3827
  F: __dxlog_file17,
3777
- L: 142,
3828
+ L: 154,
3778
3829
  S: this,
3779
3830
  C: (f, a) => f(...a)
3780
3831
  });
3781
- const onCredentialResolved = new import_async11.Trigger();
3832
+ const onCredentialResolved = new import_async9.Trigger();
3782
3833
  const protocol = new SpaceProtocol({
3783
3834
  topic: params.spaceKey,
3784
3835
  swarmIdentity: params.swarmIdentity,
@@ -3790,29 +3841,30 @@ var SpaceManager = class {
3790
3841
  }, onCredentialResolved));
3791
3842
  },
3792
3843
  onAuthFailure: (session) => session.close(),
3793
- blobStore: this._blobStore
3844
+ blobStore: this._blobStore,
3845
+ disableP2pReplication: this._disableP2pReplication
3794
3846
  });
3795
3847
  try {
3796
3848
  await protocol.start();
3797
3849
  const credential = await onCredentialResolved.wait({
3798
3850
  timeout: params.timeout
3799
3851
  });
3800
- import_log11.log.trace(traceKey, import_protocols7.trace.end({
3852
+ import_log9.log.trace(traceKey, import_protocols6.trace.end({
3801
3853
  id: this._instanceId
3802
3854
  }), {
3803
3855
  F: __dxlog_file17,
3804
- L: 165,
3856
+ L: 178,
3805
3857
  S: this,
3806
3858
  C: (f, a) => f(...a)
3807
3859
  });
3808
3860
  return credential;
3809
3861
  } catch (err) {
3810
- import_log11.log.trace(traceKey, import_protocols7.trace.error({
3862
+ import_log9.log.trace(traceKey, import_protocols6.trace.error({
3811
3863
  id: this._instanceId,
3812
3864
  error: err
3813
3865
  }), {
3814
3866
  F: __dxlog_file17,
3815
- L: 168,
3867
+ L: 181,
3816
3868
  S: this,
3817
3869
  C: (f, a) => f(...a)
3818
3870
  });
@@ -3823,13 +3875,13 @@ var SpaceManager = class {
3823
3875
  }
3824
3876
  };
3825
3877
  _ts_decorate9([
3826
- import_async11.synchronized
3878
+ import_async9.synchronized
3827
3879
  ], SpaceManager.prototype, "open", null);
3828
3880
  _ts_decorate9([
3829
- import_async11.synchronized
3881
+ import_async9.synchronized
3830
3882
  ], SpaceManager.prototype, "close", null);
3831
3883
  SpaceManager = _ts_decorate9([
3832
- (0, import_async11.trackLeaks)("open", "close")
3884
+ (0, import_async9.trackLeaks)("open", "close")
3833
3885
  ], SpaceManager);
3834
3886
  function _ts_decorate10(decorators, target, key, desc) {
3835
3887
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -3844,14 +3896,14 @@ function _ts_decorate10(decorators, target, key, desc) {
3844
3896
  var __dxlog_file18 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
3845
3897
  var EXPIRED_INVITATION_CLEANUP_INTERVAL = 60 * 60 * 1e3;
3846
3898
  var emptyEchoMetadata = () => ({
3847
- version: import_protocols8.STORAGE_VERSION,
3899
+ version: import_protocols9.STORAGE_VERSION,
3848
3900
  spaces: [],
3849
3901
  created: /* @__PURE__ */ new Date(),
3850
3902
  updated: /* @__PURE__ */ new Date()
3851
3903
  });
3852
3904
  var emptyLargeSpaceMetadata = () => ({});
3853
- var EchoMetadata = import_protocols8.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
3854
- var LargeSpaceMetadata = import_protocols8.schema.getCodecForType("dxos.echo.metadata.LargeSpaceMetadata");
3905
+ var EchoMetadata = import_protocols9.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
3906
+ var LargeSpaceMetadata = import_protocols9.schema.getCodecForType("dxos.echo.metadata.LargeSpaceMetadata");
3855
3907
  var MetadataStore = class {
3856
3908
  constructor(directory) {
3857
3909
  this._metadata = emptyEchoMetadata();
@@ -3896,7 +3948,7 @@ var MetadataStore = class {
3896
3948
  C: (f, a) => f(...a)
3897
3949
  });
3898
3950
  if (fileLength < dataSize + 8) {
3899
- throw new import_protocols8.DataCorruptionError("Metadata size is smaller than expected.", {
3951
+ throw new import_protocols9.DataCorruptionError("Metadata size is smaller than expected.", {
3900
3952
  fileLength,
3901
3953
  dataSize
3902
3954
  });
@@ -3904,7 +3956,7 @@ var MetadataStore = class {
3904
3956
  const data = await file.read(8, dataSize);
3905
3957
  const calculatedChecksum = import_crc_32.default.buf(data);
3906
3958
  if (calculatedChecksum !== checksum) {
3907
- throw new import_protocols8.DataCorruptionError("Metadata checksum is invalid.");
3959
+ throw new import_protocols9.DataCorruptionError("Metadata checksum is invalid.");
3908
3960
  }
3909
3961
  return codec2.decode(data);
3910
3962
  } finally {
@@ -3994,7 +4046,7 @@ var MetadataStore = class {
3994
4046
  async _save() {
3995
4047
  const data = {
3996
4048
  ...this._metadata,
3997
- version: import_protocols8.STORAGE_VERSION,
4049
+ version: import_protocols9.STORAGE_VERSION,
3998
4050
  created: this._metadata.created ?? /* @__PURE__ */ new Date(),
3999
4051
  updated: /* @__PURE__ */ new Date()
4000
4052
  };
@@ -4200,4 +4252,4 @@ var isLegacyInvitationFormat = (invitation) => {
4200
4252
  startAfter,
4201
4253
  valueEncoding
4202
4254
  });
4203
- //# sourceMappingURL=chunk-6MWU4MHX.cjs.map
4255
+ //# sourceMappingURL=chunk-IYTGTZ7D.cjs.map