@dxos/echo-pipeline 0.6.5 → 0.6.6-main.e1a6e1f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/lib/browser/{chunk-2MII6KJX.mjs → chunk-LNIQX6PQ.mjs} +2132 -2119
  2. package/dist/lib/browser/chunk-LNIQX6PQ.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +1 -1
  4. package/dist/lib/browser/meta.json +1 -1
  5. package/dist/lib/browser/testing/index.mjs +1 -1
  6. package/dist/lib/node/{chunk-6MWU4MHX.cjs → chunk-LTR534RP.cjs} +2125 -2112
  7. package/dist/lib/node/chunk-LTR534RP.cjs.map +7 -0
  8. package/dist/lib/node/index.cjs +35 -35
  9. package/dist/lib/node/index.cjs.map +1 -1
  10. package/dist/lib/node/meta.json +1 -1
  11. package/dist/lib/node/testing/index.cjs +11 -11
  12. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  13. package/dist/types/src/automerge/echo-replicator.d.ts +5 -1
  14. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  15. package/dist/types/src/automerge/network-protocol.d.ts +3 -28
  16. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  17. package/dist/types/src/space/space-manager.d.ts +3 -1
  18. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  19. package/dist/types/src/space/space-protocol.d.ts +10 -3
  20. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  21. package/dist/types/src/space/space.d.ts.map +1 -1
  22. package/package.json +33 -33
  23. package/src/automerge/echo-network-adapter.ts +5 -0
  24. package/src/automerge/echo-replicator.ts +5 -1
  25. package/src/automerge/network-protocol.ts +8 -34
  26. package/src/space/space-manager.ts +15 -2
  27. package/src/space/space-protocol.ts +42 -5
  28. package/src/space/space.ts +4 -4
  29. package/dist/lib/browser/chunk-2MII6KJX.mjs.map +0 -7
  30. package/dist/lib/node/chunk-6MWU4MHX.cjs.map +0 -7
@@ -26,8 +26,8 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
26
26
  mod
27
27
  ));
28
28
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
- var chunk_6MWU4MHX_exports = {};
30
- __export(chunk_6MWU4MHX_exports, {
29
+ var chunk_LTR534RP_exports = {};
30
+ __export(chunk_LTR534RP_exports, {
31
31
  AuthExtension: () => AuthExtension,
32
32
  AuthStatus: () => AuthStatus,
33
33
  AutomergeHost: () => AutomergeHost,
@@ -63,7 +63,7 @@ __export(chunk_6MWU4MHX_exports, {
63
63
  startAfter: () => startAfter,
64
64
  valueEncoding: () => valueEncoding
65
65
  });
66
- module.exports = __toCommonJS(chunk_6MWU4MHX_exports);
66
+ module.exports = __toCommonJS(chunk_LTR534RP_exports);
67
67
  var import_hypercore = require("@dxos/hypercore");
68
68
  var import_protocols = require("@dxos/protocols");
69
69
  var import_invariant = require("@dxos/invariant");
@@ -82,97 +82,98 @@ var import_async2 = require("@dxos/async");
82
82
  var import_automerge2 = require("@dxos/automerge/automerge");
83
83
  var import_context3 = require("@dxos/context");
84
84
  var import_util = require("@dxos/util");
85
- var import_context4 = require("@dxos/context");
86
85
  var import_async3 = require("@dxos/async");
87
- var import_automerge3 = require("@dxos/automerge/automerge");
88
- var import_automerge_repo = require("@dxos/automerge/automerge-repo");
89
- var import_context5 = require("@dxos/context");
86
+ var import_context4 = require("@dxos/context");
87
+ var import_crypto2 = require("@dxos/crypto");
90
88
  var import_invariant3 = require("@dxos/invariant");
91
- var import_keys2 = require("@dxos/keys");
92
89
  var import_log2 = require("@dxos/log");
93
90
  var import_protocols4 = require("@dxos/protocols");
94
- var import_tracing = require("@dxos/tracing");
95
- var import_async4 = require("@dxos/async");
96
- var import_automerge_repo2 = require("@dxos/automerge/automerge-repo");
97
- var import_context6 = require("@dxos/context");
98
- var import_invariant4 = require("@dxos/invariant");
99
- var import_log3 = require("@dxos/log");
100
- var import_util2 = require("@dxos/util");
101
- var import_indexing = require("@dxos/indexing");
102
- var import_invariant5 = require("@dxos/invariant");
103
- var import_keys3 = require("@dxos/keys");
104
- var import_async5 = require("@dxos/async");
105
- var import_context7 = require("@dxos/context");
106
- var import_crypto2 = require("@dxos/crypto");
107
- var import_invariant6 = require("@dxos/invariant");
108
- var import_log4 = require("@dxos/log");
109
- var import_protocols5 = require("@dxos/protocols");
110
91
  var import_teleport = require("@dxos/teleport");
111
- var import_async6 = require("@dxos/async");
92
+ var import_async4 = require("@dxos/async");
112
93
  var import_debug = require("@dxos/debug");
113
- var import_log5 = require("@dxos/log");
94
+ var import_log3 = require("@dxos/log");
114
95
  var import_timeframe = require("@dxos/timeframe");
115
- var import_async7 = require("@dxos/async");
116
- var import_context8 = require("@dxos/context");
96
+ var import_async5 = require("@dxos/async");
97
+ var import_context5 = require("@dxos/context");
117
98
  var import_debug2 = require("@dxos/debug");
118
99
  var import_feed_store = require("@dxos/feed-store");
119
- var import_invariant7 = require("@dxos/invariant");
120
- var import_keys4 = require("@dxos/keys");
121
- var import_log6 = require("@dxos/log");
100
+ var import_invariant4 = require("@dxos/invariant");
101
+ var import_keys2 = require("@dxos/keys");
102
+ var import_log4 = require("@dxos/log");
122
103
  var import_timeframe2 = require("@dxos/timeframe");
123
- var import_util3 = require("@dxos/util");
124
- var import_invariant8 = require("@dxos/invariant");
125
- var import_log7 = require("@dxos/log");
126
- var import_async8 = require("@dxos/async");
127
- var import_context9 = require("@dxos/context");
104
+ var import_util2 = require("@dxos/util");
105
+ var import_invariant5 = require("@dxos/invariant");
106
+ var import_log5 = require("@dxos/log");
107
+ var import_async6 = require("@dxos/async");
108
+ var import_context6 = require("@dxos/context");
128
109
  var import_crypto3 = require("@dxos/crypto");
129
- var import_invariant9 = require("@dxos/invariant");
130
- var import_keys5 = require("@dxos/keys");
131
- var import_log8 = require("@dxos/log");
110
+ var import_invariant6 = require("@dxos/invariant");
111
+ var import_keys3 = require("@dxos/keys");
112
+ var import_log6 = require("@dxos/log");
132
113
  var import_credentials = require("@dxos/protocols/proto/dxos/halo/credentials");
133
- var import_tracing2 = require("@dxos/tracing");
134
- var import_util4 = require("@dxos/util");
135
- var import_async9 = require("@dxos/async");
136
- var import_context10 = require("@dxos/context");
114
+ var import_tracing = require("@dxos/tracing");
115
+ var import_util3 = require("@dxos/util");
116
+ var import_async7 = require("@dxos/async");
117
+ var import_context7 = require("@dxos/context");
137
118
  var import_credentials2 = require("@dxos/credentials");
138
- var import_keys6 = require("@dxos/keys");
139
- var import_log9 = require("@dxos/log");
119
+ var import_keys4 = require("@dxos/keys");
120
+ var import_log7 = require("@dxos/log");
140
121
  var import_credentials3 = require("@dxos/protocols/proto/dxos/halo/credentials");
141
122
  var import_timeframe3 = require("@dxos/timeframe");
142
- var import_tracing3 = require("@dxos/tracing");
143
- var import_util5 = require("@dxos/util");
144
- var import_async10 = require("@dxos/async");
145
- var import_context11 = require("@dxos/context");
146
- var import_protocols6 = require("@dxos/protocols");
123
+ var import_tracing2 = require("@dxos/tracing");
124
+ var import_util4 = require("@dxos/util");
125
+ var import_async8 = require("@dxos/async");
126
+ var import_context8 = require("@dxos/context");
127
+ var import_protocols5 = require("@dxos/protocols");
147
128
  var import_teleport2 = require("@dxos/teleport");
148
129
  var import_crypto4 = require("@dxos/crypto");
149
- var import_keys7 = require("@dxos/keys");
150
- var import_log10 = require("@dxos/log");
130
+ var import_keys5 = require("@dxos/keys");
131
+ var import_log8 = require("@dxos/log");
151
132
  var import_network_manager = require("@dxos/network-manager");
152
133
  var import_teleport3 = require("@dxos/teleport");
153
134
  var import_teleport_extension_object_sync = require("@dxos/teleport-extension-object-sync");
154
135
  var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
155
- var import_tracing4 = require("@dxos/tracing");
156
- var import_util6 = require("@dxos/util");
157
- var import_async11 = require("@dxos/async");
136
+ var import_tracing3 = require("@dxos/tracing");
137
+ var import_util5 = require("@dxos/util");
138
+ var import_async9 = require("@dxos/async");
158
139
  var import_debug3 = require("@dxos/debug");
140
+ var import_keys6 = require("@dxos/keys");
141
+ var import_log9 = require("@dxos/log");
142
+ var import_protocols6 = require("@dxos/protocols");
143
+ var import_util6 = require("@dxos/util");
144
+ var import_codec_protobuf = require("@dxos/codec-protobuf");
145
+ var import_invariant7 = require("@dxos/invariant");
146
+ var import_keys7 = require("@dxos/keys");
147
+ var import_log10 = require("@dxos/log");
148
+ var import_async10 = require("@dxos/async");
149
+ var import_automerge3 = require("@dxos/automerge/automerge");
150
+ var import_automerge_repo = require("@dxos/automerge/automerge-repo");
151
+ var import_context9 = require("@dxos/context");
152
+ var import_invariant8 = require("@dxos/invariant");
159
153
  var import_keys8 = require("@dxos/keys");
160
154
  var import_log11 = require("@dxos/log");
161
155
  var import_protocols7 = require("@dxos/protocols");
156
+ var import_tracing4 = require("@dxos/tracing");
157
+ var import_async11 = require("@dxos/async");
158
+ var import_automerge_repo2 = require("@dxos/automerge/automerge-repo");
159
+ var import_context10 = require("@dxos/context");
160
+ var import_invariant9 = require("@dxos/invariant");
161
+ var import_log12 = require("@dxos/log");
162
162
  var import_util7 = require("@dxos/util");
163
- var import_codec_protobuf = require("@dxos/codec-protobuf");
163
+ var import_protocols8 = require("@dxos/protocols");
164
+ var import_indexing = require("@dxos/indexing");
165
+ var import_context11 = require("@dxos/context");
164
166
  var import_invariant10 = require("@dxos/invariant");
165
167
  var import_keys9 = require("@dxos/keys");
166
- var import_log12 = require("@dxos/log");
167
- var import_invariant11 = require("@dxos/invariant");
168
- var import_keys10 = require("@dxos/keys");
169
168
  var import_log13 = require("@dxos/log");
170
169
  var import_util8 = require("@dxos/util");
171
170
  var import_automerge_repo3 = require("@dxos/automerge/automerge-repo");
172
171
  var import_context12 = require("@dxos/context");
173
- var import_invariant12 = require("@dxos/invariant");
172
+ var import_invariant11 = require("@dxos/invariant");
174
173
  var import_log14 = require("@dxos/log");
175
174
  var import_teleport_extension_automerge_replicator = require("@dxos/teleport-extension-automerge-replicator");
175
+ var import_invariant12 = require("@dxos/invariant");
176
+ var import_keys10 = require("@dxos/keys");
176
177
  var import_tracing5 = require("@dxos/tracing");
177
178
  var import_util9 = require("@dxos/util");
178
179
  var import_crc_32 = __toESM(require("crc-32"));
@@ -181,7 +182,7 @@ var import_context13 = require("@dxos/context");
181
182
  var import_invariant13 = require("@dxos/invariant");
182
183
  var import_keys11 = require("@dxos/keys");
183
184
  var import_log15 = require("@dxos/log");
184
- var import_protocols8 = require("@dxos/protocols");
185
+ var import_protocols9 = require("@dxos/protocols");
185
186
  var import_services = require("@dxos/protocols/proto/dxos/client/services");
186
187
  var import_util10 = require("@dxos/util");
187
188
  var codec = import_protocols.schema.getCodecForType("dxos.echo.feed.FeedMessage");
@@ -578,113 +579,107 @@ var diffCollectionState = (local, remote) => {
578
579
  different
579
580
  };
580
581
  };
581
- var LevelDBStorageAdapter = class extends import_context4.Resource {
582
- constructor(_params) {
583
- super();
584
- this._params = _params;
585
- }
586
- async load(keyArray) {
587
- try {
588
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
589
- return void 0;
590
- }
591
- const startMs = Date.now();
592
- const chunk = await this._params.db.get(keyArray, {
593
- ...encodingOptions
594
- });
595
- this._params.monitor?.recordBytesLoaded(chunk.byteLength);
596
- this._params.monitor?.recordLoadDuration(Date.now() - startMs);
597
- return chunk;
598
- } catch (err) {
599
- if (isLevelDbNotFoundError(err)) {
600
- return void 0;
601
- }
602
- throw err;
603
- }
604
- }
605
- async save(keyArray, binary) {
606
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
607
- return void 0;
608
- }
609
- const startMs = Date.now();
610
- const batch = this._params.db.batch();
611
- await this._params.callbacks?.beforeSave?.({
612
- path: keyArray,
613
- batch
582
+ var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/auth.ts";
583
+ var AuthExtension = class extends import_teleport.RpcExtension {
584
+ constructor(_authParams) {
585
+ super({
586
+ requested: {
587
+ AuthService: import_protocols4.schema.getService("dxos.mesh.teleport.auth.AuthService")
588
+ },
589
+ exposed: {
590
+ AuthService: import_protocols4.schema.getService("dxos.mesh.teleport.auth.AuthService")
591
+ },
592
+ timeout: 60 * 1e3
614
593
  });
615
- batch.put(keyArray, Buffer.from(binary), {
616
- ...encodingOptions
594
+ this._authParams = _authParams;
595
+ this._ctx = new import_context4.Context({
596
+ onError: (err) => {
597
+ import_log2.log.catch(err, void 0, {
598
+ F: __dxlog_file3,
599
+ L: 28,
600
+ S: this,
601
+ C: (f, a) => f(...a)
602
+ });
603
+ }
604
+ }, {
605
+ F: __dxlog_file3,
606
+ L: 26
617
607
  });
618
- await batch.write();
619
- this._params.monitor?.recordBytesStored(binary.byteLength);
620
- await this._params.callbacks?.afterSave?.(keyArray);
621
- this._params.monitor?.recordStoreDuration(Date.now() - startMs);
622
608
  }
623
- async remove(keyArray) {
624
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
625
- return void 0;
626
- }
627
- await this._params.db.del(keyArray, {
628
- ...encodingOptions
609
+ async getHandlers() {
610
+ return {
611
+ AuthService: {
612
+ authenticate: async ({ challenge }) => {
613
+ try {
614
+ const credential = await this._authParams.provider(challenge);
615
+ if (!credential) {
616
+ throw new Error("auth rejected");
617
+ }
618
+ return {
619
+ credential
620
+ };
621
+ } catch (err) {
622
+ import_log2.log.error("failed to generate auth credentials", err, {
623
+ F: __dxlog_file3,
624
+ L: 55,
625
+ S: this,
626
+ C: (f, a) => f(...a)
627
+ });
628
+ throw new Error("auth rejected");
629
+ }
630
+ }
631
+ }
632
+ };
633
+ }
634
+ async onOpen(context) {
635
+ await super.onOpen(context);
636
+ (0, import_async3.scheduleTask)(this._ctx, async () => {
637
+ try {
638
+ const challenge = (0, import_crypto2.randomBytes)(32);
639
+ const { credential } = await this.rpc.AuthService.authenticate({
640
+ challenge
641
+ });
642
+ (0, import_invariant3.invariant)(credential?.length > 0, "invalid credential", {
643
+ F: __dxlog_file3,
644
+ L: 69,
645
+ S: this,
646
+ A: [
647
+ "credential?.length > 0",
648
+ "'invalid credential'"
649
+ ]
650
+ });
651
+ const success = await this._authParams.verifier(challenge, credential);
652
+ (0, import_invariant3.invariant)(success, "credential not verified", {
653
+ F: __dxlog_file3,
654
+ L: 71,
655
+ S: this,
656
+ A: [
657
+ "success",
658
+ "'credential not verified'"
659
+ ]
660
+ });
661
+ (0, import_async3.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
662
+ } catch (err) {
663
+ (0, import_log2.log)("auth failed", err, {
664
+ F: __dxlog_file3,
665
+ L: 74,
666
+ S: this,
667
+ C: (f, a) => f(...a)
668
+ });
669
+ this.close();
670
+ this._authParams.onAuthFailure();
671
+ }
629
672
  });
630
673
  }
631
- async loadRange(keyPrefix) {
632
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
633
- return [];
634
- }
635
- const startMs = Date.now();
636
- const result = [];
637
- for await (const [key, value] of this._params.db.iterator({
638
- gte: keyPrefix,
639
- lte: [
640
- ...keyPrefix,
641
- "\uFFFF"
642
- ],
643
- ...encodingOptions
644
- })) {
645
- result.push({
646
- key,
647
- data: value
648
- });
649
- this._params.monitor?.recordBytesLoaded(value.byteLength);
650
- }
651
- this._params.monitor?.recordLoadDuration(Date.now() - startMs);
652
- return result;
674
+ async onClose() {
675
+ await this._ctx.dispose();
676
+ await super.onClose();
653
677
  }
654
- async removeRange(keyPrefix) {
655
- if (this._lifecycleState !== import_context4.LifecycleState.OPEN) {
656
- return void 0;
657
- }
658
- const batch = this._params.db.batch();
659
- for await (const [key] of this._params.db.iterator({
660
- gte: keyPrefix,
661
- lte: [
662
- ...keyPrefix,
663
- "\uFFFF"
664
- ],
665
- ...encodingOptions
666
- })) {
667
- batch.del(key, {
668
- ...encodingOptions
669
- });
670
- }
671
- await batch.write();
678
+ async onAbort() {
679
+ await this._ctx.dispose();
680
+ await super.onAbort();
672
681
  }
673
682
  };
674
- var keyEncoder = {
675
- encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
676
- decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
677
- format: "buffer"
678
- };
679
- var encodingOptions = {
680
- keyEncoding: keyEncoder,
681
- valueEncoding: "buffer"
682
- };
683
- var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
684
- var MESSAGE_TYPE_COLLECTION_QUERY = "collection-query";
685
- var isCollectionQueryMessage = (message) => message.type === MESSAGE_TYPE_COLLECTION_QUERY;
686
- var MESSAGE_TYPE_COLLECTION_STATE = "collection-state";
687
- var isCollectionStateMessage = (message) => message.type === MESSAGE_TYPE_COLLECTION_STATE;
688
683
  function _ts_decorate(decorators, target, key, desc) {
689
684
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
690
685
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -695,918 +690,791 @@ function _ts_decorate(decorators, target, key, desc) {
695
690
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
696
691
  return c > 3 && r && Object.defineProperty(target, key, r), r;
697
692
  }
698
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
699
- var EchoNetworkAdapter = class extends import_automerge_repo2.NetworkAdapter {
700
- constructor(_params) {
701
- super();
702
- this._params = _params;
703
- this._replicators = /* @__PURE__ */ new Set();
704
- this._connections = /* @__PURE__ */ new Map();
705
- this._lifecycleState = import_context6.LifecycleState.CLOSED;
706
- this._connected = new import_async4.Trigger();
693
+ var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
694
+ var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
695
+ feedKey,
696
+ index
697
+ }));
698
+ var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
699
+ feedKey,
700
+ index
701
+ ]));
702
+ var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
703
+ feedKey,
704
+ index: index + 1
705
+ }));
706
+ var TimeframeClock = class {
707
+ constructor(_timeframe = new import_timeframe.Timeframe()) {
708
+ this._timeframe = _timeframe;
709
+ this.update = new import_async4.Event();
710
+ this._pendingTimeframe = _timeframe;
707
711
  }
708
- connect(peerId, peerMetadata) {
709
- this.peerId = peerId;
710
- this.peerMetadata = peerMetadata;
711
- this._connected.wake();
712
+ /**
713
+ * Timeframe that was processed by ECHO.
714
+ */
715
+ get timeframe() {
716
+ return this._timeframe;
712
717
  }
713
- send(message) {
714
- this._send(message);
718
+ /**
719
+ * Timeframe that is currently being processed by ECHO.
720
+ * Will be equal to `timeframe` after the processing is complete.
721
+ */
722
+ get pendingTimeframe() {
723
+ return this._pendingTimeframe;
715
724
  }
716
- disconnect() {
725
+ setTimeframe(timeframe) {
726
+ this._timeframe = timeframe;
727
+ this._pendingTimeframe = timeframe;
728
+ this.update.emit(this._timeframe);
717
729
  }
718
- async open() {
719
- if (this._lifecycleState === import_context6.LifecycleState.OPEN) {
720
- return;
721
- }
722
- this._lifecycleState = import_context6.LifecycleState.OPEN;
723
- (0, import_log3.log)("emit ready", void 0, {
724
- F: __dxlog_file3,
725
- L: 78,
726
- S: this,
727
- C: (f, a) => f(...a)
728
- });
729
- this.emit("ready", {
730
- network: this
731
- });
730
+ updatePendingTimeframe(key, seq) {
731
+ this._pendingTimeframe = import_timeframe.Timeframe.merge(this._pendingTimeframe, new import_timeframe.Timeframe([
732
+ [
733
+ key,
734
+ seq
735
+ ]
736
+ ]));
732
737
  }
733
- async close() {
734
- if (this._lifecycleState === import_context6.LifecycleState.CLOSED) {
735
- return this;
736
- }
737
- for (const replicator of this._replicators) {
738
- await replicator.disconnect();
739
- }
740
- this._replicators.clear();
741
- this._lifecycleState = import_context6.LifecycleState.CLOSED;
738
+ updateTimeframe() {
739
+ this._timeframe = this._pendingTimeframe;
740
+ this.update.emit(this._timeframe);
742
741
  }
743
- async whenConnected() {
744
- await this._connected.wait({
745
- timeout: 1e4
746
- });
742
+ hasGaps(timeframe) {
743
+ const gaps = import_timeframe.Timeframe.dependencies(timeframe, this._timeframe);
744
+ return !gaps.isEmpty();
747
745
  }
748
- async addReplicator(replicator) {
749
- (0, import_invariant4.invariant)(this._lifecycleState === import_context6.LifecycleState.OPEN, void 0, {
750
- F: __dxlog_file3,
751
- L: 104,
752
- S: this,
753
- A: [
754
- "this._lifecycleState === LifecycleState.OPEN",
755
- ""
756
- ]
757
- });
758
- (0, import_invariant4.invariant)(this.peerId, void 0, {
759
- F: __dxlog_file3,
760
- L: 105,
761
- S: this,
762
- A: [
763
- "this.peerId",
764
- ""
765
- ]
766
- });
767
- (0, import_invariant4.invariant)(!this._replicators.has(replicator), void 0, {
768
- F: __dxlog_file3,
769
- L: 106,
746
+ async waitUntilReached(target) {
747
+ (0, import_log3.log)("waitUntilReached", {
748
+ target,
749
+ current: this._timeframe
750
+ }, {
751
+ F: __dxlog_file4,
752
+ L: 70,
770
753
  S: this,
771
- A: [
772
- "!this._replicators.has(replicator)",
773
- ""
774
- ]
754
+ C: (f, a) => f(...a)
775
755
  });
776
- this._replicators.add(replicator);
777
- await replicator.connect({
778
- peerId: this.peerId,
779
- onConnectionOpen: this._onConnectionOpen.bind(this),
780
- onConnectionClosed: this._onConnectionClosed.bind(this),
781
- onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
782
- getContainingSpaceForDocument: this._params.getContainingSpaceForDocument
756
+ await this.update.waitForCondition(() => {
757
+ (0, import_log3.log)("check if reached", {
758
+ target,
759
+ current: this._timeframe,
760
+ deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
761
+ }, {
762
+ F: __dxlog_file4,
763
+ L: 72,
764
+ S: this,
765
+ C: (f, a) => f(...a)
766
+ });
767
+ return import_timeframe.Timeframe.dependencies(target, this._timeframe).isEmpty();
783
768
  });
784
769
  }
785
- async removeReplicator(replicator) {
786
- (0, import_invariant4.invariant)(this._lifecycleState === import_context6.LifecycleState.OPEN, void 0, {
787
- F: __dxlog_file3,
788
- L: 120,
789
- S: this,
790
- A: [
791
- "this._lifecycleState === LifecycleState.OPEN",
792
- ""
793
- ]
770
+ };
771
+ _ts_decorate([
772
+ (0, import_debug.timed)(5e3)
773
+ ], TimeframeClock.prototype, "waitUntilReached", null);
774
+ var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
775
+ var createMessageSelector = (timeframeClock) => {
776
+ return (messages) => {
777
+ for (let i = 0; i < messages.length; i++) {
778
+ const { data: { timeframe } } = messages[i];
779
+ (0, import_invariant5.invariant)(timeframe, void 0, {
780
+ F: __dxlog_file5,
781
+ L: 25,
782
+ S: void 0,
783
+ A: [
784
+ "timeframe",
785
+ ""
786
+ ]
787
+ });
788
+ if (!timeframeClock.hasGaps(timeframe)) {
789
+ return i;
790
+ }
791
+ }
792
+ (0, import_log5.log)("Skipping...", void 0, {
793
+ F: __dxlog_file5,
794
+ L: 33,
795
+ S: void 0,
796
+ C: (f, a) => f(...a)
794
797
  });
795
- (0, import_invariant4.invariant)(this._replicators.has(replicator), void 0, {
796
- F: __dxlog_file3,
797
- L: 121,
798
- S: this,
799
- A: [
800
- "this._replicators.has(replicator)",
801
- ""
802
- ]
798
+ };
799
+ };
800
+ function _ts_decorate2(decorators, target, key, desc) {
801
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
802
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
803
+ r = Reflect.decorate(decorators, target, key, desc);
804
+ else
805
+ for (var i = decorators.length - 1; i >= 0; i--)
806
+ if (d = decorators[i])
807
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
808
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
809
+ }
810
+ var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
811
+ var PipelineState = class {
812
+ constructor(_feeds, _timeframeClock) {
813
+ this._feeds = _feeds;
814
+ this._timeframeClock = _timeframeClock;
815
+ this._ctx = new import_context5.Context(void 0, {
816
+ F: __dxlog_file6,
817
+ L: 41
803
818
  });
804
- await replicator.disconnect();
805
- this._replicators.delete(replicator);
806
- }
807
- async shouldAdvertise(peerId, params) {
808
- const connection = this._connections.get(peerId);
809
- if (!connection) {
810
- return false;
811
- }
812
- return connection.connection.shouldAdvertise(params);
819
+ this.timeframeUpdate = this._timeframeClock.update;
820
+ this.stalled = new import_async5.Event();
821
+ this._startTimeframe = new import_timeframe2.Timeframe();
822
+ this._reachedTarget = false;
813
823
  }
814
- shouldSyncCollection(peerId, params) {
815
- const connection = this._connections.get(peerId);
816
- if (!connection) {
817
- return false;
818
- }
819
- return connection.connection.shouldSyncCollection(params);
824
+ /**
825
+ * Latest theoretical timeframe based on the last mutation in each feed.
826
+ * NOTE: This might never be reached if the mutation dependencies
827
+ */
828
+ // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
829
+ get endTimeframe() {
830
+ return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
831
+ feedKey: feed.key,
832
+ index: feed.length - 1
833
+ })));
820
834
  }
821
- queryCollectionState(collectionId, targetId) {
822
- const message = {
823
- type: "collection-query",
824
- senderId: this.peerId,
825
- targetId,
826
- collectionId
827
- };
828
- this._send(message);
835
+ get startTimeframe() {
836
+ return this._startTimeframe;
829
837
  }
830
- sendCollectionState(collectionId, targetId, state) {
831
- const message = {
832
- type: "collection-state",
833
- senderId: this.peerId,
834
- targetId,
835
- collectionId,
836
- state
837
- };
838
- this._send(message);
838
+ get timeframe() {
839
+ return this._timeframeClock.timeframe;
839
840
  }
840
- _send(message) {
841
- const connectionEntry = this._connections.get(message.targetId);
842
- if (!connectionEntry) {
843
- throw new Error("Connection not found.");
844
- }
845
- const writeStart = Date.now();
846
- connectionEntry.writer.write(message).then(() => {
847
- const durationMs = Date.now() - writeStart;
848
- this._params.monitor?.recordMessageSent(message, durationMs);
849
- }).catch((err) => {
850
- if (connectionEntry.isOpen) {
851
- import_log3.log.catch(err, void 0, {
852
- F: __dxlog_file3,
853
- L: 181,
854
- S: this,
855
- C: (f, a) => f(...a)
856
- });
857
- }
858
- this._params.monitor?.recordMessageSendingFailed(message);
859
- });
841
+ get pendingTimeframe() {
842
+ return this._timeframeClock.pendingTimeframe;
860
843
  }
861
- // TODO(dmaretskyi): Remove.
862
- getPeersInterestedInCollection(collectionId) {
863
- return Array.from(this._connections.values()).map((connection) => {
864
- return connection.connection.shouldSyncCollection({
865
- collectionId
866
- }) ? connection.connection.peerId : null;
867
- }).filter(import_util2.nonNullable);
844
+ get targetTimeframe() {
845
+ return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
868
846
  }
869
- _onConnectionOpen(connection) {
870
- (0, import_log3.log)("Connection opened", {
871
- peerId: connection.peerId
847
+ get reachedTarget() {
848
+ return this._reachedTarget;
849
+ }
850
+ get feeds() {
851
+ return Array.from(this._feeds.values());
852
+ }
853
+ async waitUntilTimeframe(target) {
854
+ await this._timeframeClock.waitUntilReached(target);
855
+ }
856
+ setTargetTimeframe(target) {
857
+ this._targetTimeframe = target;
858
+ }
859
+ /**
860
+ * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
861
+ *
862
+ * This function will resolve immediately if the pipeline is stalled.
863
+ *
864
+ * @param timeout Timeout in milliseconds to specify the maximum wait time.
865
+ */
866
+ async waitUntilReachedTargetTimeframe({ ctx = new import_context5.Context(void 0, {
867
+ F: __dxlog_file6,
868
+ L: 129
869
+ }), timeout, breakOnStall = true } = {}) {
870
+ (0, import_log4.log)("waitUntilReachedTargetTimeframe", {
871
+ timeout,
872
+ current: this.timeframe,
873
+ target: this.targetTimeframe
872
874
  }, {
873
- F: __dxlog_file3,
874
- L: 199,
875
+ F: __dxlog_file6,
876
+ L: 133,
875
877
  S: this,
876
878
  C: (f, a) => f(...a)
877
879
  });
878
- (0, import_invariant4.invariant)(!this._connections.has(connection.peerId), void 0, {
879
- F: __dxlog_file3,
880
- L: 200,
881
- S: this,
882
- A: [
883
- "!this._connections.has(connection.peerId as PeerId)",
884
- ""
885
- ]
886
- });
887
- const reader = connection.readable.getReader();
888
- const writer = connection.writable.getWriter();
889
- const connectionEntry = {
890
- connection,
891
- reader,
892
- writer,
893
- isOpen: true
894
- };
895
- this._connections.set(connection.peerId, connectionEntry);
896
- queueMicrotask(async () => {
897
- try {
898
- while (true) {
899
- const { done, value } = await reader.read();
880
+ this._reachedTargetPromise ??= Promise.race([
881
+ this._timeframeClock.update.waitForCondition(() => {
882
+ return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
883
+ }),
884
+ ...breakOnStall ? [
885
+ this.stalled.discardParameter().waitForCount(1)
886
+ ] : []
887
+ ]);
888
+ let done = false;
889
+ if (timeout) {
890
+ return Promise.race([
891
+ (0, import_context5.rejectOnDispose)(ctx),
892
+ (0, import_context5.rejectOnDispose)(this._ctx),
893
+ this._reachedTargetPromise.then(() => {
894
+ done = true;
895
+ this._reachedTarget = true;
896
+ }),
897
+ (0, import_async5.sleepWithContext)(this._ctx, timeout).then(() => {
900
898
  if (done) {
901
- break;
899
+ return;
902
900
  }
903
- this._onMessage(value);
904
- }
905
- } catch (err) {
906
- if (connectionEntry.isOpen) {
907
- import_log3.log.catch(err, void 0, {
908
- F: __dxlog_file3,
909
- L: 219,
901
+ import_log4.log.warn("waitUntilReachedTargetTimeframe timed out", {
902
+ timeout,
903
+ current: this.timeframe,
904
+ target: this.targetTimeframe,
905
+ dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
906
+ }, {
907
+ F: __dxlog_file6,
908
+ L: 161,
910
909
  S: this,
911
910
  C: (f, a) => f(...a)
912
911
  });
913
- }
914
- }
915
- });
916
- (0, import_log3.log)("emit peer-candidate", {
917
- peerId: connection.peerId
918
- }, {
919
- F: __dxlog_file3,
920
- L: 224,
912
+ })
913
+ ]);
914
+ } else {
915
+ return this._reachedTargetPromise;
916
+ }
917
+ }
918
+ };
919
+ var Pipeline = class {
920
+ constructor() {
921
+ this._timeframeClock = new TimeframeClock(new import_timeframe2.Timeframe());
922
+ this._feeds = new import_util2.ComplexMap(import_keys2.PublicKey.hash);
923
+ this._state = new PipelineState(this._feeds, this._timeframeClock);
924
+ this._processingTrigger = new import_async5.Trigger().wake();
925
+ this._pauseTrigger = new import_async5.Trigger().wake();
926
+ this._downloads = new import_util2.ComplexMap((value) => import_keys2.PublicKey.hash(value.key));
927
+ this._isStopping = false;
928
+ this._isStarted = false;
929
+ this._isBeingConsumed = false;
930
+ this._isPaused = false;
931
+ }
932
+ get state() {
933
+ return this._state;
934
+ }
935
+ get writer() {
936
+ (0, import_invariant4.invariant)(this._writer, "Writer not set.", {
937
+ F: __dxlog_file6,
938
+ L: 243,
921
939
  S: this,
922
- C: (f, a) => f(...a)
940
+ A: [
941
+ "this._writer",
942
+ "'Writer not set.'"
943
+ ]
923
944
  });
924
- this._emitPeerCandidate(connection);
925
- this._params.monitor?.recordPeerConnected(connection.peerId);
945
+ return this._writer;
926
946
  }
927
- _onMessage(message) {
928
- if (isCollectionQueryMessage(message)) {
929
- this._params.onCollectionStateQueried(message.collectionId, message.senderId);
930
- } else if (isCollectionStateMessage(message)) {
931
- this._params.onCollectionStateReceived(message.collectionId, message.senderId, message.state);
932
- } else {
933
- this.emit("message", message);
947
+ hasFeed(feedKey) {
948
+ return this._feeds.has(feedKey);
949
+ }
950
+ getFeeds() {
951
+ return this._feedSetIterator.feeds;
952
+ }
953
+ // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
954
+ // which might be opening feeds during the mutation processing, which w
955
+ async addFeed(feed) {
956
+ this._feeds.set(feed.key, feed);
957
+ if (this._feedSetIterator) {
958
+ await this._feedSetIterator.addFeed(feed);
959
+ }
960
+ if (this._isStarted && !this._isPaused) {
961
+ this._setFeedDownloadState(feed);
934
962
  }
935
- this._params.monitor?.recordMessageReceived(message);
936
963
  }
937
- /**
938
- * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
939
- * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
940
- */
941
- _onConnectionAuthScopeChanged(connection) {
942
- (0, import_log3.log)("Connection auth scope changed", {
943
- peerId: connection.peerId
944
- }, {
945
- F: __dxlog_file3,
946
- L: 245,
964
+ setWriteFeed(feed) {
965
+ (0, import_invariant4.invariant)(!this._writer, "Writer already set.", {
966
+ F: __dxlog_file6,
967
+ L: 270,
947
968
  S: this,
948
- C: (f, a) => f(...a)
969
+ A: [
970
+ "!this._writer",
971
+ "'Writer already set.'"
972
+ ]
949
973
  });
950
- const entry = this._connections.get(connection.peerId);
951
- (0, import_invariant4.invariant)(entry, void 0, {
952
- F: __dxlog_file3,
953
- L: 247,
974
+ (0, import_invariant4.invariant)(feed.properties.writable, "Feed must be writable.", {
975
+ F: __dxlog_file6,
976
+ L: 271,
954
977
  S: this,
955
978
  A: [
956
- "entry",
957
- ""
979
+ "feed.properties.writable",
980
+ "'Feed must be writable.'"
958
981
  ]
959
982
  });
960
- this.emit("peer-disconnected", {
961
- peerId: connection.peerId
962
- });
963
- this._emitPeerCandidate(connection);
983
+ this._writer = createMappedFeedWriter((payload) => ({
984
+ timeframe: this._timeframeClock.timeframe,
985
+ payload
986
+ }), feed.createFeedWriter());
964
987
  }
965
- _onConnectionClosed(connection) {
966
- (0, import_log3.log)("Connection closed", {
967
- peerId: connection.peerId
968
- }, {
969
- F: __dxlog_file3,
970
- L: 253,
971
- S: this,
972
- C: (f, a) => f(...a)
973
- });
974
- const entry = this._connections.get(connection.peerId);
975
- (0, import_invariant4.invariant)(entry, void 0, {
976
- F: __dxlog_file3,
977
- L: 255,
988
+ async start() {
989
+ (0, import_invariant4.invariant)(!this._isStarted, "Pipeline is already started.", {
990
+ F: __dxlog_file6,
991
+ L: 284,
978
992
  S: this,
979
993
  A: [
980
- "entry",
981
- ""
994
+ "!this._isStarted",
995
+ "'Pipeline is already started.'"
982
996
  ]
983
997
  });
984
- entry.isOpen = false;
985
- this.emit("peer-disconnected", {
986
- peerId: connection.peerId
987
- });
988
- this._params.monitor?.recordPeerDisconnected(connection.peerId);
989
- void entry.reader.cancel().catch((err) => import_log3.log.catch(err, void 0, {
990
- F: __dxlog_file3,
991
- L: 261,
998
+ (0, import_log4.log)("starting...", void 0, {
999
+ F: __dxlog_file6,
1000
+ L: 285,
992
1001
  S: this,
993
1002
  C: (f, a) => f(...a)
994
- }));
995
- void entry.writer.abort().catch((err) => import_log3.log.catch(err, void 0, {
996
- F: __dxlog_file3,
997
- L: 262,
1003
+ });
1004
+ await this._initIterator();
1005
+ await this._feedSetIterator.open();
1006
+ this._isStarted = true;
1007
+ (0, import_log4.log)("started", void 0, {
1008
+ F: __dxlog_file6,
1009
+ L: 289,
998
1010
  S: this,
999
1011
  C: (f, a) => f(...a)
1000
- }));
1001
- this._connections.delete(connection.peerId);
1002
- }
1003
- _emitPeerCandidate(connection) {
1004
- this.emit("peer-candidate", {
1005
- peerId: connection.peerId,
1006
- peerMetadata: createEchoPeerMetadata()
1007
1012
  });
1013
+ if (!this._isPaused) {
1014
+ for (const feed of this._feeds.values()) {
1015
+ this._setFeedDownloadState(feed);
1016
+ }
1017
+ }
1008
1018
  }
1009
- };
1010
- _ts_decorate([
1011
- import_async4.synchronized
1012
- ], EchoNetworkAdapter.prototype, "open", null);
1013
- _ts_decorate([
1014
- import_async4.synchronized
1015
- ], EchoNetworkAdapter.prototype, "close", null);
1016
- _ts_decorate([
1017
- import_async4.synchronized
1018
- ], EchoNetworkAdapter.prototype, "addReplicator", null);
1019
- _ts_decorate([
1020
- import_async4.synchronized
1021
- ], EchoNetworkAdapter.prototype, "removeReplicator", null);
1022
- var createEchoPeerMetadata = () => ({
1023
- // TODO(dmaretskyi): Refactor this.
1024
- dxos_peerSource: "EchoNetworkAdapter"
1025
- });
1026
- var isEchoPeerMetadata = (metadata) => metadata?.dxos_peerSource === "EchoNetworkAdapter";
1027
- var HeadsStore = class {
1028
- constructor({ db }) {
1029
- this._db = db;
1030
- }
1031
- setHeads(documentId, heads, batch) {
1032
- batch.put(documentId, heads, {
1033
- sublevel: this._db,
1034
- keyEncoding: "utf8",
1035
- valueEncoding: import_indexing.headsEncoding
1036
- });
1037
- }
1038
- // TODO(dmaretskyi): Make batched.
1039
- async getHeads(documentIds) {
1040
- return this._db.getMany(documentIds, {
1041
- keyEncoding: "utf8",
1042
- valueEncoding: import_indexing.headsEncoding
1043
- });
1044
- }
1045
- };
1046
- function _ts_decorate2(decorators, target, key, desc) {
1047
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1048
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1049
- r = Reflect.decorate(decorators, target, key, desc);
1050
- else
1051
- for (var i = decorators.length - 1; i >= 0; i--)
1052
- if (d = decorators[i])
1053
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1054
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1055
- }
1056
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
1057
- var AutomergeHost = class extends import_context5.Resource {
1058
- constructor({ db, indexMetadataStore, dataMonitor }) {
1059
- super();
1060
- this._collectionSynchronizer = new CollectionSynchronizer({
1061
- queryCollectionState: this._queryCollectionState.bind(this),
1062
- sendCollectionState: this._sendCollectionState.bind(this),
1063
- shouldSyncCollection: this._shouldSyncCollection.bind(this)
1064
- });
1065
- this._db = db;
1066
- this._storage = new LevelDBStorageAdapter({
1067
- db: db.sublevel("automerge"),
1068
- callbacks: {
1069
- beforeSave: async (params) => this._beforeSave(params),
1070
- afterSave: async (key) => this._afterSave(key)
1071
- },
1072
- monitor: dataMonitor
1073
- });
1074
- this._echoNetworkAdapter = new EchoNetworkAdapter({
1075
- getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
1076
- onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
1077
- onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
1078
- monitor: dataMonitor
1079
- });
1080
- this._headsStore = new HeadsStore({
1081
- db: db.sublevel("heads")
1019
+ async stop() {
1020
+ (0, import_log4.log)("stopping...", void 0, {
1021
+ F: __dxlog_file6,
1022
+ L: 300,
1023
+ S: this,
1024
+ C: (f, a) => f(...a)
1082
1025
  });
1083
- this._indexMetadataStore = indexMetadataStore;
1084
- }
1085
- async _open() {
1086
- this._peerId = `host-${import_keys2.PublicKey.random().toHex()}`;
1087
- await this._storage.open?.();
1088
- this._repo = new import_automerge_repo.Repo({
1089
- peerId: this._peerId,
1090
- sharePolicy: this._sharePolicy.bind(this),
1091
- storage: this._storage,
1092
- network: [
1093
- // Upstream swarm.
1094
- this._echoNetworkAdapter
1095
- ]
1026
+ this._isStopping = true;
1027
+ for (const [feed, handle] of this._downloads.entries()) {
1028
+ feed.undownload(handle);
1029
+ }
1030
+ this._downloads.clear();
1031
+ await this._feedSetIterator?.close();
1032
+ await this._processingTrigger.wait();
1033
+ await this._state._ctx.dispose();
1034
+ this._state._ctx = new import_context5.Context(void 0, {
1035
+ F: __dxlog_file6,
1036
+ L: 309
1096
1037
  });
1097
- import_async3.Event.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => this._onPeerConnected(e.peerId));
1098
- import_async3.Event.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => this._onPeerDisconnected(e.peerId));
1099
- this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
1100
- this._onRemoteCollectionStateUpdated(collectionId, peerId);
1038
+ this._state._reachedTargetPromise = void 0;
1039
+ this._state._reachedTarget = false;
1040
+ this._isStarted = false;
1041
+ (0, import_log4.log)("stopped", void 0, {
1042
+ F: __dxlog_file6,
1043
+ L: 313,
1044
+ S: this,
1045
+ C: (f, a) => f(...a)
1101
1046
  });
1102
- await this._echoNetworkAdapter.open();
1103
- await this._collectionSynchronizer.open();
1104
- await this._echoNetworkAdapter.open();
1105
- await this._echoNetworkAdapter.whenConnected();
1106
- }
1107
- async _close() {
1108
- await this._collectionSynchronizer.close();
1109
- await this._storage.close?.();
1110
- await this._echoNetworkAdapter.close();
1111
- await this._ctx.dispose();
1112
1047
  }
1113
1048
  /**
1114
- * @deprecated To be abstracted away.
1049
+ * @param timeframe Timeframe of already processed messages.
1050
+ * The pipeline will start processing messages AFTER this timeframe.
1115
1051
  */
1116
- get repo() {
1117
- return this._repo;
1118
- }
1119
- get peerId() {
1120
- return this._peerId;
1121
- }
1122
- get loadedDocsCount() {
1123
- return Object.keys(this._repo.handles).length;
1124
- }
1125
- async addReplicator(replicator) {
1126
- await this._echoNetworkAdapter.addReplicator(replicator);
1127
- }
1128
- async removeReplicator(replicator) {
1129
- await this._echoNetworkAdapter.removeReplicator(replicator);
1052
+ async setCursor(timeframe) {
1053
+ (0, import_invariant4.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
1054
+ F: __dxlog_file6,
1055
+ L: 322,
1056
+ S: this,
1057
+ A: [
1058
+ "!this._isStarted || this._isPaused",
1059
+ "'Invalid state.'"
1060
+ ]
1061
+ });
1062
+ this._state._startTimeframe = timeframe;
1063
+ this._timeframeClock.setTimeframe(timeframe);
1064
+ if (this._feedSetIterator) {
1065
+ await this._feedSetIterator.close();
1066
+ await this._initIterator();
1067
+ await this._feedSetIterator.open();
1068
+ }
1130
1069
  }
1131
1070
  /**
1132
- * Loads the document handle from the repo and waits for it to be ready.
1071
+ * Calling pause while processing will cause a deadlock.
1133
1072
  */
1134
- async loadDoc(ctx, documentId, opts) {
1135
- let handle;
1136
- if (typeof documentId === "string") {
1137
- handle = this._repo.handles[documentId];
1138
- }
1139
- if (!handle) {
1140
- handle = this._repo.find(documentId);
1073
+ async pause() {
1074
+ if (this._isPaused) {
1075
+ return;
1141
1076
  }
1142
- if (!handle.isReady()) {
1143
- if (!opts?.timeout) {
1144
- await (0, import_context5.cancelWithContext)(ctx, handle.whenReady());
1145
- } else {
1146
- await (0, import_context5.cancelWithContext)(ctx, (0, import_async3.asyncTimeout)(handle.whenReady(), opts.timeout));
1147
- }
1077
+ this._pauseTrigger.reset();
1078
+ await this._processingTrigger.wait();
1079
+ this._isPaused = true;
1080
+ }
1081
+ async unpause() {
1082
+ (0, import_invariant4.invariant)(this._isPaused, "Pipeline is not paused.", {
1083
+ F: __dxlog_file6,
1084
+ L: 351,
1085
+ S: this,
1086
+ A: [
1087
+ "this._isPaused",
1088
+ "'Pipeline is not paused.'"
1089
+ ]
1090
+ });
1091
+ this._pauseTrigger.wake();
1092
+ this._isPaused = false;
1093
+ for (const feed of this._feeds.values()) {
1094
+ this._setFeedDownloadState(feed);
1148
1095
  }
1149
- return handle;
1150
1096
  }
1151
1097
  /**
1152
- * Create new persisted document.
1098
+ * Starts to iterate over the ordered messages from the added feeds.
1099
+ * Updates the timeframe clock after the message has bee processed.
1153
1100
  */
1154
- createDoc(initialValue, opts) {
1155
- if (opts?.preserveHistory) {
1156
- if (!(0, import_automerge3.isAutomerge)(initialValue)) {
1157
- throw new TypeError("Initial value must be an Automerge document");
1101
+ async *consume() {
1102
+ (0, import_invariant4.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
1103
+ F: __dxlog_file6,
1104
+ L: 366,
1105
+ S: this,
1106
+ A: [
1107
+ "!this._isBeingConsumed",
1108
+ "'Pipeline is already being consumed.'"
1109
+ ]
1110
+ });
1111
+ this._isBeingConsumed = true;
1112
+ (0, import_invariant4.invariant)(this._feedSetIterator, "Iterator not initialized.", {
1113
+ F: __dxlog_file6,
1114
+ L: 369,
1115
+ S: this,
1116
+ A: [
1117
+ "this._feedSetIterator",
1118
+ "'Iterator not initialized.'"
1119
+ ]
1120
+ });
1121
+ let lastFeedSetIterator = this._feedSetIterator;
1122
+ let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1123
+ while (!this._isStopping) {
1124
+ await this._pauseTrigger.wait();
1125
+ if (lastFeedSetIterator !== this._feedSetIterator) {
1126
+ (0, import_invariant4.invariant)(this._feedSetIterator, "Iterator not initialized.", {
1127
+ F: __dxlog_file6,
1128
+ L: 378,
1129
+ S: this,
1130
+ A: [
1131
+ "this._feedSetIterator",
1132
+ "'Iterator not initialized.'"
1133
+ ]
1134
+ });
1135
+ lastFeedSetIterator = this._feedSetIterator;
1136
+ iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1137
+ }
1138
+ const { done, value } = await iterable.next();
1139
+ if (!done) {
1140
+ const block = value ?? (0, import_debug2.failUndefined)();
1141
+ this._processingTrigger.reset();
1142
+ this._timeframeClock.updatePendingTimeframe(import_keys2.PublicKey.from(block.feedKey), block.seq);
1143
+ yield block;
1144
+ this._processingTrigger.wake();
1145
+ this._timeframeClock.updateTimeframe();
1158
1146
  }
1159
- return this._repo.import((0, import_automerge3.save)(initialValue));
1160
- } else {
1161
- return this._repo.create(initialValue);
1162
1147
  }
1148
+ this._isBeingConsumed = false;
1163
1149
  }
1164
- async waitUntilHeadsReplicated(heads) {
1165
- const entries = heads.entries;
1166
- if (!entries?.length) {
1167
- return;
1150
+ _setFeedDownloadState(feed) {
1151
+ let handle = this._downloads.get(feed);
1152
+ if (handle) {
1153
+ feed.undownload(handle);
1168
1154
  }
1169
- const documentIds = entries.map((entry) => entry.documentId);
1170
- const documentHeads = await this.getHeads(documentIds);
1171
- const headsToWait = entries.filter((entry, index) => {
1172
- const targetHeads = entry.heads;
1173
- if (!targetHeads || targetHeads.length === 0) {
1174
- return false;
1155
+ const timeframe = this._state._startTimeframe;
1156
+ const seq = timeframe.get(feed.key) ?? -1;
1157
+ (0, import_log4.log)("download", {
1158
+ feed: feed.key.truncate(),
1159
+ seq,
1160
+ length: feed.length
1161
+ }, {
1162
+ F: __dxlog_file6,
1163
+ L: 407,
1164
+ S: this,
1165
+ C: (f, a) => f(...a)
1166
+ });
1167
+ handle = feed.download({
1168
+ start: seq + 1,
1169
+ linear: true
1170
+ }, (err, data) => {
1171
+ if (err) {
1172
+ } else {
1173
+ import_log4.log.info("downloaded", {
1174
+ data
1175
+ }, {
1176
+ F: __dxlog_file6,
1177
+ L: 412,
1178
+ S: this,
1179
+ C: (f, a) => f(...a)
1180
+ });
1175
1181
  }
1176
- const currentHeads = documentHeads[index];
1177
- return !(currentHeads !== null && (0, import_automerge3.equals)(currentHeads, targetHeads));
1178
1182
  });
1179
- if (headsToWait.length > 0) {
1180
- await Promise.all(headsToWait.map(async (entry, index) => {
1181
- const handle = await this.loadDoc(import_context5.Context.default(void 0, {
1182
- F: __dxlog_file4,
1183
- L: 226
1184
- }), entry.documentId);
1185
- await waitForHeads(handle, entry.heads);
1186
- }));
1183
+ this._downloads.set(feed, handle);
1184
+ }
1185
+ async _initIterator() {
1186
+ this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
1187
+ start: startAfter(this._timeframeClock.timeframe),
1188
+ stallTimeout: 1e3
1189
+ });
1190
+ this._feedSetIterator.stalled.on((iterator) => {
1191
+ import_log4.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
1192
+ F: __dxlog_file6,
1193
+ L: 426,
1194
+ S: this,
1195
+ C: (f, a) => f(...a)
1196
+ });
1197
+ this._state.stalled.emit();
1198
+ });
1199
+ for (const feed of this._feeds.values()) {
1200
+ await this._feedSetIterator.addFeed(feed);
1187
1201
  }
1188
- await this._repo.flush(documentIds.filter((documentId) => !!this._repo.handles[documentId]));
1189
1202
  }
1190
- async reIndexHeads(documentIds) {
1191
- for (const documentId of documentIds) {
1192
- import_log2.log.info("re-indexing heads for document", {
1193
- documentId
1203
+ };
1204
+ _ts_decorate2([
1205
+ import_async5.synchronized
1206
+ ], Pipeline.prototype, "start", null);
1207
+ _ts_decorate2([
1208
+ import_async5.synchronized
1209
+ ], Pipeline.prototype, "stop", null);
1210
+ _ts_decorate2([
1211
+ import_async5.synchronized
1212
+ ], Pipeline.prototype, "setCursor", null);
1213
+ _ts_decorate2([
1214
+ import_async5.synchronized
1215
+ ], Pipeline.prototype, "pause", null);
1216
+ _ts_decorate2([
1217
+ import_async5.synchronized
1218
+ ], Pipeline.prototype, "unpause", null);
1219
+ function _ts_decorate3(decorators, target, key, desc) {
1220
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1221
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1222
+ r = Reflect.decorate(decorators, target, key, desc);
1223
+ else
1224
+ for (var i = decorators.length - 1; i >= 0; i--)
1225
+ if (d = decorators[i])
1226
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1227
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1228
+ }
1229
+ var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
1230
+ var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
1231
+ var CONTROL_PIPELINE_SNAPSHOT_DELAY = 1e4;
1232
+ var USE_SNAPSHOTS = true;
1233
+ var ControlPipeline = class {
1234
+ constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
1235
+ this._ctx = new import_context7.Context(void 0, {
1236
+ F: __dxlog_file7,
1237
+ L: 47
1238
+ });
1239
+ this._lastTimeframeSaveTime = Date.now();
1240
+ this.onFeedAdmitted = new import_util4.Callback();
1241
+ this._usage = new import_tracing2.TimeUsageCounter();
1242
+ this._mutations = new import_tracing2.TimeSeriesCounter();
1243
+ this._snapshotTask = new import_async7.DeferredTask(this._ctx, async () => {
1244
+ await (0, import_async7.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
1245
+ await this._saveSnapshot();
1246
+ });
1247
+ this._spaceKey = spaceKey;
1248
+ this._metadata = metadataStore;
1249
+ this._pipeline = new Pipeline();
1250
+ void this._pipeline.addFeed(genesisFeed);
1251
+ this._spaceStateMachine = new import_credentials2.SpaceStateMachine(spaceKey);
1252
+ this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
1253
+ (0, import_log7.log)("feed admitted", {
1254
+ key: info.key
1194
1255
  }, {
1195
- F: __dxlog_file4,
1196
- L: 238,
1256
+ F: __dxlog_file7,
1257
+ L: 82,
1197
1258
  S: this,
1198
1259
  C: (f, a) => f(...a)
1199
1260
  });
1200
- const handle = this._repo.find(documentId);
1201
- await handle.whenReady([
1202
- "ready",
1203
- "requesting"
1204
- ]);
1205
- if (handle.inState([
1206
- "requesting"
1207
- ])) {
1208
- import_log2.log.warn("document is not available locally, skipping", {
1209
- documentId
1261
+ if (info.assertion.designation === import_credentials3.AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
1262
+ queueMicrotask(async () => {
1263
+ try {
1264
+ const feed = await feedProvider(info.key);
1265
+ if (!this._pipeline.hasFeed(feed.key)) {
1266
+ await this._pipeline.addFeed(feed);
1267
+ }
1268
+ } catch (err) {
1269
+ import_log7.log.catch(err, void 0, {
1270
+ F: __dxlog_file7,
1271
+ L: 93,
1272
+ S: this,
1273
+ C: (f, a) => f(...a)
1274
+ });
1275
+ }
1276
+ });
1277
+ }
1278
+ await this.onFeedAdmitted.callIfSet(info);
1279
+ });
1280
+ this.onMemberRoleChanged = this._spaceStateMachine.onMemberRoleChanged;
1281
+ this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
1282
+ this.onDelegatedInvitation = this._spaceStateMachine.onDelegatedInvitation;
1283
+ this.onDelegatedInvitationRemoved = this._spaceStateMachine.onDelegatedInvitationRemoved;
1284
+ }
1285
+ get spaceState() {
1286
+ return this._spaceStateMachine;
1287
+ }
1288
+ get pipeline() {
1289
+ return this._pipeline;
1290
+ }
1291
+ async setWriteFeed(feed) {
1292
+ await this._pipeline.addFeed(feed);
1293
+ this._pipeline.setWriteFeed(feed);
1294
+ }
1295
+ async start() {
1296
+ const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
1297
+ (0, import_log7.log)("load snapshot", {
1298
+ key: this._spaceKey,
1299
+ present: !!snapshot,
1300
+ tf: snapshot?.timeframe
1301
+ }, {
1302
+ F: __dxlog_file7,
1303
+ L: 123,
1304
+ S: this,
1305
+ C: (f, a) => f(...a)
1306
+ });
1307
+ if (USE_SNAPSHOTS && snapshot) {
1308
+ await this._processSnapshot(snapshot);
1309
+ }
1310
+ (0, import_log7.log)("starting...", void 0, {
1311
+ F: __dxlog_file7,
1312
+ L: 128,
1313
+ S: this,
1314
+ C: (f, a) => f(...a)
1315
+ });
1316
+ setTimeout(async () => {
1317
+ void this._consumePipeline(new import_context7.Context(void 0, {
1318
+ F: __dxlog_file7,
1319
+ L: 130
1320
+ }));
1321
+ });
1322
+ await this._pipeline.start();
1323
+ (0, import_log7.log)("started", void 0, {
1324
+ F: __dxlog_file7,
1325
+ L: 134,
1326
+ S: this,
1327
+ C: (f, a) => f(...a)
1328
+ });
1329
+ }
1330
+ async _processSnapshot(snapshot) {
1331
+ await this._pipeline.setCursor(snapshot.timeframe);
1332
+ for (const message of snapshot.messages ?? []) {
1333
+ const result = await this._spaceStateMachine.process(message.credential, {
1334
+ sourceFeed: message.feedKey,
1335
+ skipVerification: true
1336
+ });
1337
+ if (!result) {
1338
+ import_log7.log.warn("credential processing failed from snapshot", {
1339
+ message
1210
1340
  }, {
1211
- F: __dxlog_file4,
1212
- L: 242,
1341
+ F: __dxlog_file7,
1342
+ L: 147,
1213
1343
  S: this,
1214
1344
  C: (f, a) => f(...a)
1215
1345
  });
1216
- continue;
1217
1346
  }
1218
- const doc = handle.docSync();
1219
- (0, import_invariant3.invariant)(doc, void 0, {
1220
- F: __dxlog_file4,
1221
- L: 247,
1222
- S: this,
1223
- A: [
1224
- "doc",
1225
- ""
1226
- ]
1227
- });
1228
- const heads = (0, import_automerge3.getHeads)(doc);
1229
- const batch = this._db.batch();
1230
- this._headsStore.setHeads(documentId, heads, batch);
1231
- await batch.write();
1232
1347
  }
1233
- import_log2.log.info("done re-indexing heads", void 0, {
1234
- F: __dxlog_file4,
1235
- L: 254,
1348
+ }
1349
+ async _saveSnapshot() {
1350
+ await this._pipeline.pause();
1351
+ const snapshot = {
1352
+ timeframe: this._pipeline.state.timeframe,
1353
+ messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
1354
+ feedKey: entry.sourceFeed,
1355
+ credential: entry.credential
1356
+ }))
1357
+ };
1358
+ await this._pipeline.unpause();
1359
+ (0, import_log7.log)("save snapshot", {
1360
+ key: this._spaceKey,
1361
+ snapshot
1362
+ }, {
1363
+ F: __dxlog_file7,
1364
+ L: 163,
1236
1365
  S: this,
1237
1366
  C: (f, a) => f(...a)
1238
1367
  });
1368
+ await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
1239
1369
  }
1240
- // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
1241
- // Hosts, running in the worker, don't share documents unless requested by other peers.
1242
- // NOTE: If both peers return sharePolicy=false the replication will not happen
1243
- // https://github.com/automerge/automerge-repo/pull/292
1244
- async _sharePolicy(peerId, documentId) {
1245
- if (peerId.startsWith("client-")) {
1246
- return false;
1247
- }
1248
- if (!documentId) {
1249
- return false;
1370
+ async _consumePipeline(ctx) {
1371
+ for await (const msg of this._pipeline.consume()) {
1372
+ const span = this._usage.beginRecording();
1373
+ this._mutations.inc();
1374
+ try {
1375
+ await this._processMessage(ctx, msg);
1376
+ } catch (err) {
1377
+ import_log7.log.catch(err, void 0, {
1378
+ F: __dxlog_file7,
1379
+ L: 176,
1380
+ S: this,
1381
+ C: (f, a) => f(...a)
1382
+ });
1383
+ }
1384
+ span.end();
1250
1385
  }
1251
- const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
1252
- if (isEchoPeerMetadata(peerMetadata)) {
1253
- return this._echoNetworkAdapter.shouldAdvertise(peerId, {
1254
- documentId
1386
+ }
1387
+ async _processMessage(ctx, msg) {
1388
+ (0, import_log7.log)("processing", {
1389
+ key: msg.feedKey,
1390
+ seq: msg.seq
1391
+ }, {
1392
+ F: __dxlog_file7,
1393
+ L: 186,
1394
+ S: this,
1395
+ C: (f, a) => f(...a)
1396
+ });
1397
+ if (msg.data.payload.credential) {
1398
+ const timer = import_util4.tracer.mark("dxos.echo.pipeline.control");
1399
+ const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
1400
+ sourceFeed: import_keys4.PublicKey.from(msg.feedKey)
1255
1401
  });
1402
+ timer.end();
1403
+ if (!result) {
1404
+ import_log7.log.warn("processing failed", {
1405
+ msg
1406
+ }, {
1407
+ F: __dxlog_file7,
1408
+ L: 195,
1409
+ S: this,
1410
+ C: (f, a) => f(...a)
1411
+ });
1412
+ } else {
1413
+ await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
1414
+ }
1415
+ this._snapshotTask.schedule();
1256
1416
  }
1257
- return false;
1258
1417
  }
1259
- async _beforeSave({ path, batch }) {
1260
- const handle = this._repo.handles[path[0]];
1261
- if (!handle) {
1262
- return;
1263
- }
1264
- const doc = handle.docSync();
1265
- if (!doc) {
1266
- return;
1418
+ async _noteTargetStateIfNeeded(timeframe) {
1419
+ if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
1420
+ this._lastTimeframeSaveTime = Date.now();
1421
+ await this._saveTargetTimeframe(timeframe);
1267
1422
  }
1268
- const heads = (0, import_automerge3.getHeads)(doc);
1269
- this._headsStore.setHeads(handle.documentId, heads, batch);
1270
- const spaceKey = getSpaceKeyFromDoc(doc) ?? void 0;
1271
- const objectIds = Object.keys(doc.objects ?? {});
1272
- const encodedIds = objectIds.map((objectId) => import_protocols4.objectPointerCodec.encode({
1273
- documentId: handle.documentId,
1274
- objectId,
1275
- spaceKey
1276
- }));
1277
- const idToLastHash = new Map(encodedIds.map((id) => [
1278
- id,
1279
- heads
1280
- ]));
1281
- this._indexMetadataStore.markDirty(idToLastHash, batch);
1282
- }
1283
- _shouldSyncCollection(collectionId, peerId) {
1284
- const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
1285
- if (isEchoPeerMetadata(peerMetadata)) {
1286
- return this._echoNetworkAdapter.shouldSyncCollection(peerId, {
1287
- collectionId
1288
- });
1289
- }
1290
- return false;
1291
- }
1292
- /**
1293
- * Called by AutomergeStorageAdapter after levelDB batch commit.
1294
- */
1295
- async _afterSave(path) {
1296
- this._indexMetadataStore.notifyMarkedDirty();
1297
- const documentId = path[0];
1298
- const document = this._repo.handles[documentId]?.docSync();
1299
- if (document) {
1300
- const heads = (0, import_automerge3.getHeads)(document);
1301
- this._onHeadsChanged(documentId, heads);
1302
- }
1303
- }
1304
- _automergePeers() {
1305
- return this._repo.peers;
1306
- }
1307
- async _getContainingSpaceForDocument(documentId) {
1308
- const doc = this._repo.handles[documentId]?.docSync();
1309
- if (!doc) {
1310
- return null;
1311
- }
1312
- const spaceKeyHex = getSpaceKeyFromDoc(doc);
1313
- if (!spaceKeyHex) {
1314
- return null;
1315
- }
1316
- return import_keys2.PublicKey.from(spaceKeyHex);
1317
- }
1318
- /**
1319
- * Flush documents to disk.
1320
- */
1321
- async flush({ documentIds } = {}) {
1322
- await this._repo.flush(documentIds);
1323
- }
1324
- async getHeads(documentIds) {
1325
- const result = [];
1326
- const storeRequestIds = [];
1327
- const storeResultIndices = [];
1328
- for (const documentId of documentIds) {
1329
- const doc = this._repo.handles[documentId]?.docSync();
1330
- if (doc) {
1331
- result.push((0, import_automerge3.getHeads)(doc));
1332
- } else {
1333
- storeRequestIds.push(documentId);
1334
- storeResultIndices.push(result.length);
1335
- result.push(void 0);
1336
- }
1337
- }
1338
- if (storeRequestIds.length > 0) {
1339
- const storedHeads = await this._headsStore.getHeads(storeRequestIds);
1340
- for (let i = 0; i < storedHeads.length; i++) {
1341
- result[storeResultIndices[i]] = storedHeads[i];
1342
- }
1343
- }
1344
- return result;
1345
- }
1346
- //
1347
- // Collection sync.
1348
- //
1349
- getLocalCollectionState(collectionId) {
1350
- return this._collectionSynchronizer.getLocalCollectionState(collectionId);
1351
- }
1352
- getRemoteCollectionStates(collectionId) {
1353
- return this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
1354
- }
1355
- refreshCollection(collectionId) {
1356
- this._collectionSynchronizer.refreshCollection(collectionId);
1357
- }
1358
- async getCollectionSyncState(collectionId) {
1359
- const result = {
1360
- peers: []
1361
- };
1362
- const localState = this.getLocalCollectionState(collectionId);
1363
- const remoteState = this.getRemoteCollectionStates(collectionId);
1364
- if (!localState) {
1365
- return result;
1366
- }
1367
- for (const [peerId, state] of remoteState) {
1368
- const diff = diffCollectionState(localState, state);
1369
- result.peers.push({
1370
- peerId,
1371
- differentDocuments: diff.different.length
1372
- });
1373
- }
1374
- return result;
1375
1423
  }
1376
- /**
1377
- * Update the local collection state based on the locally stored document heads.
1378
- */
1379
- async updateLocalCollectionState(collectionId, documentIds) {
1380
- const heads = await this.getHeads(documentIds);
1381
- const documents = Object.fromEntries(heads.map((heads2, index) => [
1382
- documentIds[index],
1383
- heads2 ?? []
1384
- ]));
1385
- this._collectionSynchronizer.setLocalCollectionState(collectionId, {
1386
- documents
1424
+ async stop() {
1425
+ (0, import_log7.log)("stopping...", void 0, {
1426
+ F: __dxlog_file7,
1427
+ L: 215,
1428
+ S: this,
1429
+ C: (f, a) => f(...a)
1387
1430
  });
1388
- }
1389
- _onCollectionStateQueried(collectionId, peerId) {
1390
- this._collectionSynchronizer.onCollectionStateQueried(collectionId, peerId);
1391
- }
1392
- _onCollectionStateReceived(collectionId, peerId, state) {
1393
- this._collectionSynchronizer.onRemoteStateReceived(collectionId, peerId, decodeCollectionState(state));
1394
- }
1395
- _queryCollectionState(collectionId, peerId) {
1396
- this._echoNetworkAdapter.queryCollectionState(collectionId, peerId);
1397
- }
1398
- _sendCollectionState(collectionId, peerId, state) {
1399
- this._echoNetworkAdapter.sendCollectionState(collectionId, peerId, encodeCollectionState(state));
1400
- }
1401
- _onPeerConnected(peerId) {
1402
- this._collectionSynchronizer.onConnectionOpen(peerId);
1403
- }
1404
- _onPeerDisconnected(peerId) {
1405
- this._collectionSynchronizer.onConnectionClosed(peerId);
1406
- }
1407
- _onRemoteCollectionStateUpdated(collectionId, peerId) {
1408
- const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
1409
- const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
1410
- if (!localState || !remoteState) {
1411
- return;
1412
- }
1413
- const { different } = diffCollectionState(localState, remoteState);
1414
- if (different.length === 0) {
1415
- return;
1416
- }
1417
- import_log2.log.info("replication documents after collection sync", {
1418
- count: different.length
1419
- }, {
1420
- F: __dxlog_file4,
1421
- L: 463,
1431
+ await this._ctx.dispose();
1432
+ await this._pipeline.stop();
1433
+ await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1434
+ (0, import_log7.log)("stopped", void 0, {
1435
+ F: __dxlog_file7,
1436
+ L: 219,
1422
1437
  S: this,
1423
1438
  C: (f, a) => f(...a)
1424
1439
  });
1425
- for (const documentId of different) {
1426
- this._repo.find(documentId);
1427
- }
1428
1440
  }
1429
- _onHeadsChanged(documentId, heads) {
1430
- for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
1431
- const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
1432
- if (state?.documents[documentId]) {
1433
- const newState = structuredClone(state);
1434
- newState.documents[documentId] = heads;
1435
- this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
1436
- }
1441
+ async _saveTargetTimeframe(timeframe) {
1442
+ try {
1443
+ const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
1444
+ await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
1445
+ this._targetTimeframe = newTimeframe;
1446
+ } catch (err) {
1447
+ (0, import_log7.log)(err, void 0, {
1448
+ F: __dxlog_file7,
1449
+ L: 228,
1450
+ S: this,
1451
+ C: (f, a) => f(...a)
1452
+ });
1437
1453
  }
1438
1454
  }
1439
1455
  };
1440
- _ts_decorate2([
1441
- import_tracing.trace.info()
1442
- ], AutomergeHost.prototype, "_peerId", void 0);
1443
- _ts_decorate2([
1444
- import_tracing.trace.info({
1445
- depth: null
1446
- })
1447
- ], AutomergeHost.prototype, "_automergePeers", null);
1448
- _ts_decorate2([
1449
- import_tracing.trace.span({
1456
+ _ts_decorate3([
1457
+ import_tracing2.trace.metricsCounter()
1458
+ ], ControlPipeline.prototype, "_usage", void 0);
1459
+ _ts_decorate3([
1460
+ import_tracing2.trace.metricsCounter()
1461
+ ], ControlPipeline.prototype, "_mutations", void 0);
1462
+ _ts_decorate3([
1463
+ import_tracing2.trace.span({
1450
1464
  showInBrowserTimeline: true
1451
1465
  })
1452
- ], AutomergeHost.prototype, "flush", null);
1453
- AutomergeHost = _ts_decorate2([
1454
- import_tracing.trace.resource()
1455
- ], AutomergeHost);
1456
- var getSpaceKeyFromDoc = (doc) => {
1457
- const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
1458
- if (rawSpaceKey == null) {
1459
- return null;
1460
- }
1461
- return String(rawSpaceKey);
1462
- };
1463
- var waitForHeads = async (handle, heads) => {
1464
- const unavailableHeads = new Set(heads);
1465
- await handle.whenReady();
1466
- await import_async3.Event.wrap(handle, "change").waitForCondition(() => {
1467
- for (const changeHash of unavailableHeads.values()) {
1468
- if (changeIsPresentInDoc(handle.docSync(), changeHash)) {
1469
- unavailableHeads.delete(changeHash);
1470
- }
1471
- }
1472
- return unavailableHeads.size === 0;
1473
- });
1474
- };
1475
- var changeIsPresentInDoc = (doc, changeHash) => {
1476
- return !!(0, import_automerge3.getBackend)(doc).getChangeByHash(changeHash);
1477
- };
1478
- var decodeCollectionState = (state) => {
1479
- (0, import_invariant3.invariant)(typeof state === "object" && state !== null, "Invalid state", {
1480
- F: __dxlog_file4,
1481
- L: 516,
1482
- S: void 0,
1483
- A: [
1484
- "typeof state === 'object' && state !== null",
1485
- "'Invalid state'"
1486
- ]
1487
- });
1488
- return state;
1489
- };
1490
- var encodeCollectionState = (state) => {
1491
- return state;
1492
- };
1493
- var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
1494
- var deriveCollectionIdFromSpaceId = (spaceId) => `space:${spaceId}`;
1495
- var getSpaceIdFromCollectionId = (collectionId) => {
1496
- const spaceId = collectionId.replace(/^space:/, "");
1497
- (0, import_invariant5.invariant)(import_keys3.SpaceId.isValid(spaceId), void 0, {
1498
- F: __dxlog_file5,
1499
- L: 12,
1500
- S: void 0,
1501
- A: [
1502
- "SpaceId.isValid(spaceId)",
1503
- ""
1504
- ]
1505
- });
1506
- return spaceId;
1507
- };
1508
- var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/auth.ts";
1509
- var AuthExtension = class extends import_teleport.RpcExtension {
1510
- constructor(_authParams) {
1511
- super({
1512
- requested: {
1513
- AuthService: import_protocols5.schema.getService("dxos.mesh.teleport.auth.AuthService")
1514
- },
1515
- exposed: {
1516
- AuthService: import_protocols5.schema.getService("dxos.mesh.teleport.auth.AuthService")
1517
- },
1518
- timeout: 60 * 1e3
1519
- });
1520
- this._authParams = _authParams;
1521
- this._ctx = new import_context7.Context({
1522
- onError: (err) => {
1523
- import_log4.log.catch(err, void 0, {
1524
- F: __dxlog_file6,
1525
- L: 28,
1526
- S: this,
1527
- C: (f, a) => f(...a)
1528
- });
1529
- }
1530
- }, {
1531
- F: __dxlog_file6,
1532
- L: 26
1533
- });
1534
- }
1535
- async getHandlers() {
1536
- return {
1537
- AuthService: {
1538
- authenticate: async ({ challenge }) => {
1539
- try {
1540
- const credential = await this._authParams.provider(challenge);
1541
- if (!credential) {
1542
- throw new Error("auth rejected");
1543
- }
1544
- return {
1545
- credential
1546
- };
1547
- } catch (err) {
1548
- import_log4.log.error("failed to generate auth credentials", err, {
1549
- F: __dxlog_file6,
1550
- L: 55,
1551
- S: this,
1552
- C: (f, a) => f(...a)
1553
- });
1554
- throw new Error("auth rejected");
1555
- }
1556
- }
1557
- }
1558
- };
1559
- }
1560
- async onOpen(context) {
1561
- await super.onOpen(context);
1562
- (0, import_async5.scheduleTask)(this._ctx, async () => {
1563
- try {
1564
- const challenge = (0, import_crypto2.randomBytes)(32);
1565
- const { credential } = await this.rpc.AuthService.authenticate({
1566
- challenge
1567
- });
1568
- (0, import_invariant6.invariant)(credential?.length > 0, "invalid credential", {
1569
- F: __dxlog_file6,
1570
- L: 69,
1571
- S: this,
1572
- A: [
1573
- "credential?.length > 0",
1574
- "'invalid credential'"
1575
- ]
1576
- });
1577
- const success = await this._authParams.verifier(challenge, credential);
1578
- (0, import_invariant6.invariant)(success, "credential not verified", {
1579
- F: __dxlog_file6,
1580
- L: 71,
1581
- S: this,
1582
- A: [
1583
- "success",
1584
- "'credential not verified'"
1585
- ]
1586
- });
1587
- (0, import_async5.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
1588
- } catch (err) {
1589
- (0, import_log4.log)("auth failed", err, {
1590
- F: __dxlog_file6,
1591
- L: 74,
1592
- S: this,
1593
- C: (f, a) => f(...a)
1594
- });
1595
- this.close();
1596
- this._authParams.onAuthFailure();
1597
- }
1598
- });
1599
- }
1600
- async onClose() {
1601
- await this._ctx.dispose();
1602
- await super.onClose();
1603
- }
1604
- async onAbort() {
1605
- await this._ctx.dispose();
1606
- await super.onAbort();
1607
- }
1608
- };
1609
- function _ts_decorate3(decorators, target, key, desc) {
1466
+ ], ControlPipeline.prototype, "start", null);
1467
+ _ts_decorate3([
1468
+ import_tracing2.trace.span()
1469
+ ], ControlPipeline.prototype, "_consumePipeline", null);
1470
+ _ts_decorate3([
1471
+ import_tracing2.trace.span()
1472
+ ], ControlPipeline.prototype, "_processMessage", null);
1473
+ ControlPipeline = _ts_decorate3([
1474
+ import_tracing2.trace.resource(),
1475
+ (0, import_async7.trackLeaks)("start", "stop")
1476
+ ], ControlPipeline);
1477
+ function _ts_decorate4(decorators, target, key, desc) {
1610
1478
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1611
1479
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1612
1480
  r = Reflect.decorate(decorators, target, key, desc);
@@ -1616,114 +1484,289 @@ function _ts_decorate3(decorators, target, key, desc) {
1616
1484
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1617
1485
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1618
1486
  }
1619
- var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
1620
- var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
1621
- feedKey,
1622
- index
1623
- }));
1624
- var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
1625
- feedKey,
1626
- index
1627
- ]));
1628
- var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
1629
- feedKey,
1630
- index: index + 1
1631
- }));
1632
- var TimeframeClock = class {
1633
- constructor(_timeframe = new import_timeframe.Timeframe()) {
1634
- this._timeframe = _timeframe;
1635
- this.update = new import_async6.Event();
1636
- this._pendingTimeframe = _timeframe;
1487
+ var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
1488
+ var Space = class extends import_context6.Resource {
1489
+ constructor(params) {
1490
+ super();
1491
+ this.onCredentialProcessed = new import_util3.Callback();
1492
+ this.stateUpdate = new import_async6.Event();
1493
+ (0, import_invariant6.invariant)(params.spaceKey && params.feedProvider, void 0, {
1494
+ F: __dxlog_file8,
1495
+ L: 76,
1496
+ S: this,
1497
+ A: [
1498
+ "params.spaceKey && params.feedProvider",
1499
+ ""
1500
+ ]
1501
+ });
1502
+ this._id = params.id;
1503
+ this._key = params.spaceKey;
1504
+ this._genesisFeedKey = params.genesisFeed.key;
1505
+ this._feedProvider = params.feedProvider;
1506
+ this._snapshotManager = params.snapshotManager;
1507
+ this._controlPipeline = new ControlPipeline({
1508
+ spaceKey: params.spaceKey,
1509
+ genesisFeed: params.genesisFeed,
1510
+ feedProvider: params.feedProvider,
1511
+ metadataStore: params.metadataStore
1512
+ });
1513
+ this._controlPipeline.onFeedAdmitted.set(async (info) => {
1514
+ const sparse = info.assertion.designation === import_credentials.AdmittedFeed.Designation.DATA;
1515
+ if (!info.key.equals(params.genesisFeed.key)) {
1516
+ (0, import_async6.scheduleMicroTask)(this._ctx, async () => {
1517
+ await this.protocol.addFeed(await params.feedProvider(info.key, {
1518
+ sparse
1519
+ }));
1520
+ });
1521
+ }
1522
+ });
1523
+ this._controlPipeline.onCredentialProcessed.set(async (credential) => {
1524
+ await this.onCredentialProcessed.callIfSet(credential);
1525
+ (0, import_log6.log)("onCredentialProcessed", {
1526
+ credential
1527
+ }, {
1528
+ F: __dxlog_file8,
1529
+ L: 104,
1530
+ S: this,
1531
+ C: (f, a) => f(...a)
1532
+ });
1533
+ this.stateUpdate.emit();
1534
+ });
1535
+ this._controlPipeline.onDelegatedInvitation.set(async (invitation) => {
1536
+ (0, import_log6.log)("onDelegatedInvitation", {
1537
+ invitation
1538
+ }, {
1539
+ F: __dxlog_file8,
1540
+ L: 108,
1541
+ S: this,
1542
+ C: (f, a) => f(...a)
1543
+ });
1544
+ await params.onDelegatedInvitationStatusChange(invitation, true);
1545
+ });
1546
+ this._controlPipeline.onDelegatedInvitationRemoved.set(async (invitation) => {
1547
+ (0, import_log6.log)("onDelegatedInvitationRemoved", {
1548
+ invitation
1549
+ }, {
1550
+ F: __dxlog_file8,
1551
+ L: 112,
1552
+ S: this,
1553
+ C: (f, a) => f(...a)
1554
+ });
1555
+ await params.onDelegatedInvitationStatusChange(invitation, false);
1556
+ });
1557
+ this._controlPipeline.onMemberRoleChanged.set(async (changedMembers) => {
1558
+ (0, import_log6.log)("onMemberRoleChanged", () => ({
1559
+ changedMembers: changedMembers.map((m) => [
1560
+ m.key,
1561
+ m.role
1562
+ ])
1563
+ }), {
1564
+ F: __dxlog_file8,
1565
+ L: 116,
1566
+ S: this,
1567
+ C: (f, a) => f(...a)
1568
+ });
1569
+ await params.onMemberRolesChanged(changedMembers);
1570
+ });
1571
+ this.protocol = params.protocol;
1637
1572
  }
1638
- /**
1639
- * Timeframe that was processed by ECHO.
1640
- */
1641
- get timeframe() {
1642
- return this._timeframe;
1573
+ get id() {
1574
+ return this._id;
1575
+ }
1576
+ get key() {
1577
+ return this._key;
1578
+ }
1579
+ get isOpen() {
1580
+ return this._lifecycleState === import_context6.LifecycleState.OPEN;
1581
+ }
1582
+ get genesisFeedKey() {
1583
+ return this._genesisFeedKey;
1584
+ }
1585
+ get controlFeedKey() {
1586
+ return this._controlFeed?.key;
1587
+ }
1588
+ get dataFeedKey() {
1589
+ return this._dataFeed?.key;
1590
+ }
1591
+ get spaceState() {
1592
+ return this._controlPipeline.spaceState;
1643
1593
  }
1644
1594
  /**
1645
- * Timeframe that is currently being processed by ECHO.
1646
- * Will be equal to `timeframe` after the processing is complete.
1595
+ * @test-only
1647
1596
  */
1648
- get pendingTimeframe() {
1649
- return this._pendingTimeframe;
1597
+ get controlPipeline() {
1598
+ return this._controlPipeline.pipeline;
1650
1599
  }
1651
- setTimeframe(timeframe) {
1652
- this._timeframe = timeframe;
1653
- this._pendingTimeframe = timeframe;
1654
- this.update.emit(this._timeframe);
1600
+ get snapshotManager() {
1601
+ return this._snapshotManager;
1655
1602
  }
1656
- updatePendingTimeframe(key, seq) {
1657
- this._pendingTimeframe = import_timeframe.Timeframe.merge(this._pendingTimeframe, new import_timeframe.Timeframe([
1658
- [
1659
- key,
1660
- seq
1603
+ async setControlFeed(feed) {
1604
+ (0, import_invariant6.invariant)(!this._controlFeed, "Control feed already set.", {
1605
+ F: __dxlog_file8,
1606
+ L: 168,
1607
+ S: this,
1608
+ A: [
1609
+ "!this._controlFeed",
1610
+ "'Control feed already set.'"
1661
1611
  ]
1662
- ]));
1612
+ });
1613
+ this._controlFeed = feed;
1614
+ await this._controlPipeline.setWriteFeed(feed);
1615
+ return this;
1663
1616
  }
1664
- updateTimeframe() {
1665
- this._timeframe = this._pendingTimeframe;
1666
- this.update.emit(this._timeframe);
1617
+ async setDataFeed(feed) {
1618
+ (0, import_invariant6.invariant)(!this._dataFeed, "Data feed already set.", {
1619
+ F: __dxlog_file8,
1620
+ L: 175,
1621
+ S: this,
1622
+ A: [
1623
+ "!this._dataFeed",
1624
+ "'Data feed already set.'"
1625
+ ]
1626
+ });
1627
+ this._dataFeed = feed;
1628
+ return this;
1667
1629
  }
1668
- hasGaps(timeframe) {
1669
- const gaps = import_timeframe.Timeframe.dependencies(timeframe, this._timeframe);
1670
- return !gaps.isEmpty();
1630
+ /**
1631
+ * Use for diagnostics.
1632
+ */
1633
+ getControlFeeds() {
1634
+ return Array.from(this._controlPipeline.spaceState.feeds.values());
1671
1635
  }
1672
- async waitUntilReached(target) {
1673
- (0, import_log5.log)("waitUntilReached", {
1674
- target,
1675
- current: this._timeframe
1636
+ async _open(ctx) {
1637
+ (0, import_log6.log)("opening...", void 0, {
1638
+ F: __dxlog_file8,
1639
+ L: 189,
1640
+ S: this,
1641
+ C: (f, a) => f(...a)
1642
+ });
1643
+ await this._controlPipeline.start();
1644
+ await this.protocol.start();
1645
+ await this.protocol.addFeed(await this._feedProvider(this._genesisFeedKey));
1646
+ (0, import_log6.log)("opened", void 0, {
1647
+ F: __dxlog_file8,
1648
+ L: 196,
1649
+ S: this,
1650
+ C: (f, a) => f(...a)
1651
+ });
1652
+ }
1653
+ async _close() {
1654
+ (0, import_log6.log)("closing...", {
1655
+ key: this._key
1676
1656
  }, {
1677
- F: __dxlog_file7,
1678
- L: 70,
1657
+ F: __dxlog_file8,
1658
+ L: 201,
1679
1659
  S: this,
1680
1660
  C: (f, a) => f(...a)
1681
1661
  });
1682
- await this.update.waitForCondition(() => {
1683
- (0, import_log5.log)("check if reached", {
1684
- target,
1685
- current: this._timeframe,
1686
- deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
1687
- }, {
1688
- F: __dxlog_file7,
1689
- L: 72,
1690
- S: this,
1691
- C: (f, a) => f(...a)
1692
- });
1693
- return import_timeframe.Timeframe.dependencies(target, this._timeframe).isEmpty();
1662
+ await this.protocol.stop();
1663
+ await this._controlPipeline.stop();
1664
+ (0, import_log6.log)("closed", void 0, {
1665
+ F: __dxlog_file8,
1666
+ L: 207,
1667
+ S: this,
1668
+ C: (f, a) => f(...a)
1669
+ });
1670
+ }
1671
+ };
1672
+ _ts_decorate4([
1673
+ import_tracing.trace.info()
1674
+ ], Space.prototype, "protocol", void 0);
1675
+ _ts_decorate4([
1676
+ import_tracing.trace.info()
1677
+ ], Space.prototype, "_controlPipeline", void 0);
1678
+ _ts_decorate4([
1679
+ import_log6.logInfo,
1680
+ import_tracing.trace.info()
1681
+ ], Space.prototype, "id", null);
1682
+ _ts_decorate4([
1683
+ import_log6.logInfo,
1684
+ import_tracing.trace.info()
1685
+ ], Space.prototype, "key", null);
1686
+ _ts_decorate4([
1687
+ import_tracing.trace.span()
1688
+ ], Space.prototype, "_open", null);
1689
+ _ts_decorate4([
1690
+ import_async6.synchronized
1691
+ ], Space.prototype, "_close", null);
1692
+ Space = _ts_decorate4([
1693
+ (0, import_async6.trackLeaks)("open", "close"),
1694
+ import_tracing.trace.resource()
1695
+ ], Space);
1696
+ var SPACE_IDS_CACHE = new import_util3.ComplexMap(import_keys3.PublicKey.hash);
1697
+ var createIdFromSpaceKey = async (spaceKey) => {
1698
+ const cachedValue = SPACE_IDS_CACHE.get(spaceKey);
1699
+ if (cachedValue !== void 0) {
1700
+ return cachedValue;
1701
+ }
1702
+ const digest = await import_crypto3.subtleCrypto.digest("SHA-256", spaceKey.asUint8Array());
1703
+ const bytes = new Uint8Array(digest).slice(0, import_keys3.SpaceId.byteLength);
1704
+ const spaceId = import_keys3.SpaceId.encode(bytes);
1705
+ SPACE_IDS_CACHE.set(spaceKey, spaceId);
1706
+ return spaceId;
1707
+ };
1708
+ var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/admission-discovery-extension.ts";
1709
+ var CredentialRetrieverExtension = class extends import_teleport2.RpcExtension {
1710
+ constructor(_request, _onResult) {
1711
+ super({
1712
+ requested: {
1713
+ AdmissionDiscoveryService: import_protocols5.schema.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
1714
+ }
1715
+ });
1716
+ this._request = _request;
1717
+ this._onResult = _onResult;
1718
+ this._ctx = new import_context8.Context(void 0, {
1719
+ F: __dxlog_file9,
1720
+ L: 25
1721
+ });
1722
+ }
1723
+ async getHandlers() {
1724
+ return {};
1725
+ }
1726
+ async onOpen(context) {
1727
+ await super.onOpen(context);
1728
+ (0, import_async8.scheduleTask)(this._ctx, async () => {
1729
+ try {
1730
+ const result = await this.rpc.AdmissionDiscoveryService.getAdmissionCredential(this._request);
1731
+ this._onResult.wake(result.admissionCredential);
1732
+ } catch (err) {
1733
+ context.close(err);
1734
+ }
1694
1735
  });
1695
1736
  }
1737
+ async onClose() {
1738
+ await this._ctx.dispose();
1739
+ }
1740
+ async onAbort() {
1741
+ await this._ctx.dispose();
1742
+ }
1696
1743
  };
1697
- _ts_decorate3([
1698
- (0, import_debug.timed)(5e3)
1699
- ], TimeframeClock.prototype, "waitUntilReached", null);
1700
- var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
1701
- var createMessageSelector = (timeframeClock) => {
1702
- return (messages) => {
1703
- for (let i = 0; i < messages.length; i++) {
1704
- const { data: { timeframe } } = messages[i];
1705
- (0, import_invariant8.invariant)(timeframe, void 0, {
1706
- F: __dxlog_file8,
1707
- L: 25,
1708
- S: void 0,
1709
- A: [
1710
- "timeframe",
1711
- ""
1712
- ]
1713
- });
1714
- if (!timeframeClock.hasGaps(timeframe)) {
1715
- return i;
1744
+ var CredentialServerExtension = class extends import_teleport2.RpcExtension {
1745
+ constructor(_space) {
1746
+ super({
1747
+ exposed: {
1748
+ AdmissionDiscoveryService: import_protocols5.schema.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
1716
1749
  }
1717
- }
1718
- (0, import_log7.log)("Skipping...", void 0, {
1719
- F: __dxlog_file8,
1720
- L: 33,
1721
- S: void 0,
1722
- C: (f, a) => f(...a)
1723
1750
  });
1724
- };
1751
+ this._space = _space;
1752
+ }
1753
+ async getHandlers() {
1754
+ return {
1755
+ AdmissionDiscoveryService: {
1756
+ getAdmissionCredential: async (request) => {
1757
+ const memberInfo = this._space.spaceState.members.get(request.memberKey);
1758
+ if (!memberInfo?.credential) {
1759
+ throw new import_protocols5.ProtocolError("Space member not found.", request);
1760
+ }
1761
+ return {
1762
+ admissionCredential: memberInfo.credential
1763
+ };
1764
+ }
1765
+ }
1766
+ };
1767
+ }
1725
1768
  };
1726
- function _ts_decorate4(decorators, target, key, desc) {
1769
+ function _ts_decorate5(decorators, target, key, desc) {
1727
1770
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1728
1771
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1729
1772
  r = Reflect.decorate(decorators, target, key, desc);
@@ -1733,674 +1776,677 @@ function _ts_decorate4(decorators, target, key, desc) {
1733
1776
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1734
1777
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1735
1778
  }
1736
- var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
1737
- var PipelineState = class {
1738
- constructor(_feeds, _timeframeClock) {
1739
- this._feeds = _feeds;
1740
- this._timeframeClock = _timeframeClock;
1741
- this._ctx = new import_context8.Context(void 0, {
1742
- F: __dxlog_file9,
1743
- L: 41
1779
+ var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
1780
+ var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
1781
+ var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
1782
+ var SpaceProtocol = class {
1783
+ constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore, disableP2pReplication }) {
1784
+ this._feeds = /* @__PURE__ */ new Set();
1785
+ this._sessions = new import_util5.ComplexMap(import_keys5.PublicKey.hash);
1786
+ this._topology = new import_network_manager.MMSTTopology({
1787
+ originateConnections: 4,
1788
+ maxPeers: 10,
1789
+ sampleSize: 20
1744
1790
  });
1745
- this.timeframeUpdate = this._timeframeClock.update;
1746
- this.stalled = new import_async7.Event();
1747
- this._startTimeframe = new import_timeframe2.Timeframe();
1748
- this._reachedTarget = false;
1749
- }
1750
- /**
1751
- * Latest theoretical timeframe based on the last mutation in each feed.
1752
- * NOTE: This might never be reached if the mutation dependencies
1753
- */
1754
- // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
1755
- get endTimeframe() {
1756
- return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
1757
- feedKey: feed.key,
1758
- index: feed.length - 1
1759
- })));
1760
- }
1761
- get startTimeframe() {
1762
- return this._startTimeframe;
1763
- }
1764
- get timeframe() {
1765
- return this._timeframeClock.timeframe;
1766
- }
1767
- get pendingTimeframe() {
1768
- return this._timeframeClock.pendingTimeframe;
1769
- }
1770
- get targetTimeframe() {
1771
- return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
1791
+ this.feedAdded = new import_util5.CallbackCollection();
1792
+ this._spaceKey = topic;
1793
+ this._networkManager = networkManager;
1794
+ this._swarmIdentity = swarmIdentity;
1795
+ this._onSessionAuth = onSessionAuth;
1796
+ this._onAuthFailure = onAuthFailure;
1797
+ this.blobSync = new import_teleport_extension_object_sync.BlobSync({
1798
+ blobStore
1799
+ });
1800
+ this._topic = import_crypto4.subtleCrypto.digest("SHA-256", topic.asBuffer()).then(import_crypto4.discoveryKey).then(import_keys5.PublicKey.from);
1801
+ this._disableP2pReplication = disableP2pReplication ?? false;
1772
1802
  }
1773
- get reachedTarget() {
1774
- return this._reachedTarget;
1803
+ get sessions() {
1804
+ return this._sessions;
1775
1805
  }
1776
1806
  get feeds() {
1777
- return Array.from(this._feeds.values());
1778
- }
1779
- async waitUntilTimeframe(target) {
1780
- await this._timeframeClock.waitUntilReached(target);
1807
+ return this._feeds;
1781
1808
  }
1782
- setTargetTimeframe(target) {
1783
- this._targetTimeframe = target;
1809
+ get _ownPeerKey() {
1810
+ return this._swarmIdentity.peerKey;
1784
1811
  }
1785
- /**
1786
- * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
1787
- *
1788
- * This function will resolve immediately if the pipeline is stalled.
1789
- *
1790
- * @param timeout Timeout in milliseconds to specify the maximum wait time.
1791
- */
1792
- async waitUntilReachedTargetTimeframe({ ctx = new import_context8.Context(void 0, {
1793
- F: __dxlog_file9,
1794
- L: 129
1795
- }), timeout, breakOnStall = true } = {}) {
1796
- (0, import_log6.log)("waitUntilReachedTargetTimeframe", {
1797
- timeout,
1798
- current: this.timeframe,
1799
- target: this.targetTimeframe
1812
+ // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
1813
+ async addFeed(feed) {
1814
+ (0, import_log8.log)("addFeed", {
1815
+ key: feed.key
1800
1816
  }, {
1801
- F: __dxlog_file9,
1802
- L: 133,
1817
+ F: __dxlog_file10,
1818
+ L: 127,
1803
1819
  S: this,
1804
1820
  C: (f, a) => f(...a)
1805
1821
  });
1806
- this._reachedTargetPromise ??= Promise.race([
1807
- this._timeframeClock.update.waitForCondition(() => {
1808
- return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
1809
- }),
1810
- ...breakOnStall ? [
1811
- this.stalled.discardParameter().waitForCount(1)
1812
- ] : []
1813
- ]);
1814
- let done = false;
1815
- if (timeout) {
1816
- return Promise.race([
1817
- (0, import_context8.rejectOnDispose)(ctx),
1818
- (0, import_context8.rejectOnDispose)(this._ctx),
1819
- this._reachedTargetPromise.then(() => {
1820
- done = true;
1821
- this._reachedTarget = true;
1822
- }),
1823
- (0, import_async7.sleepWithContext)(this._ctx, timeout).then(() => {
1824
- if (done) {
1825
- return;
1826
- }
1827
- import_log6.log.warn("waitUntilReachedTargetTimeframe timed out", {
1828
- timeout,
1829
- current: this.timeframe,
1830
- target: this.targetTimeframe,
1831
- dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
1832
- }, {
1833
- F: __dxlog_file9,
1834
- L: 161,
1835
- S: this,
1836
- C: (f, a) => f(...a)
1837
- });
1838
- })
1839
- ]);
1840
- } else {
1841
- return this._reachedTargetPromise;
1822
+ this._feeds.add(feed);
1823
+ for (const session of this._sessions.values()) {
1824
+ session.replicator.addFeed(feed);
1825
+ }
1826
+ await this.feedAdded.callSerial(feed);
1827
+ }
1828
+ // TODO(burdon): Rename open? Common open/close interfaces for all services?
1829
+ async start() {
1830
+ if (this._connection) {
1831
+ return;
1832
+ }
1833
+ const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(""));
1834
+ await this.blobSync.open();
1835
+ (0, import_log8.log)("starting...", void 0, {
1836
+ F: __dxlog_file10,
1837
+ L: 148,
1838
+ S: this,
1839
+ C: (f, a) => f(...a)
1840
+ });
1841
+ const topic = await this._topic;
1842
+ this._connection = await this._networkManager.joinSwarm({
1843
+ protocolProvider: this._createProtocolProvider(credentials),
1844
+ peerId: this._swarmIdentity.peerKey,
1845
+ topic,
1846
+ topology: this._topology,
1847
+ label: `swarm ${topic.truncate()} for space ${this._spaceKey.truncate()}`
1848
+ });
1849
+ (0, import_log8.log)("started", void 0, {
1850
+ F: __dxlog_file10,
1851
+ L: 158,
1852
+ S: this,
1853
+ C: (f, a) => f(...a)
1854
+ });
1855
+ }
1856
+ updateTopology() {
1857
+ this._topology.forceUpdate();
1858
+ }
1859
+ async stop() {
1860
+ await this.blobSync.close();
1861
+ if (this._connection) {
1862
+ (0, import_log8.log)("stopping...", void 0, {
1863
+ F: __dxlog_file10,
1864
+ L: 169,
1865
+ S: this,
1866
+ C: (f, a) => f(...a)
1867
+ });
1868
+ await this._connection.close();
1869
+ (0, import_log8.log)("stopped", void 0, {
1870
+ F: __dxlog_file10,
1871
+ L: 171,
1872
+ S: this,
1873
+ C: (f, a) => f(...a)
1874
+ });
1842
1875
  }
1843
1876
  }
1844
- };
1845
- var Pipeline = class {
1846
- constructor() {
1847
- this._timeframeClock = new TimeframeClock(new import_timeframe2.Timeframe());
1848
- this._feeds = new import_util3.ComplexMap(import_keys4.PublicKey.hash);
1849
- this._state = new PipelineState(this._feeds, this._timeframeClock);
1850
- this._processingTrigger = new import_async7.Trigger().wake();
1851
- this._pauseTrigger = new import_async7.Trigger().wake();
1852
- this._downloads = new import_util3.ComplexMap((value) => import_keys4.PublicKey.hash(value.key));
1853
- this._isStopping = false;
1854
- this._isStarted = false;
1855
- this._isBeingConsumed = false;
1856
- this._isPaused = false;
1877
+ _createProtocolProvider(credentials) {
1878
+ return (wireParams) => {
1879
+ const session = new SpaceProtocolSession({
1880
+ wireParams,
1881
+ swarmIdentity: this._swarmIdentity,
1882
+ onSessionAuth: this._onSessionAuth,
1883
+ onAuthFailure: this._onAuthFailure,
1884
+ blobSync: this.blobSync,
1885
+ disableP2pReplication: this._disableP2pReplication
1886
+ });
1887
+ this._sessions.set(wireParams.remotePeerId, session);
1888
+ for (const feed of this._feeds) {
1889
+ session.replicator.addFeed(feed);
1890
+ }
1891
+ return session;
1892
+ };
1893
+ }
1894
+ };
1895
+ _ts_decorate5([
1896
+ import_log8.logInfo,
1897
+ import_tracing3.trace.info()
1898
+ ], SpaceProtocol.prototype, "_topic", void 0);
1899
+ _ts_decorate5([
1900
+ import_tracing3.trace.info()
1901
+ ], SpaceProtocol.prototype, "_spaceKey", void 0);
1902
+ _ts_decorate5([
1903
+ import_log8.logInfo
1904
+ ], SpaceProtocol.prototype, "_ownPeerKey", null);
1905
+ SpaceProtocol = _ts_decorate5([
1906
+ import_tracing3.trace.resource()
1907
+ ], SpaceProtocol);
1908
+ var AuthStatus;
1909
+ (function(AuthStatus2) {
1910
+ AuthStatus2["INITIAL"] = "INITIAL";
1911
+ AuthStatus2["SUCCESS"] = "SUCCESS";
1912
+ AuthStatus2["FAILURE"] = "FAILURE";
1913
+ })(AuthStatus || (AuthStatus = {}));
1914
+ var SpaceProtocolSession = class {
1915
+ // TODO(dmaretskyi): Allow to pass in extra extensions.
1916
+ constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync, disableP2pReplication }) {
1917
+ this.replicator = new import_teleport_extension_replicator.ReplicatorExtension().setOptions({
1918
+ upload: true
1919
+ });
1920
+ this._authStatus = "INITIAL";
1921
+ this._wireParams = wireParams;
1922
+ this._swarmIdentity = swarmIdentity;
1923
+ this._onSessionAuth = onSessionAuth;
1924
+ this._onAuthFailure = onAuthFailure;
1925
+ this._blobSync = blobSync;
1926
+ this._teleport = new import_teleport3.Teleport(wireParams);
1927
+ this._disableP2pReplication = disableP2pReplication ?? false;
1928
+ }
1929
+ get authStatus() {
1930
+ return this._authStatus;
1857
1931
  }
1858
- get state() {
1859
- return this._state;
1932
+ get stats() {
1933
+ return this._teleport.stats;
1860
1934
  }
1861
- get writer() {
1862
- (0, import_invariant7.invariant)(this._writer, "Writer not set.", {
1863
- F: __dxlog_file9,
1864
- L: 243,
1935
+ get stream() {
1936
+ return this._teleport.stream;
1937
+ }
1938
+ async open(sessionId) {
1939
+ await this._teleport.open(sessionId);
1940
+ this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
1941
+ provider: this._swarmIdentity.credentialProvider,
1942
+ verifier: this._swarmIdentity.credentialAuthenticator,
1943
+ onAuthSuccess: () => {
1944
+ (0, import_log8.log)("Peer authenticated", void 0, {
1945
+ F: __dxlog_file10,
1946
+ L: 282,
1947
+ S: this,
1948
+ C: (f, a) => f(...a)
1949
+ });
1950
+ this._authStatus = "SUCCESS";
1951
+ this._onSessionAuth?.(this._teleport);
1952
+ },
1953
+ onAuthFailure: () => {
1954
+ this._authStatus = "FAILURE";
1955
+ this._onAuthFailure?.(this._teleport);
1956
+ }
1957
+ }));
1958
+ if (!this._disableP2pReplication) {
1959
+ this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
1960
+ }
1961
+ this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
1962
+ }
1963
+ async close() {
1964
+ (0, import_log8.log)("close", void 0, {
1965
+ F: __dxlog_file10,
1966
+ L: 301,
1865
1967
  S: this,
1866
- A: [
1867
- "this._writer",
1868
- "'Writer not set.'"
1869
- ]
1968
+ C: (f, a) => f(...a)
1870
1969
  });
1871
- return this._writer;
1970
+ await this._teleport.close();
1872
1971
  }
1873
- hasFeed(feedKey) {
1874
- return this._feeds.has(feedKey);
1972
+ async abort() {
1973
+ await this._teleport.abort();
1875
1974
  }
1876
- getFeeds() {
1877
- return this._feedSetIterator.feeds;
1975
+ };
1976
+ _ts_decorate5([
1977
+ import_log8.logInfo
1978
+ ], SpaceProtocolSession.prototype, "_wireParams", void 0);
1979
+ _ts_decorate5([
1980
+ import_log8.logInfo
1981
+ ], SpaceProtocolSession.prototype, "authStatus", null);
1982
+ var isCollectionQueryMessage = (message) => message.type === import_protocols8.MESSAGE_TYPE_COLLECTION_QUERY;
1983
+ var isCollectionStateMessage = (message) => message.type === import_protocols8.MESSAGE_TYPE_COLLECTION_STATE;
1984
+ function _ts_decorate6(decorators, target, key, desc) {
1985
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1986
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1987
+ r = Reflect.decorate(decorators, target, key, desc);
1988
+ else
1989
+ for (var i = decorators.length - 1; i >= 0; i--)
1990
+ if (d = decorators[i])
1991
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1992
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1993
+ }
1994
+ var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
1995
+ var EchoNetworkAdapter = class extends import_automerge_repo2.NetworkAdapter {
1996
+ constructor(_params) {
1997
+ super();
1998
+ this._params = _params;
1999
+ this._replicators = /* @__PURE__ */ new Set();
2000
+ this._connections = /* @__PURE__ */ new Map();
2001
+ this._lifecycleState = import_context10.LifecycleState.CLOSED;
2002
+ this._connected = new import_async11.Trigger();
1878
2003
  }
1879
- // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
1880
- // which might be opening feeds during the mutation processing, which w
1881
- async addFeed(feed) {
1882
- this._feeds.set(feed.key, feed);
1883
- if (this._feedSetIterator) {
1884
- await this._feedSetIterator.addFeed(feed);
1885
- }
1886
- if (this._isStarted && !this._isPaused) {
1887
- this._setFeedDownloadState(feed);
1888
- }
2004
+ connect(peerId, peerMetadata) {
2005
+ this.peerId = peerId;
2006
+ this.peerMetadata = peerMetadata;
2007
+ this._connected.wake();
1889
2008
  }
1890
- setWriteFeed(feed) {
1891
- (0, import_invariant7.invariant)(!this._writer, "Writer already set.", {
1892
- F: __dxlog_file9,
1893
- L: 270,
1894
- S: this,
1895
- A: [
1896
- "!this._writer",
1897
- "'Writer already set.'"
1898
- ]
1899
- });
1900
- (0, import_invariant7.invariant)(feed.properties.writable, "Feed must be writable.", {
1901
- F: __dxlog_file9,
1902
- L: 271,
1903
- S: this,
1904
- A: [
1905
- "feed.properties.writable",
1906
- "'Feed must be writable.'"
1907
- ]
1908
- });
1909
- this._writer = createMappedFeedWriter((payload) => ({
1910
- timeframe: this._timeframeClock.timeframe,
1911
- payload
1912
- }), feed.createFeedWriter());
2009
+ send(message) {
2010
+ this._send(message);
1913
2011
  }
1914
- async start() {
1915
- (0, import_invariant7.invariant)(!this._isStarted, "Pipeline is already started.", {
1916
- F: __dxlog_file9,
1917
- L: 284,
1918
- S: this,
1919
- A: [
1920
- "!this._isStarted",
1921
- "'Pipeline is already started.'"
1922
- ]
1923
- });
1924
- (0, import_log6.log)("starting...", void 0, {
1925
- F: __dxlog_file9,
1926
- L: 285,
2012
+ disconnect() {
2013
+ }
2014
+ async open() {
2015
+ if (this._lifecycleState === import_context10.LifecycleState.OPEN) {
2016
+ return;
2017
+ }
2018
+ this._lifecycleState = import_context10.LifecycleState.OPEN;
2019
+ (0, import_log12.log)("emit ready", void 0, {
2020
+ F: __dxlog_file11,
2021
+ L: 79,
1927
2022
  S: this,
1928
2023
  C: (f, a) => f(...a)
1929
2024
  });
1930
- await this._initIterator();
1931
- await this._feedSetIterator.open();
1932
- this._isStarted = true;
1933
- (0, import_log6.log)("started", void 0, {
1934
- F: __dxlog_file9,
1935
- L: 289,
1936
- S: this,
1937
- C: (f, a) => f(...a)
2025
+ this.emit("ready", {
2026
+ network: this
1938
2027
  });
1939
- if (!this._isPaused) {
1940
- for (const feed of this._feeds.values()) {
1941
- this._setFeedDownloadState(feed);
1942
- }
1943
- }
1944
2028
  }
1945
- async stop() {
1946
- (0, import_log6.log)("stopping...", void 0, {
1947
- F: __dxlog_file9,
1948
- L: 300,
1949
- S: this,
1950
- C: (f, a) => f(...a)
1951
- });
1952
- this._isStopping = true;
1953
- for (const [feed, handle] of this._downloads.entries()) {
1954
- feed.undownload(handle);
2029
+ async close() {
2030
+ if (this._lifecycleState === import_context10.LifecycleState.CLOSED) {
2031
+ return this;
1955
2032
  }
1956
- this._downloads.clear();
1957
- await this._feedSetIterator?.close();
1958
- await this._processingTrigger.wait();
1959
- await this._state._ctx.dispose();
1960
- this._state._ctx = new import_context8.Context(void 0, {
1961
- F: __dxlog_file9,
1962
- L: 309
2033
+ for (const replicator of this._replicators) {
2034
+ await replicator.disconnect();
2035
+ }
2036
+ this._replicators.clear();
2037
+ this._lifecycleState = import_context10.LifecycleState.CLOSED;
2038
+ }
2039
+ async whenConnected() {
2040
+ await this._connected.wait({
2041
+ timeout: 1e4
1963
2042
  });
1964
- this._state._reachedTargetPromise = void 0;
1965
- this._state._reachedTarget = false;
1966
- this._isStarted = false;
1967
- (0, import_log6.log)("stopped", void 0, {
1968
- F: __dxlog_file9,
1969
- L: 313,
2043
+ }
2044
+ async addReplicator(replicator) {
2045
+ (0, import_invariant9.invariant)(this._lifecycleState === import_context10.LifecycleState.OPEN, void 0, {
2046
+ F: __dxlog_file11,
2047
+ L: 105,
1970
2048
  S: this,
1971
- C: (f, a) => f(...a)
2049
+ A: [
2050
+ "this._lifecycleState === LifecycleState.OPEN",
2051
+ ""
2052
+ ]
1972
2053
  });
1973
- }
1974
- /**
1975
- * @param timeframe Timeframe of already processed messages.
1976
- * The pipeline will start processing messages AFTER this timeframe.
1977
- */
1978
- async setCursor(timeframe) {
1979
- (0, import_invariant7.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
1980
- F: __dxlog_file9,
1981
- L: 322,
2054
+ (0, import_invariant9.invariant)(this.peerId, void 0, {
2055
+ F: __dxlog_file11,
2056
+ L: 106,
1982
2057
  S: this,
1983
2058
  A: [
1984
- "!this._isStarted || this._isPaused",
1985
- "'Invalid state.'"
2059
+ "this.peerId",
2060
+ ""
1986
2061
  ]
1987
2062
  });
1988
- this._state._startTimeframe = timeframe;
1989
- this._timeframeClock.setTimeframe(timeframe);
1990
- if (this._feedSetIterator) {
1991
- await this._feedSetIterator.close();
1992
- await this._initIterator();
1993
- await this._feedSetIterator.open();
1994
- }
1995
- }
1996
- /**
1997
- * Calling pause while processing will cause a deadlock.
1998
- */
1999
- async pause() {
2000
- if (this._isPaused) {
2001
- return;
2002
- }
2003
- this._pauseTrigger.reset();
2004
- await this._processingTrigger.wait();
2005
- this._isPaused = true;
2006
- }
2007
- async unpause() {
2008
- (0, import_invariant7.invariant)(this._isPaused, "Pipeline is not paused.", {
2009
- F: __dxlog_file9,
2010
- L: 351,
2063
+ (0, import_invariant9.invariant)(!this._replicators.has(replicator), void 0, {
2064
+ F: __dxlog_file11,
2065
+ L: 107,
2011
2066
  S: this,
2012
2067
  A: [
2013
- "this._isPaused",
2014
- "'Pipeline is not paused.'"
2068
+ "!this._replicators.has(replicator)",
2069
+ ""
2015
2070
  ]
2016
2071
  });
2017
- this._pauseTrigger.wake();
2018
- this._isPaused = false;
2019
- for (const feed of this._feeds.values()) {
2020
- this._setFeedDownloadState(feed);
2021
- }
2072
+ this._replicators.add(replicator);
2073
+ await replicator.connect({
2074
+ peerId: this.peerId,
2075
+ onConnectionOpen: this._onConnectionOpen.bind(this),
2076
+ onConnectionClosed: this._onConnectionClosed.bind(this),
2077
+ onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
2078
+ getContainingSpaceForDocument: this._params.getContainingSpaceForDocument,
2079
+ getContainingSpaceIdForDocument: async (documentId) => {
2080
+ const key = await this._params.getContainingSpaceForDocument(documentId);
2081
+ return key ? createIdFromSpaceKey(key) : null;
2082
+ }
2083
+ });
2022
2084
  }
2023
- /**
2024
- * Starts to iterate over the ordered messages from the added feeds.
2025
- * Updates the timeframe clock after the message has bee processed.
2026
- */
2027
- async *consume() {
2028
- (0, import_invariant7.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
2029
- F: __dxlog_file9,
2030
- L: 366,
2085
+ async removeReplicator(replicator) {
2086
+ (0, import_invariant9.invariant)(this._lifecycleState === import_context10.LifecycleState.OPEN, void 0, {
2087
+ F: __dxlog_file11,
2088
+ L: 125,
2031
2089
  S: this,
2032
2090
  A: [
2033
- "!this._isBeingConsumed",
2034
- "'Pipeline is already being consumed.'"
2091
+ "this._lifecycleState === LifecycleState.OPEN",
2092
+ ""
2035
2093
  ]
2036
2094
  });
2037
- this._isBeingConsumed = true;
2038
- (0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
2039
- F: __dxlog_file9,
2040
- L: 369,
2095
+ (0, import_invariant9.invariant)(this._replicators.has(replicator), void 0, {
2096
+ F: __dxlog_file11,
2097
+ L: 126,
2041
2098
  S: this,
2042
2099
  A: [
2043
- "this._feedSetIterator",
2044
- "'Iterator not initialized.'"
2100
+ "this._replicators.has(replicator)",
2101
+ ""
2045
2102
  ]
2046
2103
  });
2047
- let lastFeedSetIterator = this._feedSetIterator;
2048
- let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
2049
- while (!this._isStopping) {
2050
- await this._pauseTrigger.wait();
2051
- if (lastFeedSetIterator !== this._feedSetIterator) {
2052
- (0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
2053
- F: __dxlog_file9,
2054
- L: 378,
2055
- S: this,
2056
- A: [
2057
- "this._feedSetIterator",
2058
- "'Iterator not initialized.'"
2059
- ]
2060
- });
2061
- lastFeedSetIterator = this._feedSetIterator;
2062
- iterable = lastFeedSetIterator[Symbol.asyncIterator]();
2063
- }
2064
- const { done, value } = await iterable.next();
2065
- if (!done) {
2066
- const block = value ?? (0, import_debug2.failUndefined)();
2067
- this._processingTrigger.reset();
2068
- this._timeframeClock.updatePendingTimeframe(import_keys4.PublicKey.from(block.feedKey), block.seq);
2069
- yield block;
2070
- this._processingTrigger.wake();
2071
- this._timeframeClock.updateTimeframe();
2072
- }
2104
+ await replicator.disconnect();
2105
+ this._replicators.delete(replicator);
2106
+ }
2107
+ async shouldAdvertise(peerId, params) {
2108
+ const connection = this._connections.get(peerId);
2109
+ if (!connection) {
2110
+ return false;
2073
2111
  }
2074
- this._isBeingConsumed = false;
2112
+ return connection.connection.shouldAdvertise(params);
2075
2113
  }
2076
- _setFeedDownloadState(feed) {
2077
- let handle = this._downloads.get(feed);
2078
- if (handle) {
2079
- feed.undownload(handle);
2114
+ shouldSyncCollection(peerId, params) {
2115
+ const connection = this._connections.get(peerId);
2116
+ if (!connection) {
2117
+ return false;
2080
2118
  }
2081
- const timeframe = this._state._startTimeframe;
2082
- const seq = timeframe.get(feed.key) ?? -1;
2083
- (0, import_log6.log)("download", {
2084
- feed: feed.key.truncate(),
2085
- seq,
2086
- length: feed.length
2087
- }, {
2088
- F: __dxlog_file9,
2089
- L: 407,
2090
- S: this,
2091
- C: (f, a) => f(...a)
2092
- });
2093
- handle = feed.download({
2094
- start: seq + 1,
2095
- linear: true
2096
- }, (err, data) => {
2097
- if (err) {
2098
- } else {
2099
- import_log6.log.info("downloaded", {
2100
- data
2101
- }, {
2102
- F: __dxlog_file9,
2103
- L: 412,
2119
+ return connection.connection.shouldSyncCollection(params);
2120
+ }
2121
+ queryCollectionState(collectionId, targetId) {
2122
+ const message = {
2123
+ type: "collection-query",
2124
+ senderId: this.peerId,
2125
+ targetId,
2126
+ collectionId
2127
+ };
2128
+ this._send(message);
2129
+ }
2130
+ sendCollectionState(collectionId, targetId, state) {
2131
+ const message = {
2132
+ type: "collection-state",
2133
+ senderId: this.peerId,
2134
+ targetId,
2135
+ collectionId,
2136
+ state
2137
+ };
2138
+ this._send(message);
2139
+ }
2140
+ _send(message) {
2141
+ const connectionEntry = this._connections.get(message.targetId);
2142
+ if (!connectionEntry) {
2143
+ throw new Error("Connection not found.");
2144
+ }
2145
+ const writeStart = Date.now();
2146
+ connectionEntry.writer.write(message).then(() => {
2147
+ const durationMs = Date.now() - writeStart;
2148
+ this._params.monitor?.recordMessageSent(message, durationMs);
2149
+ }).catch((err) => {
2150
+ if (connectionEntry.isOpen) {
2151
+ import_log12.log.catch(err, void 0, {
2152
+ F: __dxlog_file11,
2153
+ L: 186,
2104
2154
  S: this,
2105
2155
  C: (f, a) => f(...a)
2106
2156
  });
2107
2157
  }
2158
+ this._params.monitor?.recordMessageSendingFailed(message);
2108
2159
  });
2109
- this._downloads.set(feed, handle);
2110
2160
  }
2111
- async _initIterator() {
2112
- this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
2113
- start: startAfter(this._timeframeClock.timeframe),
2114
- stallTimeout: 1e3
2115
- });
2116
- this._feedSetIterator.stalled.on((iterator) => {
2117
- import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
2118
- F: __dxlog_file9,
2119
- L: 426,
2120
- S: this,
2121
- C: (f, a) => f(...a)
2122
- });
2123
- this._state.stalled.emit();
2124
- });
2125
- for (const feed of this._feeds.values()) {
2126
- await this._feedSetIterator.addFeed(feed);
2127
- }
2161
+ // TODO(dmaretskyi): Remove.
2162
+ getPeersInterestedInCollection(collectionId) {
2163
+ return Array.from(this._connections.values()).map((connection) => {
2164
+ return connection.connection.shouldSyncCollection({
2165
+ collectionId
2166
+ }) ? connection.connection.peerId : null;
2167
+ }).filter(import_util7.nonNullable);
2128
2168
  }
2129
- };
2130
- _ts_decorate4([
2131
- import_async7.synchronized
2132
- ], Pipeline.prototype, "start", null);
2133
- _ts_decorate4([
2134
- import_async7.synchronized
2135
- ], Pipeline.prototype, "stop", null);
2136
- _ts_decorate4([
2137
- import_async7.synchronized
2138
- ], Pipeline.prototype, "setCursor", null);
2139
- _ts_decorate4([
2140
- import_async7.synchronized
2141
- ], Pipeline.prototype, "pause", null);
2142
- _ts_decorate4([
2143
- import_async7.synchronized
2144
- ], Pipeline.prototype, "unpause", null);
2145
- function _ts_decorate5(decorators, target, key, desc) {
2146
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2147
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2148
- r = Reflect.decorate(decorators, target, key, desc);
2149
- else
2150
- for (var i = decorators.length - 1; i >= 0; i--)
2151
- if (d = decorators[i])
2152
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2153
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2154
- }
2155
- var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
2156
- var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
2157
- var CONTROL_PIPELINE_SNAPSHOT_DELAY = 1e4;
2158
- var USE_SNAPSHOTS = true;
2159
- var ControlPipeline = class {
2160
- constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
2161
- this._ctx = new import_context10.Context(void 0, {
2162
- F: __dxlog_file10,
2163
- L: 47
2169
+ _onConnectionOpen(connection) {
2170
+ (0, import_log12.log)("Connection opened", {
2171
+ peerId: connection.peerId
2172
+ }, {
2173
+ F: __dxlog_file11,
2174
+ L: 204,
2175
+ S: this,
2176
+ C: (f, a) => f(...a)
2164
2177
  });
2165
- this._lastTimeframeSaveTime = Date.now();
2166
- this.onFeedAdmitted = new import_util5.Callback();
2167
- this._usage = new import_tracing3.TimeUsageCounter();
2168
- this._mutations = new import_tracing3.TimeSeriesCounter();
2169
- this._snapshotTask = new import_async9.DeferredTask(this._ctx, async () => {
2170
- await (0, import_async9.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
2171
- await this._saveSnapshot();
2178
+ (0, import_invariant9.invariant)(!this._connections.has(connection.peerId), void 0, {
2179
+ F: __dxlog_file11,
2180
+ L: 205,
2181
+ S: this,
2182
+ A: [
2183
+ "!this._connections.has(connection.peerId as PeerId)",
2184
+ ""
2185
+ ]
2172
2186
  });
2173
- this._spaceKey = spaceKey;
2174
- this._metadata = metadataStore;
2175
- this._pipeline = new Pipeline();
2176
- void this._pipeline.addFeed(genesisFeed);
2177
- this._spaceStateMachine = new import_credentials2.SpaceStateMachine(spaceKey);
2178
- this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
2179
- (0, import_log9.log)("feed admitted", {
2180
- key: info.key
2181
- }, {
2182
- F: __dxlog_file10,
2183
- L: 82,
2184
- S: this,
2185
- C: (f, a) => f(...a)
2186
- });
2187
- if (info.assertion.designation === import_credentials3.AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
2188
- queueMicrotask(async () => {
2189
- try {
2190
- const feed = await feedProvider(info.key);
2191
- if (!this._pipeline.hasFeed(feed.key)) {
2192
- await this._pipeline.addFeed(feed);
2193
- }
2194
- } catch (err) {
2195
- import_log9.log.catch(err, void 0, {
2196
- F: __dxlog_file10,
2197
- L: 93,
2198
- S: this,
2199
- C: (f, a) => f(...a)
2200
- });
2187
+ const reader = connection.readable.getReader();
2188
+ const writer = connection.writable.getWriter();
2189
+ const connectionEntry = {
2190
+ connection,
2191
+ reader,
2192
+ writer,
2193
+ isOpen: true
2194
+ };
2195
+ this._connections.set(connection.peerId, connectionEntry);
2196
+ queueMicrotask(async () => {
2197
+ try {
2198
+ while (true) {
2199
+ const { done, value } = await reader.read();
2200
+ if (done) {
2201
+ break;
2201
2202
  }
2202
- });
2203
+ this._onMessage(value);
2204
+ }
2205
+ } catch (err) {
2206
+ if (connectionEntry.isOpen) {
2207
+ import_log12.log.catch(err, void 0, {
2208
+ F: __dxlog_file11,
2209
+ L: 224,
2210
+ S: this,
2211
+ C: (f, a) => f(...a)
2212
+ });
2213
+ }
2203
2214
  }
2204
- await this.onFeedAdmitted.callIfSet(info);
2205
2215
  });
2206
- this.onMemberRoleChanged = this._spaceStateMachine.onMemberRoleChanged;
2207
- this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
2208
- this.onDelegatedInvitation = this._spaceStateMachine.onDelegatedInvitation;
2209
- this.onDelegatedInvitationRemoved = this._spaceStateMachine.onDelegatedInvitationRemoved;
2210
- }
2211
- get spaceState() {
2212
- return this._spaceStateMachine;
2213
- }
2214
- get pipeline() {
2215
- return this._pipeline;
2216
+ (0, import_log12.log)("emit peer-candidate", {
2217
+ peerId: connection.peerId
2218
+ }, {
2219
+ F: __dxlog_file11,
2220
+ L: 229,
2221
+ S: this,
2222
+ C: (f, a) => f(...a)
2223
+ });
2224
+ this._emitPeerCandidate(connection);
2225
+ this._params.monitor?.recordPeerConnected(connection.peerId);
2216
2226
  }
2217
- async setWriteFeed(feed) {
2218
- await this._pipeline.addFeed(feed);
2219
- this._pipeline.setWriteFeed(feed);
2227
+ _onMessage(message) {
2228
+ if (isCollectionQueryMessage(message)) {
2229
+ this._params.onCollectionStateQueried(message.collectionId, message.senderId);
2230
+ } else if (isCollectionStateMessage(message)) {
2231
+ this._params.onCollectionStateReceived(message.collectionId, message.senderId, message.state);
2232
+ } else {
2233
+ this.emit("message", message);
2234
+ }
2235
+ this._params.monitor?.recordMessageReceived(message);
2220
2236
  }
2221
- async start() {
2222
- const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
2223
- (0, import_log9.log)("load snapshot", {
2224
- key: this._spaceKey,
2225
- present: !!snapshot,
2226
- tf: snapshot?.timeframe
2237
+ /**
2238
+ * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
2239
+ * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
2240
+ */
2241
+ _onConnectionAuthScopeChanged(connection) {
2242
+ (0, import_log12.log)("Connection auth scope changed", {
2243
+ peerId: connection.peerId
2227
2244
  }, {
2228
- F: __dxlog_file10,
2229
- L: 123,
2245
+ F: __dxlog_file11,
2246
+ L: 250,
2230
2247
  S: this,
2231
2248
  C: (f, a) => f(...a)
2232
2249
  });
2233
- if (USE_SNAPSHOTS && snapshot) {
2234
- await this._processSnapshot(snapshot);
2235
- }
2236
- (0, import_log9.log)("starting...", void 0, {
2237
- F: __dxlog_file10,
2238
- L: 128,
2250
+ const entry = this._connections.get(connection.peerId);
2251
+ (0, import_invariant9.invariant)(entry, void 0, {
2252
+ F: __dxlog_file11,
2253
+ L: 252,
2239
2254
  S: this,
2240
- C: (f, a) => f(...a)
2255
+ A: [
2256
+ "entry",
2257
+ ""
2258
+ ]
2241
2259
  });
2242
- setTimeout(async () => {
2243
- void this._consumePipeline(new import_context10.Context(void 0, {
2244
- F: __dxlog_file10,
2245
- L: 130
2246
- }));
2260
+ this.emit("peer-disconnected", {
2261
+ peerId: connection.peerId
2247
2262
  });
2248
- await this._pipeline.start();
2249
- (0, import_log9.log)("started", void 0, {
2250
- F: __dxlog_file10,
2251
- L: 134,
2263
+ this._emitPeerCandidate(connection);
2264
+ }
2265
+ _onConnectionClosed(connection) {
2266
+ (0, import_log12.log)("Connection closed", {
2267
+ peerId: connection.peerId
2268
+ }, {
2269
+ F: __dxlog_file11,
2270
+ L: 258,
2252
2271
  S: this,
2253
2272
  C: (f, a) => f(...a)
2254
2273
  });
2255
- }
2256
- async _processSnapshot(snapshot) {
2257
- await this._pipeline.setCursor(snapshot.timeframe);
2258
- for (const message of snapshot.messages ?? []) {
2259
- const result = await this._spaceStateMachine.process(message.credential, {
2260
- sourceFeed: message.feedKey,
2261
- skipVerification: true
2262
- });
2263
- if (!result) {
2264
- import_log9.log.warn("credential processing failed from snapshot", {
2265
- message
2266
- }, {
2267
- F: __dxlog_file10,
2268
- L: 147,
2269
- S: this,
2270
- C: (f, a) => f(...a)
2271
- });
2272
- }
2273
- }
2274
- }
2275
- async _saveSnapshot() {
2276
- await this._pipeline.pause();
2277
- const snapshot = {
2278
- timeframe: this._pipeline.state.timeframe,
2279
- messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
2280
- feedKey: entry.sourceFeed,
2281
- credential: entry.credential
2282
- }))
2283
- };
2284
- await this._pipeline.unpause();
2285
- (0, import_log9.log)("save snapshot", {
2286
- key: this._spaceKey,
2287
- snapshot
2288
- }, {
2289
- F: __dxlog_file10,
2290
- L: 163,
2274
+ const entry = this._connections.get(connection.peerId);
2275
+ (0, import_invariant9.invariant)(entry, void 0, {
2276
+ F: __dxlog_file11,
2277
+ L: 260,
2278
+ S: this,
2279
+ A: [
2280
+ "entry",
2281
+ ""
2282
+ ]
2283
+ });
2284
+ entry.isOpen = false;
2285
+ this.emit("peer-disconnected", {
2286
+ peerId: connection.peerId
2287
+ });
2288
+ this._params.monitor?.recordPeerDisconnected(connection.peerId);
2289
+ void entry.reader.cancel().catch((err) => import_log12.log.catch(err, void 0, {
2290
+ F: __dxlog_file11,
2291
+ L: 266,
2292
+ S: this,
2293
+ C: (f, a) => f(...a)
2294
+ }));
2295
+ void entry.writer.abort().catch((err) => import_log12.log.catch(err, void 0, {
2296
+ F: __dxlog_file11,
2297
+ L: 267,
2291
2298
  S: this,
2292
2299
  C: (f, a) => f(...a)
2300
+ }));
2301
+ this._connections.delete(connection.peerId);
2302
+ }
2303
+ _emitPeerCandidate(connection) {
2304
+ this.emit("peer-candidate", {
2305
+ peerId: connection.peerId,
2306
+ peerMetadata: createEchoPeerMetadata()
2293
2307
  });
2294
- await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
2295
2308
  }
2296
- async _consumePipeline(ctx) {
2297
- for await (const msg of this._pipeline.consume()) {
2298
- const span = this._usage.beginRecording();
2299
- this._mutations.inc();
2300
- try {
2301
- await this._processMessage(ctx, msg);
2302
- } catch (err) {
2303
- import_log9.log.catch(err, void 0, {
2304
- F: __dxlog_file10,
2305
- L: 176,
2306
- S: this,
2307
- C: (f, a) => f(...a)
2308
- });
2309
- }
2310
- span.end();
2311
- }
2309
+ };
2310
+ _ts_decorate6([
2311
+ import_async11.synchronized
2312
+ ], EchoNetworkAdapter.prototype, "open", null);
2313
+ _ts_decorate6([
2314
+ import_async11.synchronized
2315
+ ], EchoNetworkAdapter.prototype, "close", null);
2316
+ _ts_decorate6([
2317
+ import_async11.synchronized
2318
+ ], EchoNetworkAdapter.prototype, "addReplicator", null);
2319
+ _ts_decorate6([
2320
+ import_async11.synchronized
2321
+ ], EchoNetworkAdapter.prototype, "removeReplicator", null);
2322
+ var createEchoPeerMetadata = () => ({
2323
+ // TODO(dmaretskyi): Refactor this.
2324
+ dxos_peerSource: "EchoNetworkAdapter"
2325
+ });
2326
+ var isEchoPeerMetadata = (metadata) => metadata?.dxos_peerSource === "EchoNetworkAdapter";
2327
+ var HeadsStore = class {
2328
+ constructor({ db }) {
2329
+ this._db = db;
2312
2330
  }
2313
- async _processMessage(ctx, msg) {
2314
- (0, import_log9.log)("processing", {
2315
- key: msg.feedKey,
2316
- seq: msg.seq
2317
- }, {
2318
- F: __dxlog_file10,
2319
- L: 186,
2320
- S: this,
2321
- C: (f, a) => f(...a)
2331
+ setHeads(documentId, heads, batch) {
2332
+ batch.put(documentId, heads, {
2333
+ sublevel: this._db,
2334
+ keyEncoding: "utf8",
2335
+ valueEncoding: import_indexing.headsEncoding
2322
2336
  });
2323
- if (msg.data.payload.credential) {
2324
- const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
2325
- const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
2326
- sourceFeed: import_keys6.PublicKey.from(msg.feedKey)
2337
+ }
2338
+ // TODO(dmaretskyi): Make batched.
2339
+ async getHeads(documentIds) {
2340
+ return this._db.getMany(documentIds, {
2341
+ keyEncoding: "utf8",
2342
+ valueEncoding: import_indexing.headsEncoding
2343
+ });
2344
+ }
2345
+ };
2346
+ var LevelDBStorageAdapter = class extends import_context11.Resource {
2347
+ constructor(_params) {
2348
+ super();
2349
+ this._params = _params;
2350
+ }
2351
+ async load(keyArray) {
2352
+ try {
2353
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2354
+ return void 0;
2355
+ }
2356
+ const startMs = Date.now();
2357
+ const chunk = await this._params.db.get(keyArray, {
2358
+ ...encodingOptions
2327
2359
  });
2328
- timer.end();
2329
- if (!result) {
2330
- import_log9.log.warn("processing failed", {
2331
- msg
2332
- }, {
2333
- F: __dxlog_file10,
2334
- L: 195,
2335
- S: this,
2336
- C: (f, a) => f(...a)
2337
- });
2338
- } else {
2339
- await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
2360
+ this._params.monitor?.recordBytesLoaded(chunk.byteLength);
2361
+ this._params.monitor?.recordLoadDuration(Date.now() - startMs);
2362
+ return chunk;
2363
+ } catch (err) {
2364
+ if (isLevelDbNotFoundError(err)) {
2365
+ return void 0;
2340
2366
  }
2341
- this._snapshotTask.schedule();
2367
+ throw err;
2342
2368
  }
2343
2369
  }
2344
- async _noteTargetStateIfNeeded(timeframe) {
2345
- if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
2346
- this._lastTimeframeSaveTime = Date.now();
2347
- await this._saveTargetTimeframe(timeframe);
2370
+ async save(keyArray, binary) {
2371
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2372
+ return void 0;
2348
2373
  }
2349
- }
2350
- async stop() {
2351
- (0, import_log9.log)("stopping...", void 0, {
2352
- F: __dxlog_file10,
2353
- L: 215,
2354
- S: this,
2355
- C: (f, a) => f(...a)
2374
+ const startMs = Date.now();
2375
+ const batch = this._params.db.batch();
2376
+ await this._params.callbacks?.beforeSave?.({
2377
+ path: keyArray,
2378
+ batch
2356
2379
  });
2357
- await this._ctx.dispose();
2358
- await this._pipeline.stop();
2359
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
2360
- (0, import_log9.log)("stopped", void 0, {
2361
- F: __dxlog_file10,
2362
- L: 219,
2363
- S: this,
2364
- C: (f, a) => f(...a)
2380
+ batch.put(keyArray, Buffer.from(binary), {
2381
+ ...encodingOptions
2365
2382
  });
2383
+ await batch.write();
2384
+ this._params.monitor?.recordBytesStored(binary.byteLength);
2385
+ await this._params.callbacks?.afterSave?.(keyArray);
2386
+ this._params.monitor?.recordStoreDuration(Date.now() - startMs);
2366
2387
  }
2367
- async _saveTargetTimeframe(timeframe) {
2368
- try {
2369
- const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
2370
- await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
2371
- this._targetTimeframe = newTimeframe;
2372
- } catch (err) {
2373
- (0, import_log9.log)(err, void 0, {
2374
- F: __dxlog_file10,
2375
- L: 228,
2376
- S: this,
2377
- C: (f, a) => f(...a)
2388
+ async remove(keyArray) {
2389
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2390
+ return void 0;
2391
+ }
2392
+ await this._params.db.del(keyArray, {
2393
+ ...encodingOptions
2394
+ });
2395
+ }
2396
+ async loadRange(keyPrefix) {
2397
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2398
+ return [];
2399
+ }
2400
+ const startMs = Date.now();
2401
+ const result = [];
2402
+ for await (const [key, value] of this._params.db.iterator({
2403
+ gte: keyPrefix,
2404
+ lte: [
2405
+ ...keyPrefix,
2406
+ "\uFFFF"
2407
+ ],
2408
+ ...encodingOptions
2409
+ })) {
2410
+ result.push({
2411
+ key,
2412
+ data: value
2413
+ });
2414
+ this._params.monitor?.recordBytesLoaded(value.byteLength);
2415
+ }
2416
+ this._params.monitor?.recordLoadDuration(Date.now() - startMs);
2417
+ return result;
2418
+ }
2419
+ async removeRange(keyPrefix) {
2420
+ if (this._lifecycleState !== import_context11.LifecycleState.OPEN) {
2421
+ return void 0;
2422
+ }
2423
+ const batch = this._params.db.batch();
2424
+ for await (const [key] of this._params.db.iterator({
2425
+ gte: keyPrefix,
2426
+ lte: [
2427
+ ...keyPrefix,
2428
+ "\uFFFF"
2429
+ ],
2430
+ ...encodingOptions
2431
+ })) {
2432
+ batch.del(key, {
2433
+ ...encodingOptions
2378
2434
  });
2379
2435
  }
2436
+ await batch.write();
2380
2437
  }
2381
2438
  };
2382
- _ts_decorate5([
2383
- import_tracing3.trace.metricsCounter()
2384
- ], ControlPipeline.prototype, "_usage", void 0);
2385
- _ts_decorate5([
2386
- import_tracing3.trace.metricsCounter()
2387
- ], ControlPipeline.prototype, "_mutations", void 0);
2388
- _ts_decorate5([
2389
- import_tracing3.trace.span({
2390
- showInBrowserTimeline: true
2391
- })
2392
- ], ControlPipeline.prototype, "start", null);
2393
- _ts_decorate5([
2394
- import_tracing3.trace.span()
2395
- ], ControlPipeline.prototype, "_consumePipeline", null);
2396
- _ts_decorate5([
2397
- import_tracing3.trace.span()
2398
- ], ControlPipeline.prototype, "_processMessage", null);
2399
- ControlPipeline = _ts_decorate5([
2400
- import_tracing3.trace.resource(),
2401
- (0, import_async9.trackLeaks)("start", "stop")
2402
- ], ControlPipeline);
2403
- function _ts_decorate6(decorators, target, key, desc) {
2439
+ var keyEncoder = {
2440
+ encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
2441
+ decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
2442
+ format: "buffer"
2443
+ };
2444
+ var encodingOptions = {
2445
+ keyEncoding: keyEncoder,
2446
+ valueEncoding: "buffer"
2447
+ };
2448
+ var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
2449
+ function _ts_decorate7(decorators, target, key, desc) {
2404
2450
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2405
2451
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2406
2452
  r = Reflect.decorate(decorators, target, key, desc);
@@ -2410,495 +2456,444 @@ function _ts_decorate6(decorators, target, key, desc) {
2410
2456
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2411
2457
  return c > 3 && r && Object.defineProperty(target, key, r), r;
2412
2458
  }
2413
- var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
2414
- var Space = class extends import_context9.Resource {
2415
- constructor(params) {
2459
+ var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
2460
+ var AutomergeHost = class extends import_context9.Resource {
2461
+ constructor({ db, indexMetadataStore, dataMonitor }) {
2416
2462
  super();
2417
- this.onCredentialProcessed = new import_util4.Callback();
2418
- this.stateUpdate = new import_async8.Event();
2419
- (0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
2420
- F: __dxlog_file11,
2421
- L: 76,
2422
- S: this,
2423
- A: [
2424
- "params.spaceKey && params.feedProvider",
2425
- ""
2426
- ]
2463
+ this._collectionSynchronizer = new CollectionSynchronizer({
2464
+ queryCollectionState: this._queryCollectionState.bind(this),
2465
+ sendCollectionState: this._sendCollectionState.bind(this),
2466
+ shouldSyncCollection: this._shouldSyncCollection.bind(this)
2427
2467
  });
2428
- this._id = params.id;
2429
- this._key = params.spaceKey;
2430
- this._genesisFeedKey = params.genesisFeed.key;
2431
- this._feedProvider = params.feedProvider;
2432
- this._snapshotManager = params.snapshotManager;
2433
- this._controlPipeline = new ControlPipeline({
2434
- spaceKey: params.spaceKey,
2435
- genesisFeed: params.genesisFeed,
2436
- feedProvider: params.feedProvider,
2437
- metadataStore: params.metadataStore
2468
+ this._db = db;
2469
+ this._storage = new LevelDBStorageAdapter({
2470
+ db: db.sublevel("automerge"),
2471
+ callbacks: {
2472
+ beforeSave: async (params) => this._beforeSave(params),
2473
+ afterSave: async (key) => this._afterSave(key)
2474
+ },
2475
+ monitor: dataMonitor
2438
2476
  });
2439
- this._controlPipeline.onFeedAdmitted.set(async (info) => {
2440
- const sparse = info.assertion.designation === import_credentials.AdmittedFeed.Designation.DATA;
2441
- if (!info.key.equals(params.genesisFeed.key)) {
2442
- queueMicrotask(async () => {
2443
- this.protocol.addFeed(await params.feedProvider(info.key, {
2444
- sparse
2445
- }));
2446
- });
2447
- }
2477
+ this._echoNetworkAdapter = new EchoNetworkAdapter({
2478
+ getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
2479
+ onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
2480
+ onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
2481
+ monitor: dataMonitor
2448
2482
  });
2449
- this._controlPipeline.onCredentialProcessed.set(async (credential) => {
2450
- await this.onCredentialProcessed.callIfSet(credential);
2451
- (0, import_log8.log)("onCredentialProcessed", {
2452
- credential
2453
- }, {
2454
- F: __dxlog_file11,
2455
- L: 104,
2456
- S: this,
2457
- C: (f, a) => f(...a)
2458
- });
2459
- this.stateUpdate.emit();
2483
+ this._headsStore = new HeadsStore({
2484
+ db: db.sublevel("heads")
2460
2485
  });
2461
- this._controlPipeline.onDelegatedInvitation.set(async (invitation) => {
2462
- (0, import_log8.log)("onDelegatedInvitation", {
2463
- invitation
2464
- }, {
2465
- F: __dxlog_file11,
2466
- L: 108,
2467
- S: this,
2468
- C: (f, a) => f(...a)
2469
- });
2470
- await params.onDelegatedInvitationStatusChange(invitation, true);
2486
+ this._indexMetadataStore = indexMetadataStore;
2487
+ }
2488
+ async _open() {
2489
+ this._peerId = `host-${import_keys8.PublicKey.random().toHex()}`;
2490
+ await this._storage.open?.();
2491
+ this._repo = new import_automerge_repo.Repo({
2492
+ peerId: this._peerId,
2493
+ sharePolicy: this._sharePolicy.bind(this),
2494
+ storage: this._storage,
2495
+ network: [
2496
+ // Upstream swarm.
2497
+ this._echoNetworkAdapter
2498
+ ]
2471
2499
  });
2472
- this._controlPipeline.onDelegatedInvitationRemoved.set(async (invitation) => {
2473
- (0, import_log8.log)("onDelegatedInvitationRemoved", {
2474
- invitation
2500
+ import_async10.Event.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => this._onPeerConnected(e.peerId));
2501
+ import_async10.Event.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => this._onPeerDisconnected(e.peerId));
2502
+ this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
2503
+ this._onRemoteCollectionStateUpdated(collectionId, peerId);
2504
+ });
2505
+ await this._echoNetworkAdapter.open();
2506
+ await this._collectionSynchronizer.open();
2507
+ await this._echoNetworkAdapter.open();
2508
+ await this._echoNetworkAdapter.whenConnected();
2509
+ }
2510
+ async _close() {
2511
+ await this._collectionSynchronizer.close();
2512
+ await this._storage.close?.();
2513
+ await this._echoNetworkAdapter.close();
2514
+ await this._ctx.dispose();
2515
+ }
2516
+ /**
2517
+ * @deprecated To be abstracted away.
2518
+ */
2519
+ get repo() {
2520
+ return this._repo;
2521
+ }
2522
+ get peerId() {
2523
+ return this._peerId;
2524
+ }
2525
+ get loadedDocsCount() {
2526
+ return Object.keys(this._repo.handles).length;
2527
+ }
2528
+ async addReplicator(replicator) {
2529
+ await this._echoNetworkAdapter.addReplicator(replicator);
2530
+ }
2531
+ async removeReplicator(replicator) {
2532
+ await this._echoNetworkAdapter.removeReplicator(replicator);
2533
+ }
2534
+ /**
2535
+ * Loads the document handle from the repo and waits for it to be ready.
2536
+ */
2537
+ async loadDoc(ctx, documentId, opts) {
2538
+ let handle;
2539
+ if (typeof documentId === "string") {
2540
+ handle = this._repo.handles[documentId];
2541
+ }
2542
+ if (!handle) {
2543
+ handle = this._repo.find(documentId);
2544
+ }
2545
+ if (!handle.isReady()) {
2546
+ if (!opts?.timeout) {
2547
+ await (0, import_context9.cancelWithContext)(ctx, handle.whenReady());
2548
+ } else {
2549
+ await (0, import_context9.cancelWithContext)(ctx, (0, import_async10.asyncTimeout)(handle.whenReady(), opts.timeout));
2550
+ }
2551
+ }
2552
+ return handle;
2553
+ }
2554
+ /**
2555
+ * Create new persisted document.
2556
+ */
2557
+ createDoc(initialValue, opts) {
2558
+ if (opts?.preserveHistory) {
2559
+ if (!(0, import_automerge3.isAutomerge)(initialValue)) {
2560
+ throw new TypeError("Initial value must be an Automerge document");
2561
+ }
2562
+ return this._repo.import((0, import_automerge3.save)(initialValue));
2563
+ } else {
2564
+ return this._repo.create(initialValue);
2565
+ }
2566
+ }
2567
+ async waitUntilHeadsReplicated(heads) {
2568
+ const entries = heads.entries;
2569
+ if (!entries?.length) {
2570
+ return;
2571
+ }
2572
+ const documentIds = entries.map((entry) => entry.documentId);
2573
+ const documentHeads = await this.getHeads(documentIds);
2574
+ const headsToWait = entries.filter((entry, index) => {
2575
+ const targetHeads = entry.heads;
2576
+ if (!targetHeads || targetHeads.length === 0) {
2577
+ return false;
2578
+ }
2579
+ const currentHeads = documentHeads[index];
2580
+ return !(currentHeads !== null && (0, import_automerge3.equals)(currentHeads, targetHeads));
2581
+ });
2582
+ if (headsToWait.length > 0) {
2583
+ await Promise.all(headsToWait.map(async (entry, index) => {
2584
+ const handle = await this.loadDoc(import_context9.Context.default(void 0, {
2585
+ F: __dxlog_file12,
2586
+ L: 226
2587
+ }), entry.documentId);
2588
+ await waitForHeads(handle, entry.heads);
2589
+ }));
2590
+ }
2591
+ await this._repo.flush(documentIds.filter((documentId) => !!this._repo.handles[documentId]));
2592
+ }
2593
+ async reIndexHeads(documentIds) {
2594
+ for (const documentId of documentIds) {
2595
+ import_log11.log.info("re-indexing heads for document", {
2596
+ documentId
2475
2597
  }, {
2476
- F: __dxlog_file11,
2477
- L: 112,
2598
+ F: __dxlog_file12,
2599
+ L: 238,
2478
2600
  S: this,
2479
2601
  C: (f, a) => f(...a)
2480
2602
  });
2481
- await params.onDelegatedInvitationStatusChange(invitation, false);
2482
- });
2483
- this._controlPipeline.onMemberRoleChanged.set(async (changedMembers) => {
2484
- (0, import_log8.log)("onMemberRoleChanged", () => ({
2485
- changedMembers: changedMembers.map((m) => [
2486
- m.key,
2487
- m.role
2488
- ])
2489
- }), {
2490
- F: __dxlog_file11,
2491
- L: 116,
2603
+ const handle = this._repo.find(documentId);
2604
+ await handle.whenReady([
2605
+ "ready",
2606
+ "requesting"
2607
+ ]);
2608
+ if (handle.inState([
2609
+ "requesting"
2610
+ ])) {
2611
+ import_log11.log.warn("document is not available locally, skipping", {
2612
+ documentId
2613
+ }, {
2614
+ F: __dxlog_file12,
2615
+ L: 242,
2616
+ S: this,
2617
+ C: (f, a) => f(...a)
2618
+ });
2619
+ continue;
2620
+ }
2621
+ const doc = handle.docSync();
2622
+ (0, import_invariant8.invariant)(doc, void 0, {
2623
+ F: __dxlog_file12,
2624
+ L: 247,
2492
2625
  S: this,
2493
- C: (f, a) => f(...a)
2626
+ A: [
2627
+ "doc",
2628
+ ""
2629
+ ]
2494
2630
  });
2495
- await params.onMemberRolesChanged(changedMembers);
2496
- });
2497
- this.protocol = params.protocol;
2498
- this.protocol.addFeed(params.genesisFeed);
2499
- }
2500
- get id() {
2501
- return this._id;
2502
- }
2503
- get key() {
2504
- return this._key;
2505
- }
2506
- get isOpen() {
2507
- return this._lifecycleState === import_context9.LifecycleState.OPEN;
2508
- }
2509
- get genesisFeedKey() {
2510
- return this._genesisFeedKey;
2631
+ const heads = (0, import_automerge3.getHeads)(doc);
2632
+ const batch = this._db.batch();
2633
+ this._headsStore.setHeads(documentId, heads, batch);
2634
+ await batch.write();
2635
+ }
2636
+ import_log11.log.info("done re-indexing heads", void 0, {
2637
+ F: __dxlog_file12,
2638
+ L: 254,
2639
+ S: this,
2640
+ C: (f, a) => f(...a)
2641
+ });
2511
2642
  }
2512
- get controlFeedKey() {
2513
- return this._controlFeed?.key;
2643
+ // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
2644
+ // Hosts, running in the worker, don't share documents unless requested by other peers.
2645
+ // NOTE: If both peers return sharePolicy=false the replication will not happen
2646
+ // https://github.com/automerge/automerge-repo/pull/292
2647
+ async _sharePolicy(peerId, documentId) {
2648
+ if (peerId.startsWith("client-")) {
2649
+ return false;
2650
+ }
2651
+ if (!documentId) {
2652
+ return false;
2653
+ }
2654
+ const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
2655
+ if (isEchoPeerMetadata(peerMetadata)) {
2656
+ return this._echoNetworkAdapter.shouldAdvertise(peerId, {
2657
+ documentId
2658
+ });
2659
+ }
2660
+ return false;
2514
2661
  }
2515
- get dataFeedKey() {
2516
- return this._dataFeed?.key;
2662
+ async _beforeSave({ path, batch }) {
2663
+ const handle = this._repo.handles[path[0]];
2664
+ if (!handle) {
2665
+ return;
2666
+ }
2667
+ const doc = handle.docSync();
2668
+ if (!doc) {
2669
+ return;
2670
+ }
2671
+ const heads = (0, import_automerge3.getHeads)(doc);
2672
+ this._headsStore.setHeads(handle.documentId, heads, batch);
2673
+ const spaceKey = getSpaceKeyFromDoc(doc) ?? void 0;
2674
+ const objectIds = Object.keys(doc.objects ?? {});
2675
+ const encodedIds = objectIds.map((objectId) => import_protocols7.objectPointerCodec.encode({
2676
+ documentId: handle.documentId,
2677
+ objectId,
2678
+ spaceKey
2679
+ }));
2680
+ const idToLastHash = new Map(encodedIds.map((id) => [
2681
+ id,
2682
+ heads
2683
+ ]));
2684
+ this._indexMetadataStore.markDirty(idToLastHash, batch);
2517
2685
  }
2518
- get spaceState() {
2519
- return this._controlPipeline.spaceState;
2686
+ _shouldSyncCollection(collectionId, peerId) {
2687
+ const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
2688
+ if (isEchoPeerMetadata(peerMetadata)) {
2689
+ return this._echoNetworkAdapter.shouldSyncCollection(peerId, {
2690
+ collectionId
2691
+ });
2692
+ }
2693
+ return false;
2520
2694
  }
2521
2695
  /**
2522
- * @test-only
2696
+ * Called by AutomergeStorageAdapter after levelDB batch commit.
2523
2697
  */
2524
- get controlPipeline() {
2525
- return this._controlPipeline.pipeline;
2526
- }
2527
- get snapshotManager() {
2528
- return this._snapshotManager;
2698
+ async _afterSave(path) {
2699
+ this._indexMetadataStore.notifyMarkedDirty();
2700
+ const documentId = path[0];
2701
+ const document = this._repo.handles[documentId]?.docSync();
2702
+ if (document) {
2703
+ const heads = (0, import_automerge3.getHeads)(document);
2704
+ this._onHeadsChanged(documentId, heads);
2705
+ }
2529
2706
  }
2530
- async setControlFeed(feed) {
2531
- (0, import_invariant9.invariant)(!this._controlFeed, "Control feed already set.", {
2532
- F: __dxlog_file11,
2533
- L: 169,
2534
- S: this,
2535
- A: [
2536
- "!this._controlFeed",
2537
- "'Control feed already set.'"
2538
- ]
2539
- });
2540
- this._controlFeed = feed;
2541
- await this._controlPipeline.setWriteFeed(feed);
2542
- return this;
2707
+ _automergePeers() {
2708
+ return this._repo.peers;
2543
2709
  }
2544
- async setDataFeed(feed) {
2545
- (0, import_invariant9.invariant)(!this._dataFeed, "Data feed already set.", {
2546
- F: __dxlog_file11,
2547
- L: 176,
2548
- S: this,
2549
- A: [
2550
- "!this._dataFeed",
2551
- "'Data feed already set.'"
2552
- ]
2553
- });
2554
- this._dataFeed = feed;
2555
- return this;
2710
+ async _getContainingSpaceForDocument(documentId) {
2711
+ const doc = this._repo.handles[documentId]?.docSync();
2712
+ if (!doc) {
2713
+ return null;
2714
+ }
2715
+ const spaceKeyHex = getSpaceKeyFromDoc(doc);
2716
+ if (!spaceKeyHex) {
2717
+ return null;
2718
+ }
2719
+ return import_keys8.PublicKey.from(spaceKeyHex);
2556
2720
  }
2557
2721
  /**
2558
- * Use for diagnostics.
2722
+ * Flush documents to disk.
2559
2723
  */
2560
- getControlFeeds() {
2561
- return Array.from(this._controlPipeline.spaceState.feeds.values());
2562
- }
2563
- async _open(ctx) {
2564
- (0, import_log8.log)("opening...", void 0, {
2565
- F: __dxlog_file11,
2566
- L: 190,
2567
- S: this,
2568
- C: (f, a) => f(...a)
2569
- });
2570
- await this._controlPipeline.start();
2571
- await this.protocol.start();
2572
- (0, import_log8.log)("opened", void 0, {
2573
- F: __dxlog_file11,
2574
- L: 196,
2575
- S: this,
2576
- C: (f, a) => f(...a)
2577
- });
2578
- }
2579
- async _close() {
2580
- (0, import_log8.log)("closing...", {
2581
- key: this._key
2582
- }, {
2583
- F: __dxlog_file11,
2584
- L: 201,
2585
- S: this,
2586
- C: (f, a) => f(...a)
2587
- });
2588
- await this.protocol.stop();
2589
- await this._controlPipeline.stop();
2590
- (0, import_log8.log)("closed", void 0, {
2591
- F: __dxlog_file11,
2592
- L: 207,
2593
- S: this,
2594
- C: (f, a) => f(...a)
2595
- });
2596
- }
2597
- };
2598
- _ts_decorate6([
2599
- import_tracing2.trace.info()
2600
- ], Space.prototype, "protocol", void 0);
2601
- _ts_decorate6([
2602
- import_tracing2.trace.info()
2603
- ], Space.prototype, "_controlPipeline", void 0);
2604
- _ts_decorate6([
2605
- import_log8.logInfo,
2606
- import_tracing2.trace.info()
2607
- ], Space.prototype, "id", null);
2608
- _ts_decorate6([
2609
- import_log8.logInfo,
2610
- import_tracing2.trace.info()
2611
- ], Space.prototype, "key", null);
2612
- _ts_decorate6([
2613
- import_tracing2.trace.span()
2614
- ], Space.prototype, "_open", null);
2615
- _ts_decorate6([
2616
- import_async8.synchronized
2617
- ], Space.prototype, "_close", null);
2618
- Space = _ts_decorate6([
2619
- (0, import_async8.trackLeaks)("open", "close"),
2620
- import_tracing2.trace.resource()
2621
- ], Space);
2622
- var SPACE_IDS_CACHE = new import_util4.ComplexMap(import_keys5.PublicKey.hash);
2623
- var createIdFromSpaceKey = async (spaceKey) => {
2624
- const cachedValue = SPACE_IDS_CACHE.get(spaceKey);
2625
- if (cachedValue !== void 0) {
2626
- return cachedValue;
2724
+ async flush({ documentIds } = {}) {
2725
+ await this._repo.flush(documentIds);
2627
2726
  }
2628
- const digest = await import_crypto3.subtleCrypto.digest("SHA-256", spaceKey.asUint8Array());
2629
- const bytes = new Uint8Array(digest).slice(0, import_keys5.SpaceId.byteLength);
2630
- const spaceId = import_keys5.SpaceId.encode(bytes);
2631
- SPACE_IDS_CACHE.set(spaceKey, spaceId);
2632
- return spaceId;
2633
- };
2634
- var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/admission-discovery-extension.ts";
2635
- var CredentialRetrieverExtension = class extends import_teleport2.RpcExtension {
2636
- constructor(_request, _onResult) {
2637
- super({
2638
- requested: {
2639
- AdmissionDiscoveryService: import_protocols6.schema.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
2727
+ async getHeads(documentIds) {
2728
+ const result = [];
2729
+ const storeRequestIds = [];
2730
+ const storeResultIndices = [];
2731
+ for (const documentId of documentIds) {
2732
+ const doc = this._repo.handles[documentId]?.docSync();
2733
+ if (doc) {
2734
+ result.push((0, import_automerge3.getHeads)(doc));
2735
+ } else {
2736
+ storeRequestIds.push(documentId);
2737
+ storeResultIndices.push(result.length);
2738
+ result.push(void 0);
2640
2739
  }
2641
- });
2642
- this._request = _request;
2643
- this._onResult = _onResult;
2644
- this._ctx = new import_context11.Context(void 0, {
2645
- F: __dxlog_file12,
2646
- L: 25
2647
- });
2740
+ }
2741
+ if (storeRequestIds.length > 0) {
2742
+ const storedHeads = await this._headsStore.getHeads(storeRequestIds);
2743
+ for (let i = 0; i < storedHeads.length; i++) {
2744
+ result[storeResultIndices[i]] = storedHeads[i];
2745
+ }
2746
+ }
2747
+ return result;
2648
2748
  }
2649
- async getHandlers() {
2650
- return {};
2749
+ //
2750
+ // Collection sync.
2751
+ //
2752
+ getLocalCollectionState(collectionId) {
2753
+ return this._collectionSynchronizer.getLocalCollectionState(collectionId);
2651
2754
  }
2652
- async onOpen(context) {
2653
- await super.onOpen(context);
2654
- (0, import_async10.scheduleTask)(this._ctx, async () => {
2655
- try {
2656
- const result = await this.rpc.AdmissionDiscoveryService.getAdmissionCredential(this._request);
2657
- this._onResult.wake(result.admissionCredential);
2658
- } catch (err) {
2659
- context.close(err);
2660
- }
2661
- });
2755
+ getRemoteCollectionStates(collectionId) {
2756
+ return this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
2662
2757
  }
2663
- async onClose() {
2664
- await this._ctx.dispose();
2758
+ refreshCollection(collectionId) {
2759
+ this._collectionSynchronizer.refreshCollection(collectionId);
2665
2760
  }
2666
- async onAbort() {
2667
- await this._ctx.dispose();
2761
+ async getCollectionSyncState(collectionId) {
2762
+ const result = {
2763
+ peers: []
2764
+ };
2765
+ const localState = this.getLocalCollectionState(collectionId);
2766
+ const remoteState = this.getRemoteCollectionStates(collectionId);
2767
+ if (!localState) {
2768
+ return result;
2769
+ }
2770
+ for (const [peerId, state] of remoteState) {
2771
+ const diff = diffCollectionState(localState, state);
2772
+ result.peers.push({
2773
+ peerId,
2774
+ differentDocuments: diff.different.length
2775
+ });
2776
+ }
2777
+ return result;
2668
2778
  }
2669
- };
2670
- var CredentialServerExtension = class extends import_teleport2.RpcExtension {
2671
- constructor(_space) {
2672
- super({
2673
- exposed: {
2674
- AdmissionDiscoveryService: import_protocols6.schema.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
2675
- }
2779
+ /**
2780
+ * Update the local collection state based on the locally stored document heads.
2781
+ */
2782
+ async updateLocalCollectionState(collectionId, documentIds) {
2783
+ const heads = await this.getHeads(documentIds);
2784
+ const documents = Object.fromEntries(heads.map((heads2, index) => [
2785
+ documentIds[index],
2786
+ heads2 ?? []
2787
+ ]));
2788
+ this._collectionSynchronizer.setLocalCollectionState(collectionId, {
2789
+ documents
2676
2790
  });
2677
- this._space = _space;
2678
2791
  }
2679
- async getHandlers() {
2680
- return {
2681
- AdmissionDiscoveryService: {
2682
- getAdmissionCredential: async (request) => {
2683
- const memberInfo = this._space.spaceState.members.get(request.memberKey);
2684
- if (!memberInfo?.credential) {
2685
- throw new import_protocols6.ProtocolError("Space member not found.", request);
2686
- }
2687
- return {
2688
- admissionCredential: memberInfo.credential
2689
- };
2690
- }
2691
- }
2692
- };
2792
+ _onCollectionStateQueried(collectionId, peerId) {
2793
+ this._collectionSynchronizer.onCollectionStateQueried(collectionId, peerId);
2794
+ }
2795
+ _onCollectionStateReceived(collectionId, peerId, state) {
2796
+ this._collectionSynchronizer.onRemoteStateReceived(collectionId, peerId, decodeCollectionState(state));
2693
2797
  }
2694
- };
2695
- function _ts_decorate7(decorators, target, key, desc) {
2696
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2697
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2698
- r = Reflect.decorate(decorators, target, key, desc);
2699
- else
2700
- for (var i = decorators.length - 1; i >= 0; i--)
2701
- if (d = decorators[i])
2702
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2703
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2704
- }
2705
- var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
2706
- var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
2707
- var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
2708
- var SpaceProtocol = class {
2709
- constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore }) {
2710
- this._feeds = /* @__PURE__ */ new Set();
2711
- this._sessions = new import_util6.ComplexMap(import_keys7.PublicKey.hash);
2712
- this._topology = new import_network_manager.MMSTTopology({
2713
- originateConnections: 4,
2714
- maxPeers: 10,
2715
- sampleSize: 20
2716
- });
2717
- this._spaceKey = topic;
2718
- this._networkManager = networkManager;
2719
- this._swarmIdentity = swarmIdentity;
2720
- this._onSessionAuth = onSessionAuth;
2721
- this._onAuthFailure = onAuthFailure;
2722
- this.blobSync = new import_teleport_extension_object_sync.BlobSync({
2723
- blobStore
2724
- });
2725
- this._topic = import_crypto4.subtleCrypto.digest("SHA-256", topic.asBuffer()).then(import_crypto4.discoveryKey).then(import_keys7.PublicKey.from);
2798
+ _queryCollectionState(collectionId, peerId) {
2799
+ this._echoNetworkAdapter.queryCollectionState(collectionId, peerId);
2726
2800
  }
2727
- get sessions() {
2728
- return this._sessions;
2801
+ _sendCollectionState(collectionId, peerId, state) {
2802
+ this._echoNetworkAdapter.sendCollectionState(collectionId, peerId, encodeCollectionState(state));
2729
2803
  }
2730
- get feeds() {
2731
- return this._feeds;
2804
+ _onPeerConnected(peerId) {
2805
+ this._collectionSynchronizer.onConnectionOpen(peerId);
2732
2806
  }
2733
- get _ownPeerKey() {
2734
- return this._swarmIdentity.peerKey;
2807
+ _onPeerDisconnected(peerId) {
2808
+ this._collectionSynchronizer.onConnectionClosed(peerId);
2735
2809
  }
2736
- // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
2737
- addFeed(feed) {
2738
- (0, import_log10.log)("addFeed", {
2739
- key: feed.key
2740
- }, {
2741
- F: __dxlog_file13,
2742
- L: 109,
2743
- S: this,
2744
- C: (f, a) => f(...a)
2745
- });
2746
- this._feeds.add(feed);
2747
- for (const session of this._sessions.values()) {
2748
- session.replicator.addFeed(feed);
2810
+ _onRemoteCollectionStateUpdated(collectionId, peerId) {
2811
+ const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
2812
+ const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
2813
+ if (!localState || !remoteState) {
2814
+ return;
2749
2815
  }
2750
- }
2751
- // TODO(burdon): Rename open? Common open/close interfaces for all services?
2752
- async start() {
2753
- if (this._connection) {
2816
+ const { different } = diffCollectionState(localState, remoteState);
2817
+ if (different.length === 0) {
2754
2818
  return;
2755
2819
  }
2756
- const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(""));
2757
- await this.blobSync.open();
2758
- (0, import_log10.log)("starting...", void 0, {
2759
- F: __dxlog_file13,
2760
- L: 128,
2761
- S: this,
2762
- C: (f, a) => f(...a)
2763
- });
2764
- const topic = await this._topic;
2765
- this._connection = await this._networkManager.joinSwarm({
2766
- protocolProvider: this._createProtocolProvider(credentials),
2767
- peerId: this._swarmIdentity.peerKey,
2768
- topic,
2769
- topology: this._topology,
2770
- label: `swarm ${topic.truncate()} for space ${this._spaceKey.truncate()}`
2771
- });
2772
- (0, import_log10.log)("started", void 0, {
2773
- F: __dxlog_file13,
2774
- L: 138,
2820
+ import_log11.log.info("replication documents after collection sync", {
2821
+ count: different.length
2822
+ }, {
2823
+ F: __dxlog_file12,
2824
+ L: 463,
2775
2825
  S: this,
2776
2826
  C: (f, a) => f(...a)
2777
2827
  });
2778
- }
2779
- updateTopology() {
2780
- this._topology.forceUpdate();
2781
- }
2782
- async stop() {
2783
- await this.blobSync.close();
2784
- if (this._connection) {
2785
- (0, import_log10.log)("stopping...", void 0, {
2786
- F: __dxlog_file13,
2787
- L: 149,
2788
- S: this,
2789
- C: (f, a) => f(...a)
2790
- });
2791
- await this._connection.close();
2792
- (0, import_log10.log)("stopped", void 0, {
2793
- F: __dxlog_file13,
2794
- L: 151,
2795
- S: this,
2796
- C: (f, a) => f(...a)
2797
- });
2828
+ for (const documentId of different) {
2829
+ this._repo.find(documentId);
2798
2830
  }
2799
2831
  }
2800
- _createProtocolProvider(credentials) {
2801
- return (wireParams) => {
2802
- const session = new SpaceProtocolSession({
2803
- wireParams,
2804
- swarmIdentity: this._swarmIdentity,
2805
- onSessionAuth: this._onSessionAuth,
2806
- onAuthFailure: this._onAuthFailure,
2807
- blobSync: this.blobSync
2808
- });
2809
- this._sessions.set(wireParams.remotePeerId, session);
2810
- for (const feed of this._feeds) {
2811
- session.replicator.addFeed(feed);
2832
+ _onHeadsChanged(documentId, heads) {
2833
+ for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
2834
+ const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
2835
+ if (state?.documents[documentId]) {
2836
+ const newState = structuredClone(state);
2837
+ newState.documents[documentId] = heads;
2838
+ this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
2812
2839
  }
2813
- return session;
2814
- };
2840
+ }
2815
2841
  }
2816
2842
  };
2817
2843
  _ts_decorate7([
2818
- import_log10.logInfo,
2819
2844
  import_tracing4.trace.info()
2820
- ], SpaceProtocol.prototype, "_topic", void 0);
2845
+ ], AutomergeHost.prototype, "_peerId", void 0);
2821
2846
  _ts_decorate7([
2822
- import_tracing4.trace.info()
2823
- ], SpaceProtocol.prototype, "_spaceKey", void 0);
2847
+ import_tracing4.trace.info({
2848
+ depth: null
2849
+ })
2850
+ ], AutomergeHost.prototype, "_automergePeers", null);
2824
2851
  _ts_decorate7([
2825
- import_log10.logInfo
2826
- ], SpaceProtocol.prototype, "_ownPeerKey", null);
2827
- SpaceProtocol = _ts_decorate7([
2852
+ import_tracing4.trace.span({
2853
+ showInBrowserTimeline: true
2854
+ })
2855
+ ], AutomergeHost.prototype, "flush", null);
2856
+ AutomergeHost = _ts_decorate7([
2828
2857
  import_tracing4.trace.resource()
2829
- ], SpaceProtocol);
2830
- var AuthStatus;
2831
- (function(AuthStatus2) {
2832
- AuthStatus2["INITIAL"] = "INITIAL";
2833
- AuthStatus2["SUCCESS"] = "SUCCESS";
2834
- AuthStatus2["FAILURE"] = "FAILURE";
2835
- })(AuthStatus || (AuthStatus = {}));
2836
- var SpaceProtocolSession = class {
2837
- // TODO(dmaretskyi): Allow to pass in extra extensions.
2838
- constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync }) {
2839
- this.replicator = new import_teleport_extension_replicator.ReplicatorExtension().setOptions({
2840
- upload: true
2841
- });
2842
- this._authStatus = "INITIAL";
2843
- this._wireParams = wireParams;
2844
- this._swarmIdentity = swarmIdentity;
2845
- this._onSessionAuth = onSessionAuth;
2846
- this._onAuthFailure = onAuthFailure;
2847
- this._blobSync = blobSync;
2848
- this._teleport = new import_teleport3.Teleport(wireParams);
2849
- }
2850
- get authStatus() {
2851
- return this._authStatus;
2852
- }
2853
- get stats() {
2854
- return this._teleport.stats;
2855
- }
2856
- get stream() {
2857
- return this._teleport.stream;
2858
+ ], AutomergeHost);
2859
+ var getSpaceKeyFromDoc = (doc) => {
2860
+ const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
2861
+ if (rawSpaceKey == null) {
2862
+ return null;
2858
2863
  }
2859
- async open(sessionId) {
2860
- await this._teleport.open(sessionId);
2861
- this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
2862
- provider: this._swarmIdentity.credentialProvider,
2863
- verifier: this._swarmIdentity.credentialAuthenticator,
2864
- onAuthSuccess: () => {
2865
- (0, import_log10.log)("Peer authenticated", void 0, {
2866
- F: __dxlog_file13,
2867
- L: 248,
2868
- S: this,
2869
- C: (f, a) => f(...a)
2870
- });
2871
- this._authStatus = "SUCCESS";
2872
- this._onSessionAuth?.(this._teleport);
2873
- },
2874
- onAuthFailure: () => {
2875
- this._authStatus = "FAILURE";
2876
- this._onAuthFailure?.(this._teleport);
2864
+ return String(rawSpaceKey);
2865
+ };
2866
+ var waitForHeads = async (handle, heads) => {
2867
+ const unavailableHeads = new Set(heads);
2868
+ await handle.whenReady();
2869
+ await import_async10.Event.wrap(handle, "change").waitForCondition(() => {
2870
+ for (const changeHash of unavailableHeads.values()) {
2871
+ if (changeIsPresentInDoc(handle.docSync(), changeHash)) {
2872
+ unavailableHeads.delete(changeHash);
2877
2873
  }
2878
- }));
2879
- this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
2880
- this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
2881
- }
2882
- async close() {
2883
- (0, import_log10.log)("close", void 0, {
2884
- F: __dxlog_file13,
2885
- L: 264,
2886
- S: this,
2887
- C: (f, a) => f(...a)
2888
- });
2889
- await this._teleport.close();
2890
- }
2891
- async abort() {
2892
- await this._teleport.abort();
2893
- }
2874
+ }
2875
+ return unavailableHeads.size === 0;
2876
+ });
2894
2877
  };
2895
- _ts_decorate7([
2896
- import_log10.logInfo
2897
- ], SpaceProtocolSession.prototype, "_wireParams", void 0);
2898
- _ts_decorate7([
2899
- import_log10.logInfo
2900
- ], SpaceProtocolSession.prototype, "authStatus", null);
2901
- var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
2878
+ var changeIsPresentInDoc = (doc, changeHash) => {
2879
+ return !!(0, import_automerge3.getBackend)(doc).getChangeByHash(changeHash);
2880
+ };
2881
+ var decodeCollectionState = (state) => {
2882
+ (0, import_invariant8.invariant)(typeof state === "object" && state !== null, "Invalid state", {
2883
+ F: __dxlog_file12,
2884
+ L: 516,
2885
+ S: void 0,
2886
+ A: [
2887
+ "typeof state === 'object' && state !== null",
2888
+ "'Invalid state'"
2889
+ ]
2890
+ });
2891
+ return state;
2892
+ };
2893
+ var encodeCollectionState = (state) => {
2894
+ return state;
2895
+ };
2896
+ var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
2902
2897
  var DEFAULT_FACTORY = (params) => new import_teleport_extension_automerge_replicator.AutomergeReplicator(...params);
2903
2898
  var MeshReplicatorConnection = class extends import_context12.Resource {
2904
2899
  constructor(_params) {
@@ -2916,8 +2911,8 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2916
2911
  });
2917
2912
  this.writable = new WritableStream({
2918
2913
  write: async (message, controller) => {
2919
- (0, import_invariant12.invariant)(this._isEnabled, "Writing to a disabled connection", {
2920
- F: __dxlog_file14,
2914
+ (0, import_invariant11.invariant)(this._isEnabled, "Writing to a disabled connection", {
2915
+ F: __dxlog_file13,
2921
2916
  L: 48,
2922
2917
  S: this,
2923
2918
  A: [
@@ -2949,7 +2944,7 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2949
2944
  thisPeerId: this.peerId,
2950
2945
  remotePeerId: remotePeerId.toHex()
2951
2946
  }, {
2952
- F: __dxlog_file14,
2947
+ F: __dxlog_file13,
2953
2948
  L: 82,
2954
2949
  S: this,
2955
2950
  C: (f, a) => f(...a)
@@ -2975,8 +2970,8 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2975
2970
  }
2976
2971
  }
2977
2972
  get peerId() {
2978
- (0, import_invariant12.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
2979
- F: __dxlog_file14,
2973
+ (0, import_invariant11.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
2974
+ F: __dxlog_file13,
2980
2975
  L: 108,
2981
2976
  S: this,
2982
2977
  A: [
@@ -2997,8 +2992,8 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
2997
2992
  * Call after the remote peer has connected.
2998
2993
  */
2999
2994
  enable() {
3000
- (0, import_invariant12.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
3001
- F: __dxlog_file14,
2995
+ (0, import_invariant11.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
2996
+ F: __dxlog_file13,
3002
2997
  L: 125,
3003
2998
  S: this,
3004
2999
  A: [
@@ -3015,6 +3010,21 @@ var MeshReplicatorConnection = class extends import_context12.Resource {
3015
3010
  this._isEnabled = false;
3016
3011
  }
3017
3012
  };
3013
+ var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
3014
+ var deriveCollectionIdFromSpaceId = (spaceId) => `space:${spaceId}`;
3015
+ var getSpaceIdFromCollectionId = (collectionId) => {
3016
+ const spaceId = collectionId.replace(/^space:/, "");
3017
+ (0, import_invariant12.invariant)(import_keys10.SpaceId.isValid(spaceId), void 0, {
3018
+ F: __dxlog_file14,
3019
+ L: 12,
3020
+ S: void 0,
3021
+ A: [
3022
+ "SpaceId.isValid(spaceId)",
3023
+ ""
3024
+ ]
3025
+ });
3026
+ return spaceId;
3027
+ };
3018
3028
  var __dxlog_file15 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts";
3019
3029
  var MeshEchoReplicator = class {
3020
3030
  constructor() {
@@ -3038,7 +3048,7 @@ var MeshEchoReplicator = class {
3038
3048
  this._context = null;
3039
3049
  }
3040
3050
  createExtension(extensionFactory) {
3041
- (0, import_invariant11.invariant)(this._context, void 0, {
3051
+ (0, import_invariant10.invariant)(this._context, void 0, {
3042
3052
  F: __dxlog_file15,
3043
3053
  L: 56,
3044
3054
  S: this,
@@ -3059,7 +3069,7 @@ var MeshEchoReplicator = class {
3059
3069
  S: this,
3060
3070
  C: (f, a) => f(...a)
3061
3071
  });
3062
- (0, import_invariant11.invariant)(this._context, void 0, {
3072
+ (0, import_invariant10.invariant)(this._context, void 0, {
3063
3073
  F: __dxlog_file15,
3064
3074
  L: 63,
3065
3075
  S: this,
@@ -3100,7 +3110,7 @@ var MeshEchoReplicator = class {
3100
3110
  S: this,
3101
3111
  C: (f, a) => f(...a)
3102
3112
  });
3103
- (0, import_invariant11.invariant)(this._context, void 0, {
3113
+ (0, import_invariant10.invariant)(this._context, void 0, {
3104
3114
  F: __dxlog_file15,
3105
3115
  L: 82,
3106
3116
  S: this,
@@ -3195,7 +3205,7 @@ var MeshEchoReplicator = class {
3195
3205
  C: (f, a) => f(...a)
3196
3206
  });
3197
3207
  const spaceId = await createIdFromSpaceKey(spaceKey);
3198
- (0, import_util8.defaultMap)(this._authorizedDevices, spaceId, () => new import_util8.ComplexSet(import_keys10.PublicKey.hash)).add(deviceKey);
3208
+ (0, import_util8.defaultMap)(this._authorizedDevices, spaceId, () => new import_util8.ComplexSet(import_keys9.PublicKey.hash)).add(deviceKey);
3199
3209
  for (const connection of this._connections) {
3200
3210
  if (connection.remoteDeviceKey && connection.remoteDeviceKey.equals(deviceKey)) {
3201
3211
  if (this._connectionsPerPeer.has(connection.peerId)) {
@@ -3582,7 +3592,7 @@ var DataServiceImpl = class {
3582
3592
  synchronizer.open().then(() => {
3583
3593
  this._subscriptions.set(request.subscriptionId, synchronizer);
3584
3594
  ready();
3585
- }).catch((err) => import_log12.log.catch(err, void 0, {
3595
+ }).catch((err) => import_log10.log.catch(err, void 0, {
3586
3596
  F: __dxlog_file16,
3587
3597
  L: 64,
3588
3598
  S: this,
@@ -3593,7 +3603,7 @@ var DataServiceImpl = class {
3593
3603
  }
3594
3604
  async updateSubscription(request) {
3595
3605
  const synchronizer = this._subscriptions.get(request.subscriptionId);
3596
- (0, import_invariant10.invariant)(synchronizer, "Subscription not found", {
3606
+ (0, import_invariant7.invariant)(synchronizer, "Subscription not found", {
3597
3607
  F: __dxlog_file16,
3598
3608
  L: 71,
3599
3609
  S: this,
@@ -3614,7 +3624,7 @@ var DataServiceImpl = class {
3614
3624
  return;
3615
3625
  }
3616
3626
  const synchronizer = this._subscriptions.get(request.subscriptionId);
3617
- (0, import_invariant10.invariant)(synchronizer, "Subscription not found", {
3627
+ (0, import_invariant7.invariant)(synchronizer, "Subscription not found", {
3618
3628
  F: __dxlog_file16,
3619
3629
  L: 86,
3620
3630
  S: this,
@@ -3657,7 +3667,7 @@ var DataServiceImpl = class {
3657
3667
  await this._updateIndexes();
3658
3668
  }
3659
3669
  async getSpaceSyncState(request, options) {
3660
- (0, import_invariant10.invariant)(import_keys9.SpaceId.isValid(request.spaceId), void 0, {
3670
+ (0, import_invariant7.invariant)(import_keys7.SpaceId.isValid(request.spaceId), void 0, {
3661
3671
  F: __dxlog_file16,
3662
3672
  L: 127,
3663
3673
  S: this,
@@ -3688,14 +3698,15 @@ function _ts_decorate9(decorators, target, key, desc) {
3688
3698
  }
3689
3699
  var __dxlog_file17 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
3690
3700
  var SpaceManager = class {
3691
- constructor({ feedStore, networkManager, metadataStore, snapshotStore, blobStore }) {
3692
- this._spaces = new import_util7.ComplexMap(import_keys8.PublicKey.hash);
3693
- this._instanceId = import_keys8.PublicKey.random().toHex();
3701
+ constructor({ feedStore, networkManager, metadataStore, snapshotStore, blobStore, disableP2pReplication }) {
3702
+ this._spaces = new import_util6.ComplexMap(import_keys6.PublicKey.hash);
3703
+ this._instanceId = import_keys6.PublicKey.random().toHex();
3694
3704
  this._feedStore = feedStore;
3695
3705
  this._networkManager = networkManager;
3696
3706
  this._metadataStore = metadataStore;
3697
3707
  this._snapshotStore = snapshotStore;
3698
3708
  this._blobStore = blobStore;
3709
+ this._disableP2pReplication = disableP2pReplication ?? false;
3699
3710
  }
3700
3711
  // TODO(burdon): Remove.
3701
3712
  get spaces() {
@@ -3709,19 +3720,19 @@ var SpaceManager = class {
3709
3720
  ].map((space) => space.close()));
3710
3721
  }
3711
3722
  async constructSpace({ metadata, swarmIdentity, onAuthorizedConnection, onAuthFailure, onDelegatedInvitationStatusChange, onMemberRolesChanged, memberKey }) {
3712
- import_log11.log.trace("dxos.echo.space-manager.construct-space", import_protocols7.trace.begin({
3723
+ import_log9.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.begin({
3713
3724
  id: this._instanceId
3714
3725
  }), {
3715
3726
  F: __dxlog_file17,
3716
- L: 103,
3727
+ L: 114,
3717
3728
  S: this,
3718
3729
  C: (f, a) => f(...a)
3719
3730
  });
3720
- (0, import_log11.log)("constructing space...", {
3731
+ (0, import_log9.log)("constructing space...", {
3721
3732
  spaceKey: metadata.genesisFeedKey
3722
3733
  }, {
3723
3734
  F: __dxlog_file17,
3724
- L: 104,
3735
+ L: 115,
3725
3736
  S: this,
3726
3737
  C: (f, a) => f(...a)
3727
3738
  });
@@ -3734,7 +3745,8 @@ var SpaceManager = class {
3734
3745
  networkManager: this._networkManager,
3735
3746
  onSessionAuth: onAuthorizedConnection,
3736
3747
  onAuthFailure,
3737
- blobStore: this._blobStore
3748
+ blobStore: this._blobStore,
3749
+ disableP2pReplication: this._disableP2pReplication
3738
3750
  });
3739
3751
  const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
3740
3752
  const space = new Space({
@@ -3750,11 +3762,11 @@ var SpaceManager = class {
3750
3762
  onMemberRolesChanged
3751
3763
  });
3752
3764
  this._spaces.set(space.key, space);
3753
- import_log11.log.trace("dxos.echo.space-manager.construct-space", import_protocols7.trace.end({
3765
+ import_log9.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.end({
3754
3766
  id: this._instanceId
3755
3767
  }), {
3756
3768
  F: __dxlog_file17,
3757
- L: 135,
3769
+ L: 147,
3758
3770
  S: this,
3759
3771
  C: (f, a) => f(...a)
3760
3772
  });
@@ -3762,23 +3774,23 @@ var SpaceManager = class {
3762
3774
  }
3763
3775
  async requestSpaceAdmissionCredential(params) {
3764
3776
  const traceKey = "dxos.echo.space-manager.request-space-admission";
3765
- import_log11.log.trace(traceKey, import_protocols7.trace.begin({
3777
+ import_log9.log.trace(traceKey, import_protocols6.trace.begin({
3766
3778
  id: this._instanceId
3767
3779
  }), {
3768
3780
  F: __dxlog_file17,
3769
- L: 141,
3781
+ L: 153,
3770
3782
  S: this,
3771
3783
  C: (f, a) => f(...a)
3772
3784
  });
3773
- (0, import_log11.log)("requesting space admission credential...", {
3785
+ (0, import_log9.log)("requesting space admission credential...", {
3774
3786
  spaceKey: params.spaceKey
3775
3787
  }, {
3776
3788
  F: __dxlog_file17,
3777
- L: 142,
3789
+ L: 154,
3778
3790
  S: this,
3779
3791
  C: (f, a) => f(...a)
3780
3792
  });
3781
- const onCredentialResolved = new import_async11.Trigger();
3793
+ const onCredentialResolved = new import_async9.Trigger();
3782
3794
  const protocol = new SpaceProtocol({
3783
3795
  topic: params.spaceKey,
3784
3796
  swarmIdentity: params.swarmIdentity,
@@ -3790,29 +3802,30 @@ var SpaceManager = class {
3790
3802
  }, onCredentialResolved));
3791
3803
  },
3792
3804
  onAuthFailure: (session) => session.close(),
3793
- blobStore: this._blobStore
3805
+ blobStore: this._blobStore,
3806
+ disableP2pReplication: this._disableP2pReplication
3794
3807
  });
3795
3808
  try {
3796
3809
  await protocol.start();
3797
3810
  const credential = await onCredentialResolved.wait({
3798
3811
  timeout: params.timeout
3799
3812
  });
3800
- import_log11.log.trace(traceKey, import_protocols7.trace.end({
3813
+ import_log9.log.trace(traceKey, import_protocols6.trace.end({
3801
3814
  id: this._instanceId
3802
3815
  }), {
3803
3816
  F: __dxlog_file17,
3804
- L: 165,
3817
+ L: 178,
3805
3818
  S: this,
3806
3819
  C: (f, a) => f(...a)
3807
3820
  });
3808
3821
  return credential;
3809
3822
  } catch (err) {
3810
- import_log11.log.trace(traceKey, import_protocols7.trace.error({
3823
+ import_log9.log.trace(traceKey, import_protocols6.trace.error({
3811
3824
  id: this._instanceId,
3812
3825
  error: err
3813
3826
  }), {
3814
3827
  F: __dxlog_file17,
3815
- L: 168,
3828
+ L: 181,
3816
3829
  S: this,
3817
3830
  C: (f, a) => f(...a)
3818
3831
  });
@@ -3823,13 +3836,13 @@ var SpaceManager = class {
3823
3836
  }
3824
3837
  };
3825
3838
  _ts_decorate9([
3826
- import_async11.synchronized
3839
+ import_async9.synchronized
3827
3840
  ], SpaceManager.prototype, "open", null);
3828
3841
  _ts_decorate9([
3829
- import_async11.synchronized
3842
+ import_async9.synchronized
3830
3843
  ], SpaceManager.prototype, "close", null);
3831
3844
  SpaceManager = _ts_decorate9([
3832
- (0, import_async11.trackLeaks)("open", "close")
3845
+ (0, import_async9.trackLeaks)("open", "close")
3833
3846
  ], SpaceManager);
3834
3847
  function _ts_decorate10(decorators, target, key, desc) {
3835
3848
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -3844,14 +3857,14 @@ function _ts_decorate10(decorators, target, key, desc) {
3844
3857
  var __dxlog_file18 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
3845
3858
  var EXPIRED_INVITATION_CLEANUP_INTERVAL = 60 * 60 * 1e3;
3846
3859
  var emptyEchoMetadata = () => ({
3847
- version: import_protocols8.STORAGE_VERSION,
3860
+ version: import_protocols9.STORAGE_VERSION,
3848
3861
  spaces: [],
3849
3862
  created: /* @__PURE__ */ new Date(),
3850
3863
  updated: /* @__PURE__ */ new Date()
3851
3864
  });
3852
3865
  var emptyLargeSpaceMetadata = () => ({});
3853
- var EchoMetadata = import_protocols8.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
3854
- var LargeSpaceMetadata = import_protocols8.schema.getCodecForType("dxos.echo.metadata.LargeSpaceMetadata");
3866
+ var EchoMetadata = import_protocols9.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
3867
+ var LargeSpaceMetadata = import_protocols9.schema.getCodecForType("dxos.echo.metadata.LargeSpaceMetadata");
3855
3868
  var MetadataStore = class {
3856
3869
  constructor(directory) {
3857
3870
  this._metadata = emptyEchoMetadata();
@@ -3896,7 +3909,7 @@ var MetadataStore = class {
3896
3909
  C: (f, a) => f(...a)
3897
3910
  });
3898
3911
  if (fileLength < dataSize + 8) {
3899
- throw new import_protocols8.DataCorruptionError("Metadata size is smaller than expected.", {
3912
+ throw new import_protocols9.DataCorruptionError("Metadata size is smaller than expected.", {
3900
3913
  fileLength,
3901
3914
  dataSize
3902
3915
  });
@@ -3904,7 +3917,7 @@ var MetadataStore = class {
3904
3917
  const data = await file.read(8, dataSize);
3905
3918
  const calculatedChecksum = import_crc_32.default.buf(data);
3906
3919
  if (calculatedChecksum !== checksum) {
3907
- throw new import_protocols8.DataCorruptionError("Metadata checksum is invalid.");
3920
+ throw new import_protocols9.DataCorruptionError("Metadata checksum is invalid.");
3908
3921
  }
3909
3922
  return codec2.decode(data);
3910
3923
  } finally {
@@ -3994,7 +4007,7 @@ var MetadataStore = class {
3994
4007
  async _save() {
3995
4008
  const data = {
3996
4009
  ...this._metadata,
3997
- version: import_protocols8.STORAGE_VERSION,
4010
+ version: import_protocols9.STORAGE_VERSION,
3998
4011
  created: this._metadata.created ?? /* @__PURE__ */ new Date(),
3999
4012
  updated: /* @__PURE__ */ new Date()
4000
4013
  };
@@ -4200,4 +4213,4 @@ var isLegacyInvitationFormat = (invitation) => {
4200
4213
  startAfter,
4201
4214
  valueEncoding
4202
4215
  });
4203
- //# sourceMappingURL=chunk-6MWU4MHX.cjs.map
4216
+ //# sourceMappingURL=chunk-LTR534RP.cjs.map