@dxos/echo-pipeline 0.1.24-next.6b1f434

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/LICENSE +8 -0
  2. package/README.md +22 -0
  3. package/dist/lib/browser/chunk-3GAE5CKD.mjs +1621 -0
  4. package/dist/lib/browser/chunk-3GAE5CKD.mjs.map +7 -0
  5. package/dist/lib/browser/index.mjs +114 -0
  6. package/dist/lib/browser/index.mjs.map +7 -0
  7. package/dist/lib/browser/meta.json +1 -0
  8. package/dist/lib/browser/testing/index.mjs +238 -0
  9. package/dist/lib/browser/testing/index.mjs.map +7 -0
  10. package/dist/lib/node/index.cjs +1747 -0
  11. package/dist/lib/node/index.cjs.map +7 -0
  12. package/dist/lib/node/meta.json +1 -0
  13. package/dist/lib/node/testing/index.cjs +1835 -0
  14. package/dist/lib/node/testing/index.cjs.map +7 -0
  15. package/dist/types/src/common/codec.d.ts +12 -0
  16. package/dist/types/src/common/codec.d.ts.map +1 -0
  17. package/dist/types/src/common/feeds.d.ts +7 -0
  18. package/dist/types/src/common/feeds.d.ts.map +1 -0
  19. package/dist/types/src/common/index.d.ts +3 -0
  20. package/dist/types/src/common/index.d.ts.map +1 -0
  21. package/dist/types/src/dbhost/data-service-host.d.ts +22 -0
  22. package/dist/types/src/dbhost/data-service-host.d.ts.map +1 -0
  23. package/dist/types/src/dbhost/data-service.d.ts +21 -0
  24. package/dist/types/src/dbhost/data-service.d.ts.map +1 -0
  25. package/dist/types/src/dbhost/database-backend.d.ts +28 -0
  26. package/dist/types/src/dbhost/database-backend.d.ts.map +1 -0
  27. package/dist/types/src/dbhost/index.d.ts +6 -0
  28. package/dist/types/src/dbhost/index.d.ts.map +1 -0
  29. package/dist/types/src/dbhost/snapshot-manager.d.ts +17 -0
  30. package/dist/types/src/dbhost/snapshot-manager.d.ts.map +1 -0
  31. package/dist/types/src/dbhost/snapshot-store.d.ts +9 -0
  32. package/dist/types/src/dbhost/snapshot-store.d.ts.map +1 -0
  33. package/dist/types/src/dbhost/snapshot-store.test.d.ts +2 -0
  34. package/dist/types/src/dbhost/snapshot-store.test.d.ts.map +1 -0
  35. package/dist/types/src/errors/errors.d.ts +26 -0
  36. package/dist/types/src/errors/errors.d.ts.map +1 -0
  37. package/dist/types/src/errors/index.d.ts +2 -0
  38. package/dist/types/src/errors/index.d.ts.map +1 -0
  39. package/dist/types/src/index.d.ts +7 -0
  40. package/dist/types/src/index.d.ts.map +1 -0
  41. package/dist/types/src/metadata/index.d.ts +2 -0
  42. package/dist/types/src/metadata/index.d.ts.map +1 -0
  43. package/dist/types/src/metadata/metadata-store.d.ts +42 -0
  44. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -0
  45. package/dist/types/src/pipeline/index.d.ts +3 -0
  46. package/dist/types/src/pipeline/index.d.ts.map +1 -0
  47. package/dist/types/src/pipeline/message-selector.d.ts +11 -0
  48. package/dist/types/src/pipeline/message-selector.d.ts.map +1 -0
  49. package/dist/types/src/pipeline/pipeline.d.ts +98 -0
  50. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -0
  51. package/dist/types/src/pipeline/pipeline.test.d.ts +2 -0
  52. package/dist/types/src/pipeline/pipeline.test.d.ts.map +1 -0
  53. package/dist/types/src/pipeline/timeframe-clock.d.ts +19 -0
  54. package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -0
  55. package/dist/types/src/space/auth.d.ts +23 -0
  56. package/dist/types/src/space/auth.d.ts.map +1 -0
  57. package/dist/types/src/space/control-pipeline.d.ts +31 -0
  58. package/dist/types/src/space/control-pipeline.d.ts.map +1 -0
  59. package/dist/types/src/space/control-pipeline.test.d.ts +2 -0
  60. package/dist/types/src/space/control-pipeline.test.d.ts.map +1 -0
  61. package/dist/types/src/space/data-pipeline-controller.d.ts +63 -0
  62. package/dist/types/src/space/data-pipeline-controller.d.ts.map +1 -0
  63. package/dist/types/src/space/genesis.d.ts +5 -0
  64. package/dist/types/src/space/genesis.d.ts.map +1 -0
  65. package/dist/types/src/space/index.d.ts +7 -0
  66. package/dist/types/src/space/index.d.ts.map +1 -0
  67. package/dist/types/src/space/replication.browser.test.d.ts +2 -0
  68. package/dist/types/src/space/replication.browser.test.d.ts.map +1 -0
  69. package/dist/types/src/space/space-manager.browser.test.d.ts +2 -0
  70. package/dist/types/src/space/space-manager.browser.test.d.ts.map +1 -0
  71. package/dist/types/src/space/space-manager.d.ts +41 -0
  72. package/dist/types/src/space/space-manager.d.ts.map +1 -0
  73. package/dist/types/src/space/space-protocol.browser.test.d.ts +2 -0
  74. package/dist/types/src/space/space-protocol.browser.test.d.ts.map +1 -0
  75. package/dist/types/src/space/space-protocol.d.ts +76 -0
  76. package/dist/types/src/space/space-protocol.d.ts.map +1 -0
  77. package/dist/types/src/space/space-protocol.test.d.ts +2 -0
  78. package/dist/types/src/space/space-protocol.test.d.ts.map +1 -0
  79. package/dist/types/src/space/space.d.ts +59 -0
  80. package/dist/types/src/space/space.d.ts.map +1 -0
  81. package/dist/types/src/space/space.test.d.ts +2 -0
  82. package/dist/types/src/space/space.test.d.ts.map +1 -0
  83. package/dist/types/src/testing/index.d.ts +4 -0
  84. package/dist/types/src/testing/index.d.ts.map +1 -0
  85. package/dist/types/src/testing/test-agent-builder.d.ts +52 -0
  86. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -0
  87. package/dist/types/src/testing/test-feed-builder.d.ts +9 -0
  88. package/dist/types/src/testing/test-feed-builder.d.ts.map +1 -0
  89. package/dist/types/src/testing/util.d.ts +14 -0
  90. package/dist/types/src/testing/util.d.ts.map +1 -0
  91. package/dist/types/src/tests/database.test.d.ts +2 -0
  92. package/dist/types/src/tests/database.test.d.ts.map +1 -0
  93. package/package.json +60 -0
  94. package/src/common/codec.ts +18 -0
  95. package/src/common/feeds.ts +23 -0
  96. package/src/common/index.ts +6 -0
  97. package/src/dbhost/data-service-host.ts +103 -0
  98. package/src/dbhost/data-service.ts +66 -0
  99. package/src/dbhost/database-backend.ts +64 -0
  100. package/src/dbhost/index.ts +9 -0
  101. package/src/dbhost/snapshot-manager.ts +68 -0
  102. package/src/dbhost/snapshot-store.test.ts +30 -0
  103. package/src/dbhost/snapshot-store.ts +45 -0
  104. package/src/errors/errors.ts +52 -0
  105. package/src/errors/index.ts +5 -0
  106. package/src/index.ts +10 -0
  107. package/src/metadata/index.ts +5 -0
  108. package/src/metadata/metadata-store.ts +173 -0
  109. package/src/pipeline/index.ts +6 -0
  110. package/src/pipeline/message-selector.ts +38 -0
  111. package/src/pipeline/pipeline.test.ts +71 -0
  112. package/src/pipeline/pipeline.ts +249 -0
  113. package/src/pipeline/timeframe-clock.ts +56 -0
  114. package/src/space/auth.ts +88 -0
  115. package/src/space/control-pipeline.test.ts +162 -0
  116. package/src/space/control-pipeline.ts +103 -0
  117. package/src/space/data-pipeline-controller.ts +234 -0
  118. package/src/space/genesis.ts +29 -0
  119. package/src/space/index.ts +10 -0
  120. package/src/space/replication.browser.test.ts +64 -0
  121. package/src/space/space-manager.browser.test.ts +44 -0
  122. package/src/space/space-manager.ts +92 -0
  123. package/src/space/space-protocol.browser.test.ts +135 -0
  124. package/src/space/space-protocol.test.ts +136 -0
  125. package/src/space/space-protocol.ts +222 -0
  126. package/src/space/space.test.ts +144 -0
  127. package/src/space/space.ts +199 -0
  128. package/src/testing/index.ts +7 -0
  129. package/src/testing/test-agent-builder.ts +193 -0
  130. package/src/testing/test-feed-builder.ts +19 -0
  131. package/src/testing/util.ts +77 -0
  132. package/src/tests/database.test.ts +53 -0
  133. package/testing.d.ts +11 -0
  134. package/testing.js +5 -0
@@ -0,0 +1,1835 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // packages/core/echo/echo-pipeline/src/testing/index.ts
31
+ var testing_exports = {};
32
+ __export(testing_exports, {
33
+ MemoryNetworkManagerProvider: () => MemoryNetworkManagerProvider,
34
+ TestAgent: () => TestAgent,
35
+ TestAgentBuilder: () => TestAgentBuilder,
36
+ TestFeedBuilder: () => TestFeedBuilder,
37
+ WebsocketNetworkManagerProvider: () => WebsocketNetworkManagerProvider,
38
+ createMemoryDatabase: () => createMemoryDatabase,
39
+ createRemoteDatabaseFromDataServiceHost: () => createRemoteDatabaseFromDataServiceHost,
40
+ testLocalDatabase: () => testLocalDatabase
41
+ });
42
+ module.exports = __toCommonJS(testing_exports);
43
+
44
+ // packages/core/echo/echo-pipeline/src/testing/test-agent-builder.ts
45
+ var import_document_model = require("@dxos/document-model");
46
+ var import_keys6 = require("@dxos/keys");
47
+ var import_messaging = require("@dxos/messaging");
48
+ var import_model_factory = require("@dxos/model-factory");
49
+ var import_network_manager2 = require("@dxos/network-manager");
50
+ var import_random_access_storage = require("@dxos/random-access-storage");
51
+ var import_teleport_extension_presence = require("@dxos/teleport-extension-presence");
52
+ var import_util7 = require("@dxos/util");
53
+
54
+ // packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts
55
+ var import_node_assert = __toESM(require("node:assert"));
56
+ var import_codec_protobuf = require("@dxos/codec-protobuf");
57
+ var import_echo_db = require("@dxos/echo-db");
58
+ var import_log = require("@dxos/log");
59
+ var import_util = require("@dxos/util");
60
+ var DataServiceHost = class {
61
+ constructor(_itemManager, _itemDemuxer, _writeStream) {
62
+ this._itemManager = _itemManager;
63
+ this._itemDemuxer = _itemDemuxer;
64
+ this._writeStream = _writeStream;
65
+ this._clientTagMap = new import_util.ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
66
+ }
67
+ /**
68
+ * Real-time subscription to data objects in a space.
69
+ */
70
+ subscribe() {
71
+ return new import_codec_protobuf.Stream(({ next, ctx }) => {
72
+ const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
73
+ next({
74
+ batch: {
75
+ objects
76
+ }
77
+ });
78
+ this._itemDemuxer.mutation.on(ctx, (mutation) => {
79
+ var _a;
80
+ (0, import_log.log)("Object update", {
81
+ mutation
82
+ }, {
83
+ file: "data-service-host.ts",
84
+ line: 49,
85
+ scope: this,
86
+ callSite: (f, a) => f(...a)
87
+ });
88
+ const clientTag = this._clientTagMap.get([
89
+ mutation.meta.feedKey,
90
+ mutation.meta.seq
91
+ ]);
92
+ const batch = {
93
+ objects: [
94
+ {
95
+ ...mutation.data,
96
+ mutations: (_a = mutation.data.mutations) == null ? void 0 : _a.map((m, mutationIdx) => ({
97
+ ...m,
98
+ meta: mutation.meta
99
+ })),
100
+ meta: mutation.meta
101
+ }
102
+ ]
103
+ };
104
+ if (clientTag) {
105
+ (0, import_echo_db.tagMutationsInBatch)(batch, clientTag);
106
+ }
107
+ next({
108
+ clientTag,
109
+ feedKey: mutation.meta.feedKey,
110
+ seq: mutation.meta.seq,
111
+ batch
112
+ });
113
+ });
114
+ });
115
+ }
116
+ async write(request) {
117
+ var _a, _b;
118
+ (0, import_node_assert.default)(this._writeStream, "Cannot write mutations in readonly mode");
119
+ (0, import_node_assert.default)(((_a = request.batch.objects) == null ? void 0 : _a.length) === 1, "Only single object mutations are supported");
120
+ const receipt = await this._writeStream.write({
121
+ object: {
122
+ ...request.batch.objects[0],
123
+ mutations: (_b = request.batch.objects[0].mutations) == null ? void 0 : _b.map((m) => ({
124
+ ...m,
125
+ meta: void 0
126
+ })),
127
+ meta: void 0
128
+ }
129
+ });
130
+ if (request.clientTag) {
131
+ this._clientTagMap.set([
132
+ receipt.feedKey,
133
+ receipt.seq
134
+ ], request.clientTag);
135
+ }
136
+ return receipt;
137
+ }
138
+ };
139
+
140
+ // packages/core/echo/echo-pipeline/src/dbhost/database-backend.ts
141
+ var import_echo_db2 = require("@dxos/echo-db");
142
+ var DatabaseBackendHost = class {
143
+ constructor(_outboundStream, _snapshot, _options = {}) {
144
+ this._outboundStream = _outboundStream;
145
+ this._snapshot = _snapshot;
146
+ this._options = _options;
147
+ }
148
+ get isReadOnly() {
149
+ return !!this._outboundStream;
150
+ }
151
+ get echoProcessor() {
152
+ return this._echoProcessor;
153
+ }
154
+ async open(itemManager, modelFactory) {
155
+ this._itemManager = itemManager;
156
+ this._itemManager._debugLabel = "host";
157
+ this._itemDemuxer = new import_echo_db2.ItemDemuxer(itemManager, modelFactory, this._options);
158
+ this._echoProcessor = this._itemDemuxer.open();
159
+ if (this._snapshot) {
160
+ await this._itemDemuxer.restoreFromSnapshot(this._snapshot);
161
+ }
162
+ }
163
+ async close() {
164
+ }
165
+ getWriteStream() {
166
+ return this._outboundStream;
167
+ }
168
+ createSnapshot() {
169
+ return this._itemDemuxer.createSnapshot();
170
+ }
171
+ createDataServiceHost() {
172
+ var _a;
173
+ return new DataServiceHost(this._itemManager, this._itemDemuxer, (_a = this._outboundStream) != null ? _a : void 0);
174
+ }
175
+ };
176
+
177
+ // packages/core/echo/echo-pipeline/src/dbhost/snapshot-manager.ts
178
+ var import_async = require("@dxos/async");
179
+ var import_protocols = require("@dxos/protocols");
180
+ var import_teleport_extension_object_sync = require("@dxos/teleport-extension-object-sync");
181
+ var __decorate = function(decorators, target, key, desc) {
182
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
183
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
184
+ r = Reflect.decorate(decorators, target, key, desc);
185
+ else
186
+ for (var i = decorators.length - 1; i >= 0; i--)
187
+ if (d = decorators[i])
188
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
189
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
190
+ };
191
+ var SnapshotManager = class SnapshotManager2 {
192
+ // prettier-ignore
193
+ constructor(_snapshotStore) {
194
+ this._snapshotStore = _snapshotStore;
195
+ this._objectSync = new import_teleport_extension_object_sync.ObjectSync({
196
+ getObject: async (id) => {
197
+ const snapshot = await this._snapshotStore.loadSnapshot(id);
198
+ if (!snapshot) {
199
+ return void 0;
200
+ }
201
+ return {
202
+ id,
203
+ payload: import_protocols.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").encodeAsAny(snapshot)
204
+ };
205
+ },
206
+ setObject: async (data) => {
207
+ const snapshot = import_protocols.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(data.payload);
208
+ await this._snapshotStore.saveSnapshot(snapshot);
209
+ }
210
+ });
211
+ }
212
+ get objectSync() {
213
+ return this._objectSync;
214
+ }
215
+ async open() {
216
+ await this._objectSync.open();
217
+ }
218
+ async close() {
219
+ await this._objectSync.close();
220
+ }
221
+ async load(id) {
222
+ const local = await this._snapshotStore.loadSnapshot(id);
223
+ if (local) {
224
+ return local;
225
+ }
226
+ const remote = await this._objectSync.download(id);
227
+ return import_protocols.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(remote.payload);
228
+ }
229
+ async store(snapshot) {
230
+ const id = await this._snapshotStore.saveSnapshot(snapshot);
231
+ return id;
232
+ }
233
+ };
234
+ SnapshotManager = __decorate([
235
+ (0, import_async.trackLeaks)("open", "close")
236
+ ], SnapshotManager);
237
+
238
+ // packages/core/echo/echo-pipeline/src/dbhost/snapshot-store.ts
239
+ var import_crypto = require("@dxos/crypto");
240
+ var import_protocols2 = require("@dxos/protocols");
241
+ var SnapshotStore = class {
242
+ // prettier-ignore
243
+ constructor(_directory) {
244
+ this._directory = _directory;
245
+ }
246
+ async saveSnapshot(snapshot) {
247
+ const encoded = import_protocols2.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").encode(snapshot);
248
+ const key = await import_crypto.subtleCrypto.digest("SHA-256", encoded);
249
+ const keyString = Buffer.from(key).toString("hex");
250
+ const file = await this._directory.getOrCreateFile(keyString);
251
+ try {
252
+ await file.write(0, Buffer.from(encoded));
253
+ } finally {
254
+ await file.close();
255
+ }
256
+ return keyString;
257
+ }
258
+ async loadSnapshot(key) {
259
+ const file = await this._directory.getOrCreateFile(key);
260
+ try {
261
+ const { size } = await file.stat();
262
+ if (size === 0) {
263
+ return void 0;
264
+ }
265
+ const buffer = await file.read(0, size);
266
+ return import_protocols2.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(buffer);
267
+ } finally {
268
+ await file.close();
269
+ }
270
+ }
271
+ };
272
+
273
+ // packages/core/echo/echo-pipeline/src/dbhost/data-service.ts
274
+ var import_node_assert2 = __toESM(require("node:assert"));
275
+ var import_debug = require("@dxos/debug");
276
+ var import_keys = require("@dxos/keys");
277
+ var import_log2 = require("@dxos/log");
278
+ var import_util2 = require("@dxos/util");
279
+ var DataServiceSubscriptions = class {
280
+ constructor() {
281
+ this._spaces = new import_util2.ComplexMap(import_keys.PublicKey.hash);
282
+ }
283
+ clear() {
284
+ this._spaces.clear();
285
+ }
286
+ registerSpace(spaceKey, host) {
287
+ (0, import_log2.log)("Registering space", {
288
+ spaceKey
289
+ }, {
290
+ file: "data-service.ts",
291
+ line: 31,
292
+ scope: this,
293
+ callSite: (f, a) => f(...a)
294
+ });
295
+ this._spaces.set(spaceKey, host);
296
+ }
297
+ unregisterSpace(spaceKey) {
298
+ (0, import_log2.log)("Unregistering space", {
299
+ spaceKey
300
+ }, {
301
+ file: "data-service.ts",
302
+ line: 36,
303
+ scope: this,
304
+ callSite: (f, a) => f(...a)
305
+ });
306
+ this._spaces.delete(spaceKey);
307
+ }
308
+ getDataService(spaceKey) {
309
+ return this._spaces.get(spaceKey);
310
+ }
311
+ };
312
+ var DataServiceImpl = class {
313
+ constructor(_subscriptions) {
314
+ this._subscriptions = _subscriptions;
315
+ }
316
+ subscribe(request) {
317
+ var _a;
318
+ (0, import_node_assert2.default)(request.spaceKey);
319
+ const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
320
+ return host.subscribe();
321
+ }
322
+ write(request) {
323
+ var _a;
324
+ (0, import_node_assert2.default)(request.spaceKey);
325
+ (0, import_node_assert2.default)(request.batch);
326
+ const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
327
+ return host.write(request);
328
+ }
329
+ };
330
+
331
+ // packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
332
+ var import_node_assert3 = __toESM(require("node:assert"));
333
+ var import_async2 = require("@dxos/async");
334
+ var import_log3 = require("@dxos/log");
335
+ var import_protocols3 = require("@dxos/protocols");
336
+ var __decorate2 = function(decorators, target, key, desc) {
337
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
338
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
339
+ r = Reflect.decorate(decorators, target, key, desc);
340
+ else
341
+ for (var i = decorators.length - 1; i >= 0; i--)
342
+ if (d = decorators[i])
343
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
344
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
345
+ };
346
+ var STORAGE_VERSION = 1;
347
+ var emptyEchoMetadata = () => ({
348
+ version: STORAGE_VERSION,
349
+ spaces: [],
350
+ created: new Date(),
351
+ updated: new Date()
352
+ });
353
+ var MetadataStore = class {
354
+ // prettier-ignore
355
+ constructor(_directory) {
356
+ this._directory = _directory;
357
+ this._metadata = emptyEchoMetadata();
358
+ }
359
+ get version() {
360
+ var _a;
361
+ return (_a = this._metadata.version) != null ? _a : 0;
362
+ }
363
+ /**
364
+ * Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
365
+ * addSpaceFeed functions.
366
+ */
367
+ get spaces() {
368
+ var _a;
369
+ return (_a = this._metadata.spaces) != null ? _a : [];
370
+ }
371
+ /**
372
+ * Loads metadata from persistent storage.
373
+ */
374
+ async load() {
375
+ const file = this._directory.getOrCreateFile("EchoMetadata");
376
+ try {
377
+ const { size: fileLength } = await file.stat();
378
+ if (fileLength < 4) {
379
+ return;
380
+ }
381
+ const dataSize = fromBytesInt32(await file.read(0, 4));
382
+ (0, import_log3.log)("loaded", {
383
+ size: dataSize
384
+ }, {
385
+ file: "metadata-store.ts",
386
+ line: 68,
387
+ scope: this,
388
+ callSite: (f, a) => f(...a)
389
+ });
390
+ {
391
+ if (fileLength < dataSize + 4) {
392
+ throw new Error("Metadata storage is corrupted");
393
+ }
394
+ }
395
+ const data = await file.read(4, dataSize);
396
+ this._metadata = import_protocols3.schema.getCodecForType("dxos.echo.metadata.EchoMetadata").decode(data);
397
+ } catch (err) {
398
+ import_log3.log.error("failed to load metadata", {
399
+ err
400
+ }, {
401
+ file: "metadata-store.ts",
402
+ line: 80,
403
+ scope: this,
404
+ callSite: (f, a) => f(...a)
405
+ });
406
+ this._metadata = emptyEchoMetadata();
407
+ } finally {
408
+ await file.close();
409
+ }
410
+ }
411
+ async _save() {
412
+ var _a;
413
+ const data = {
414
+ ...this._metadata,
415
+ version: STORAGE_VERSION,
416
+ created: (_a = this._metadata.created) != null ? _a : new Date(),
417
+ updated: new Date()
418
+ };
419
+ const file = this._directory.getOrCreateFile("EchoMetadata");
420
+ try {
421
+ const encoded = Buffer.from(import_protocols3.schema.getCodecForType("dxos.echo.metadata.EchoMetadata").encode(data));
422
+ await file.write(0, toBytesInt32(encoded.length));
423
+ (0, import_log3.log)("saved", {
424
+ size: encoded.length
425
+ }, {
426
+ file: "metadata-store.ts",
427
+ line: 103,
428
+ scope: this,
429
+ callSite: (f, a) => f(...a)
430
+ });
431
+ await file.write(4, encoded);
432
+ } finally {
433
+ await file.close();
434
+ }
435
+ }
436
+ /**
437
+ * Clears storage - doesn't work for now.
438
+ */
439
+ async clear() {
440
+ (0, import_log3.log)("clearing all metadata", {}, {
441
+ file: "metadata-store.ts",
442
+ line: 116,
443
+ scope: this,
444
+ callSite: (f, a) => f(...a)
445
+ });
446
+ await this._directory.delete();
447
+ }
448
+ getIdentityRecord() {
449
+ return this._metadata.identity;
450
+ }
451
+ async setIdentityRecord(record) {
452
+ (0, import_node_assert3.default)(!this._metadata.identity, "Cannot overwrite existing identity in metadata");
453
+ this._metadata.identity = record;
454
+ await this._save();
455
+ }
456
+ async addSpace(record) {
457
+ var _a, _b, _c;
458
+ (0, import_node_assert3.default)(!((_a = this._metadata.spaces) != null ? _a : []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata");
459
+ ((_c = (_b = this._metadata).spaces) != null ? _c : _b.spaces = []).push(record);
460
+ await this._save();
461
+ }
462
+ async setSpaceLatestTimeframe(spaceKey, timeframe) {
463
+ const space = this.spaces.find((space2) => space2.key === spaceKey);
464
+ (0, import_node_assert3.default)(space, "Space not found");
465
+ space.dataTimeframe = timeframe;
466
+ await this._save();
467
+ }
468
+ async setSpaceSnapshot(spaceKey, snapshot) {
469
+ const space = this.spaces.find((space2) => space2.key === spaceKey);
470
+ (0, import_node_assert3.default)(space, "Space not found");
471
+ space.snapshot = snapshot;
472
+ await this._save();
473
+ }
474
+ async setWritableFeedKeys(spaceKey, controlFeedKey, dataFeedKey) {
475
+ const space = this.spaces.find((space2) => space2.key === spaceKey);
476
+ (0, import_node_assert3.default)(space, "Space not found");
477
+ space.controlFeedKey = controlFeedKey;
478
+ space.dataFeedKey = dataFeedKey;
479
+ await this._save();
480
+ }
481
+ };
482
+ __decorate2([
483
+ import_async2.synchronized
484
+ ], MetadataStore.prototype, "load", null);
485
+ __decorate2([
486
+ import_async2.synchronized
487
+ ], MetadataStore.prototype, "_save", null);
488
+ var toBytesInt32 = (num) => {
489
+ const buf = Buffer.alloc(4);
490
+ buf.writeInt32LE(num, 0);
491
+ return buf;
492
+ };
493
+ var fromBytesInt32 = (buf) => buf.readInt32LE(0);
494
+
495
+ // packages/core/echo/echo-pipeline/src/space/auth.ts
496
+ var import_node_assert4 = __toESM(require("node:assert"));
497
+ var import_async3 = require("@dxos/async");
498
+ var import_context = require("@dxos/context");
499
+ var import_crypto2 = require("@dxos/crypto");
500
+ var import_log4 = require("@dxos/log");
501
+ var import_protocols4 = require("@dxos/protocols");
502
+ var import_teleport = require("@dxos/teleport");
503
+ var AuthExtension = class extends import_teleport.RpcExtension {
504
+ constructor(_authParams) {
505
+ super({
506
+ requested: {
507
+ AuthService: import_protocols4.schema.getService("dxos.mesh.teleport.auth.AuthService")
508
+ },
509
+ exposed: {
510
+ AuthService: import_protocols4.schema.getService("dxos.mesh.teleport.auth.AuthService")
511
+ },
512
+ timeout: 60 * 1e3
513
+ // Long timeout because auth can wait for sync in certain cases.
514
+ });
515
+ this._authParams = _authParams;
516
+ this._ctx = new import_context.Context({
517
+ onError: (err) => {
518
+ import_log4.log.catch(err, {}, {
519
+ file: "auth.ts",
520
+ line: 29,
521
+ scope: this,
522
+ callSite: (f, a) => f(...a)
523
+ });
524
+ }
525
+ });
526
+ }
527
+ async getHandlers() {
528
+ return {
529
+ AuthService: {
530
+ authenticate: async ({ challenge }) => {
531
+ try {
532
+ const credential = await this._authParams.provider(challenge);
533
+ if (!credential) {
534
+ throw new Error("auth rejected");
535
+ }
536
+ return {
537
+ credential
538
+ };
539
+ } catch (err) {
540
+ import_log4.log.error("failed to generate auth credentials", err, {
541
+ file: "auth.ts",
542
+ line: 56,
543
+ scope: this,
544
+ callSite: (f, a) => f(...a)
545
+ });
546
+ throw new Error("auth rejected");
547
+ }
548
+ }
549
+ }
550
+ };
551
+ }
552
+ async onOpen(context) {
553
+ await super.onOpen(context);
554
+ (0, import_async3.scheduleTask)(this._ctx, async () => {
555
+ try {
556
+ const challenge = (0, import_crypto2.randomBytes)(32);
557
+ const { credential } = await this.rpc.AuthService.authenticate({
558
+ challenge
559
+ });
560
+ (0, import_node_assert4.default)((credential == null ? void 0 : credential.length) > 0, "invalid credential");
561
+ const success = await this._authParams.verifier(challenge, credential);
562
+ (0, import_node_assert4.default)(success, "credential not verified");
563
+ (0, import_async3.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
564
+ } catch (err) {
565
+ import_log4.log.warn("auth failed", err, {
566
+ file: "auth.ts",
567
+ line: 75,
568
+ scope: this,
569
+ callSite: (f, a) => f(...a)
570
+ });
571
+ this.close();
572
+ this._authParams.onAuthFailure();
573
+ }
574
+ });
575
+ }
576
+ async onClose() {
577
+ await this._ctx.dispose();
578
+ await super.onClose();
579
+ }
580
+ };
581
+
582
+ // packages/core/echo/echo-pipeline/src/space/space.ts
583
+ var import_node_assert8 = __toESM(require("node:assert"));
584
+ var import_async6 = require("@dxos/async");
585
+ var import_log8 = require("@dxos/log");
586
+ var import_credentials3 = require("@dxos/protocols/proto/dxos/halo/credentials");
587
+ var import_util4 = require("@dxos/util");
588
+
589
+ // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
590
+ var import_node_assert7 = __toESM(require("node:assert"));
591
+ var import_async5 = require("@dxos/async");
592
+ var import_feed_store = require("@dxos/feed-store");
593
+ var import_keys2 = require("@dxos/keys");
594
+ var import_log6 = require("@dxos/log");
595
+ var import_timeframe2 = require("@dxos/timeframe");
596
+
597
+ // packages/core/echo/echo-pipeline/src/common/codec.ts
598
+ var import_hypercore = require("@dxos/hypercore");
599
+ var import_protocols5 = require("@dxos/protocols");
600
+ var codec = import_protocols5.schema.getCodecForType("dxos.echo.feed.FeedMessage");
601
+ var valueEncoding = (0, import_hypercore.createCodecEncoding)(codec);
602
+
603
+ // packages/core/echo/echo-pipeline/src/common/feeds.ts
604
+ var import_node_assert5 = __toESM(require("node:assert"));
605
+ var createMappedFeedWriter = (mapper, writer) => {
606
+ (0, import_node_assert5.default)(mapper);
607
+ (0, import_node_assert5.default)(writer);
608
+ return {
609
+ write: async (data) => await writer.write(await mapper(data))
610
+ };
611
+ };
612
+
613
+ // packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
614
+ var import_debug2 = __toESM(require("debug"));
615
+ var import_node_assert6 = __toESM(require("node:assert"));
616
+ var log5 = (0, import_debug2.default)("dxos:echo-db:message-selector");
617
+ var createMessageSelector = (timeframeClock) => {
618
+ return (messages) => {
619
+ for (let i = 0; i < messages.length; i++) {
620
+ const { data: { timeframe } } = messages[i];
621
+ (0, import_node_assert6.default)(timeframe);
622
+ if (!timeframeClock.hasGaps(timeframe)) {
623
+ return i;
624
+ }
625
+ }
626
+ log5("Skipping...");
627
+ };
628
+ };
629
+
630
+ // packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
631
+ var import_async4 = require("@dxos/async");
632
+ var import_debug3 = require("@dxos/debug");
633
+ var import_log5 = require("@dxos/log");
634
+ var import_timeframe = require("@dxos/timeframe");
635
+ var __decorate3 = function(decorators, target, key, desc) {
636
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
637
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
638
+ r = Reflect.decorate(decorators, target, key, desc);
639
+ else
640
+ for (var i = decorators.length - 1; i >= 0; i--)
641
+ if (d = decorators[i])
642
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
643
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
644
+ };
645
+ var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
646
+ feedKey,
647
+ index
648
+ }));
649
+ var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
650
+ feedKey,
651
+ index
652
+ ]));
653
+ var TimeframeClock = class {
654
+ // prettier-ignore
655
+ constructor(_timeframe = new import_timeframe.Timeframe()) {
656
+ this._timeframe = _timeframe;
657
+ this.update = new import_async4.Event();
658
+ }
659
+ get timeframe() {
660
+ return this._timeframe;
661
+ }
662
+ updateTimeframe(key, seq) {
663
+ this._timeframe = import_timeframe.Timeframe.merge(this._timeframe, new import_timeframe.Timeframe([
664
+ [
665
+ key,
666
+ seq
667
+ ]
668
+ ]));
669
+ this.update.emit(this._timeframe);
670
+ }
671
+ hasGaps(timeframe) {
672
+ const gaps = import_timeframe.Timeframe.dependencies(timeframe, this._timeframe);
673
+ return !gaps.isEmpty();
674
+ }
675
+ async waitUntilReached(target) {
676
+ import_log5.log.debug("waitUntilReached", {
677
+ target,
678
+ current: this._timeframe
679
+ }, {
680
+ file: "timeframe-clock.ts",
681
+ line: 45,
682
+ scope: this,
683
+ callSite: (f, a) => f(...a)
684
+ });
685
+ await this.update.waitForCondition(() => {
686
+ (0, import_log5.log)("check if reached", {
687
+ target,
688
+ current: this._timeframe,
689
+ deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
690
+ }, {
691
+ file: "timeframe-clock.ts",
692
+ line: 47,
693
+ scope: this,
694
+ callSite: (f, a) => f(...a)
695
+ });
696
+ return import_timeframe.Timeframe.dependencies(target, this._timeframe).isEmpty();
697
+ });
698
+ }
699
+ };
700
+ __decorate3([
701
+ (0, import_debug3.timed)(5e3)
702
+ ], TimeframeClock.prototype, "waitUntilReached", null);
703
+
704
+ // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
705
+ var PipelineState = class {
706
+ // prettier-ignore
707
+ constructor(_iterator, _timeframeClock) {
708
+ this._iterator = _iterator;
709
+ this._timeframeClock = _timeframeClock;
710
+ this.timeframeUpdate = this._timeframeClock.update;
711
+ }
712
+ /**
713
+ * Latest theoretical timeframe based on the last mutation in each feed.
714
+ * NOTE: This might never be reached if the mutation dependencies
715
+ */
716
+ get endTimeframe() {
717
+ return mapFeedIndexesToTimeframe(this._iterator.feeds.filter((feed) => feed.properties.length > 0).map((feed) => ({
718
+ feedKey: feed.key,
719
+ index: feed.properties.length - 1
720
+ })));
721
+ }
722
+ get timeframe() {
723
+ return this._timeframeClock.timeframe;
724
+ }
725
+ get targetTimeframe() {
726
+ return this._targetTimeframe ? import_timeframe2.Timeframe.merge(this.endTimeframe, this._targetTimeframe) : this.endTimeframe;
727
+ }
728
+ async waitUntilTimeframe(target) {
729
+ await this._timeframeClock.waitUntilReached(target);
730
+ }
731
+ setTargetTimeframe(target) {
732
+ this._targetTimeframe = target;
733
+ }
734
+ /**
735
+ * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
736
+ *
737
+ * This function will resolve immediately if the pipeline is stalled.
738
+ *
739
+ * @param timeout Timeout in milliseconds to specify the maximum wait time.
740
+ */
741
+ async waitUntilReachedTargetTimeframe({ timeout } = {}) {
742
+ var _a;
743
+ (0, import_log6.log)("waitUntilReachedTargetTimeframe", {
744
+ timeout,
745
+ current: this.timeframe,
746
+ target: this.targetTimeframe
747
+ }, {
748
+ file: "pipeline.ts",
749
+ line: 82,
750
+ scope: this,
751
+ callSite: (f, a) => f(...a)
752
+ });
753
+ (_a = this._reachedTargetPromise) != null ? _a : this._reachedTargetPromise = Promise.race([
754
+ this._timeframeClock.update.waitForCondition(() => {
755
+ return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
756
+ }),
757
+ this._iterator.stalled.discardParameter().waitForCount(1)
758
+ ]);
759
+ let done = false;
760
+ if (timeout) {
761
+ return Promise.race([
762
+ this._reachedTargetPromise.then(() => {
763
+ done = true;
764
+ }),
765
+ (0, import_async5.sleep)(timeout).then(() => {
766
+ if (done) {
767
+ return;
768
+ }
769
+ import_log6.log.warn("waitUntilReachedTargetTimeframe timed out", {
770
+ timeout,
771
+ current: this.timeframe,
772
+ target: this.targetTimeframe,
773
+ dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
774
+ }, {
775
+ file: "pipeline.ts",
776
+ line: 107,
777
+ scope: this,
778
+ callSite: (f, a) => f(...a)
779
+ });
780
+ })
781
+ ]);
782
+ } else {
783
+ return this._reachedTargetPromise;
784
+ }
785
+ }
786
+ };
787
+ var Pipeline = class {
788
+ // prettier-ignore
789
+ constructor(_initialTimeframe = new import_timeframe2.Timeframe()) {
790
+ this._initialTimeframe = _initialTimeframe;
791
+ this._timeframeClock = new TimeframeClock(this._initialTimeframe);
792
+ this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
793
+ start: mapTimeframeToFeedIndexes(this._initialTimeframe),
794
+ stallTimeout: 1e3
795
+ });
796
+ this._state = new PipelineState(this._feedSetIterator, this._timeframeClock);
797
+ this._processingTrigger = new import_async5.Trigger().wake();
798
+ this._isOpen = false;
799
+ this._feedSetIterator.stalled.on((iterator) => {
800
+ import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
801
+ file: "pipeline.ts",
802
+ line: 183,
803
+ scope: this,
804
+ callSite: (f, a) => f(...a)
805
+ });
806
+ });
807
+ }
808
+ get state() {
809
+ return this._state;
810
+ }
811
+ get writer() {
812
+ (0, import_node_assert7.default)(this._writer, "Writer not set.");
813
+ return this._writer;
814
+ }
815
+ getFeeds() {
816
+ return this._feedSetIterator.feeds;
817
+ }
818
+ async addFeed(feed) {
819
+ await this._feedSetIterator.addFeed(feed);
820
+ }
821
+ setWriteFeed(feed) {
822
+ (0, import_node_assert7.default)(!this._writer, "Writer already set.");
823
+ (0, import_node_assert7.default)(feed.properties.writable, "Feed must be writable.");
824
+ this._writer = createMappedFeedWriter((payload) => ({
825
+ timeframe: this._timeframeClock.timeframe,
826
+ payload
827
+ }), feed.createFeedWriter());
828
+ }
829
+ async start() {
830
+ (0, import_log6.log)("starting...", {}, {
831
+ file: "pipeline.ts",
832
+ line: 218,
833
+ scope: this,
834
+ callSite: (f, a) => f(...a)
835
+ });
836
+ await this._feedSetIterator.open();
837
+ (0, import_log6.log)("started", {}, {
838
+ file: "pipeline.ts",
839
+ line: 220,
840
+ scope: this,
841
+ callSite: (f, a) => f(...a)
842
+ });
843
+ }
844
+ async stop() {
845
+ (0, import_log6.log)("stopping...", {}, {
846
+ file: "pipeline.ts",
847
+ line: 224,
848
+ scope: this,
849
+ callSite: (f, a) => f(...a)
850
+ });
851
+ await this._feedSetIterator.close();
852
+ await this._processingTrigger.wait();
853
+ (0, import_log6.log)("stopped", {}, {
854
+ file: "pipeline.ts",
855
+ line: 227,
856
+ scope: this,
857
+ callSite: (f, a) => f(...a)
858
+ });
859
+ }
860
+ /**
861
+ * Starts to iterate over the ordered messages from the added feeds.
862
+ * Updates the timeframe clock after the message has bee processed.
863
+ */
864
+ async *consume() {
865
+ (0, import_node_assert7.default)(!this._isOpen, "Pipeline is already being consumed.");
866
+ this._isOpen = true;
867
+ for await (const block of this._feedSetIterator) {
868
+ this._processingTrigger.reset();
869
+ yield block;
870
+ this._processingTrigger.wake();
871
+ this._timeframeClock.updateTimeframe(import_keys2.PublicKey.from(block.feedKey), block.seq);
872
+ }
873
+ this._isOpen = false;
874
+ }
875
+ };
876
+
877
+ // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
878
+ var import_credentials = require("@dxos/credentials");
879
+ var import_keys3 = require("@dxos/keys");
880
+ var import_log7 = require("@dxos/log");
881
+ var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
882
+ var import_util3 = require("@dxos/util");
883
+ var ControlPipeline = class {
884
+ constructor({ spaceKey, genesisFeed, feedProvider, initialTimeframe }) {
885
+ this.onFeedAdmitted = new import_util3.Callback();
886
+ this._pipeline = new Pipeline(initialTimeframe);
887
+ void this._pipeline.addFeed(genesisFeed);
888
+ this._spaceStateMachine = new import_credentials.SpaceStateMachine(spaceKey);
889
+ this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
890
+ (0, import_log7.log)("feed admitted", {
891
+ key: info.key
892
+ }, {
893
+ file: "control-pipeline.ts",
894
+ line: 41,
895
+ scope: this,
896
+ callSite: (f, a) => f(...a)
897
+ });
898
+ if (info.assertion.designation === import_credentials2.AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
899
+ try {
900
+ const feed = await feedProvider(info.key);
901
+ await this._pipeline.addFeed(feed);
902
+ } catch (err) {
903
+ import_log7.log.catch(err, {}, {
904
+ file: "control-pipeline.ts",
905
+ line: 49,
906
+ scope: this,
907
+ callSite: (f, a) => f(...a)
908
+ });
909
+ }
910
+ }
911
+ await this.onFeedAdmitted.callIfSet(info);
912
+ });
913
+ this.onMemberAdmitted = this._spaceStateMachine.onMemberAdmitted;
914
+ this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
915
+ }
916
+ get spaceState() {
917
+ return this._spaceStateMachine;
918
+ }
919
+ get pipeline() {
920
+ return this._pipeline;
921
+ }
922
+ setWriteFeed(feed) {
923
+ this._pipeline.setWriteFeed(feed);
924
+ }
925
+ async start() {
926
+ (0, import_log7.log)("starting...", {}, {
927
+ file: "control-pipeline.ts",
928
+ line: 73,
929
+ scope: this,
930
+ callSite: (f, a) => f(...a)
931
+ });
932
+ setTimeout(async () => {
933
+ for await (const msg of this._pipeline.consume()) {
934
+ try {
935
+ (0, import_log7.log)("processing", {
936
+ key: msg.feedKey,
937
+ seq: msg.seq
938
+ }, {
939
+ file: "control-pipeline.ts",
940
+ line: 78,
941
+ scope: this,
942
+ callSite: (f, a) => f(...a)
943
+ });
944
+ if (msg.data.payload.credential) {
945
+ const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, import_keys3.PublicKey.from(msg.feedKey));
946
+ if (!result) {
947
+ import_log7.log.warn("processing failed", {
948
+ msg
949
+ }, {
950
+ file: "control-pipeline.ts",
951
+ line: 85,
952
+ scope: this,
953
+ callSite: (f, a) => f(...a)
954
+ });
955
+ }
956
+ }
957
+ } catch (err) {
958
+ import_log7.log.catch(err, {}, {
959
+ file: "control-pipeline.ts",
960
+ line: 89,
961
+ scope: this,
962
+ callSite: (f, a) => f(...a)
963
+ });
964
+ }
965
+ }
966
+ });
967
+ await this._pipeline.start();
968
+ (0, import_log7.log)("started", {}, {
969
+ file: "control-pipeline.ts",
970
+ line: 95,
971
+ scope: this,
972
+ callSite: (f, a) => f(...a)
973
+ });
974
+ }
975
+ async stop() {
976
+ (0, import_log7.log)("stopping...", {}, {
977
+ file: "control-pipeline.ts",
978
+ line: 99,
979
+ scope: this,
980
+ callSite: (f, a) => f(...a)
981
+ });
982
+ await this._pipeline.stop();
983
+ (0, import_log7.log)("stopped", {}, {
984
+ file: "control-pipeline.ts",
985
+ line: 101,
986
+ scope: this,
987
+ callSite: (f, a) => f(...a)
988
+ });
989
+ }
990
+ };
991
+
992
+ // packages/core/echo/echo-pipeline/src/space/space.ts
993
+ var __decorate4 = function(decorators, target, key, desc) {
994
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
995
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
996
+ r = Reflect.decorate(decorators, target, key, desc);
997
+ else
998
+ for (var i = decorators.length - 1; i >= 0; i--)
999
+ if (d = decorators[i])
1000
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1001
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1002
+ };
1003
+ var Space = class Space2 {
1004
+ constructor({ spaceKey, protocol, genesisFeed, feedProvider }) {
1005
+ this.onCredentialProcessed = new import_util4.Callback();
1006
+ this.stateUpdate = new import_async6.Event();
1007
+ this._isOpen = false;
1008
+ (0, import_node_assert8.default)(spaceKey && feedProvider);
1009
+ this._key = spaceKey;
1010
+ this._genesisFeedKey = genesisFeed.key;
1011
+ this._feedProvider = feedProvider;
1012
+ this._controlPipeline = new ControlPipeline({
1013
+ spaceKey,
1014
+ genesisFeed,
1015
+ feedProvider
1016
+ });
1017
+ this._controlPipeline.onFeedAdmitted.set(async (info) => {
1018
+ if (info.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
1019
+ if (this._dataPipeline) {
1020
+ await this._dataPipeline.addFeed(await feedProvider(info.key));
1021
+ }
1022
+ }
1023
+ if (!info.key.equals(genesisFeed.key)) {
1024
+ this.protocol.addFeed(await feedProvider(info.key));
1025
+ }
1026
+ });
1027
+ this._controlPipeline.onCredentialProcessed.set(async (credential) => {
1028
+ await this.onCredentialProcessed.callIfSet(credential);
1029
+ (0, import_log8.log)("onCredentialProcessed", {
1030
+ credential
1031
+ }, {
1032
+ file: "space.ts",
1033
+ line: 74,
1034
+ scope: this,
1035
+ callSite: (f, a) => f(...a)
1036
+ });
1037
+ this.stateUpdate.emit();
1038
+ });
1039
+ this.protocol = protocol;
1040
+ this.protocol.addFeed(genesisFeed);
1041
+ }
1042
+ get key() {
1043
+ return this._key;
1044
+ }
1045
+ get isOpen() {
1046
+ return this._isOpen;
1047
+ }
1048
+ get genesisFeedKey() {
1049
+ return this._genesisFeedKey;
1050
+ }
1051
+ get controlFeedKey() {
1052
+ var _a;
1053
+ return (_a = this._controlFeed) == null ? void 0 : _a.key;
1054
+ }
1055
+ get dataFeedKey() {
1056
+ var _a;
1057
+ return (_a = this._dataFeed) == null ? void 0 : _a.key;
1058
+ }
1059
+ get spaceState() {
1060
+ return this._controlPipeline.spaceState;
1061
+ }
1062
+ /**
1063
+ * @test-only
1064
+ */
1065
+ get controlPipeline() {
1066
+ return this._controlPipeline.pipeline;
1067
+ }
1068
+ setControlFeed(feed) {
1069
+ (0, import_node_assert8.default)(!this._controlFeed, "Control feed already set.");
1070
+ this._controlFeed = feed;
1071
+ this._controlPipeline.setWriteFeed(feed);
1072
+ return this;
1073
+ }
1074
+ setDataFeed(feed) {
1075
+ var _a;
1076
+ (0, import_node_assert8.default)(!this._dataFeed, "Data feed already set.");
1077
+ this._dataFeed = feed;
1078
+ (_a = this._dataPipeline) == null ? void 0 : _a.setWriteFeed(feed);
1079
+ return this;
1080
+ }
1081
+ /**
1082
+ * Use for diagnostics.
1083
+ */
1084
+ getControlFeeds() {
1085
+ return Array.from(this._controlPipeline.spaceState.feeds.values());
1086
+ }
1087
+ /**
1088
+ * Use for diagnostics.
1089
+ */
1090
+ // getDataFeeds(): FeedInfo[] {
1091
+ // return this._dataPipeline?.getFeeds();
1092
+ // }
1093
+ async open() {
1094
+ (0, import_log8.log)("opening...", {}, {
1095
+ file: "space.ts",
1096
+ line: 145,
1097
+ scope: this,
1098
+ callSite: (f, a) => f(...a)
1099
+ });
1100
+ if (this._isOpen) {
1101
+ return;
1102
+ }
1103
+ await this._controlPipeline.start();
1104
+ await this.protocol.start();
1105
+ this._isOpen = true;
1106
+ (0, import_log8.log)("opened", {}, {
1107
+ file: "space.ts",
1108
+ line: 155,
1109
+ scope: this,
1110
+ callSite: (f, a) => f(...a)
1111
+ });
1112
+ }
1113
+ async close() {
1114
+ var _a;
1115
+ (0, import_log8.log)("closing...", {
1116
+ key: this._key
1117
+ }, {
1118
+ file: "space.ts",
1119
+ line: 160,
1120
+ scope: this,
1121
+ callSite: (f, a) => f(...a)
1122
+ });
1123
+ if (!this._isOpen) {
1124
+ return;
1125
+ }
1126
+ await this.protocol.stop();
1127
+ await ((_a = this._dataPipelineController) == null ? void 0 : _a.close());
1128
+ await this._controlPipeline.stop();
1129
+ this._isOpen = false;
1130
+ (0, import_log8.log)("closed", {}, {
1131
+ file: "space.ts",
1132
+ line: 171,
1133
+ scope: this,
1134
+ callSite: (f, a) => f(...a)
1135
+ });
1136
+ }
1137
+ async initDataPipeline(controller) {
1138
+ (0, import_node_assert8.default)(this._isOpen, "Space must be open to initialize data pipeline.");
1139
+ (0, import_node_assert8.default)(!this._dataPipelineController, "Data pipeline already initialized.");
1140
+ (0, import_node_assert8.default)(!this._dataPipeline, "Data pipeline already initialized.");
1141
+ this._dataPipelineController = controller;
1142
+ await this._dataPipelineController.open({
1143
+ openPipeline: async (start) => {
1144
+ (0, import_node_assert8.default)(!this._dataPipeline, "Data pipeline already initialized.");
1145
+ this._dataPipeline = new Pipeline(start);
1146
+ if (this._dataFeed) {
1147
+ this._dataPipeline.setWriteFeed(this._dataFeed);
1148
+ }
1149
+ for (const feed of this._controlPipeline.spaceState.feeds.values()) {
1150
+ if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
1151
+ await this._dataPipeline.addFeed(await this._feedProvider(feed.key));
1152
+ }
1153
+ }
1154
+ await this._dataPipeline.start();
1155
+ return this._dataPipeline;
1156
+ }
1157
+ });
1158
+ }
1159
+ };
1160
+ __decorate4([
1161
+ import_log8.logInfo
1162
+ ], Space.prototype, "key", null);
1163
+ __decorate4([
1164
+ import_async6.synchronized
1165
+ ], Space.prototype, "open", null);
1166
+ __decorate4([
1167
+ import_async6.synchronized
1168
+ ], Space.prototype, "close", null);
1169
+ Space = __decorate4([
1170
+ (0, import_async6.trackLeaks)("open", "close")
1171
+ ], Space);
1172
+
1173
+ // packages/core/echo/echo-pipeline/src/space/space-manager.ts
1174
+ var import_async7 = require("@dxos/async");
1175
+ var import_debug4 = require("@dxos/debug");
1176
+ var import_keys5 = require("@dxos/keys");
1177
+ var import_log10 = require("@dxos/log");
1178
+ var import_util6 = require("@dxos/util");
1179
+
1180
+ // packages/core/echo/echo-pipeline/src/space/space-protocol.ts
1181
+ var import_crypto3 = require("@dxos/crypto");
1182
+ var import_keys4 = require("@dxos/keys");
1183
+ var import_log9 = require("@dxos/log");
1184
+ var import_network_manager = require("@dxos/network-manager");
1185
+ var import_teleport2 = require("@dxos/teleport");
1186
+ var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
1187
+ var import_util5 = require("@dxos/util");
1188
+ var __decorate5 = function(decorators, target, key, desc) {
1189
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1190
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1191
+ r = Reflect.decorate(decorators, target, key, desc);
1192
+ else
1193
+ for (var i = decorators.length - 1; i >= 0; i--)
1194
+ if (d = decorators[i])
1195
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1196
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1197
+ };
1198
+ var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
1199
+ var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
1200
+ var SpaceProtocol = class {
1201
+ constructor({ topic, swarmIdentity, networkManager, onSessionAuth }) {
1202
+ this._feeds = /* @__PURE__ */ new Set();
1203
+ this._sessions = new import_util5.ComplexMap(import_keys4.PublicKey.hash);
1204
+ this._networkManager = networkManager;
1205
+ this._swarmIdentity = swarmIdentity;
1206
+ this._onSessionAuth = onSessionAuth;
1207
+ this._topic = import_keys4.PublicKey.from((0, import_crypto3.discoveryKey)((0, import_crypto3.sha256)(topic.toHex())));
1208
+ }
1209
+ get sessions() {
1210
+ return this._sessions;
1211
+ }
1212
+ get feeds() {
1213
+ return this._feeds;
1214
+ }
1215
+ get _ownPeerKey() {
1216
+ return this._swarmIdentity.peerKey;
1217
+ }
1218
+ // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
1219
+ addFeed(feed) {
1220
+ (0, import_log9.log)("addFeed", {
1221
+ key: feed.key
1222
+ }, {
1223
+ file: "space-protocol.ts",
1224
+ line: 84,
1225
+ scope: this,
1226
+ callSite: (f, a) => f(...a)
1227
+ });
1228
+ this._feeds.add(feed);
1229
+ for (const session of this._sessions.values()) {
1230
+ session.replicator.addFeed(feed);
1231
+ }
1232
+ }
1233
+ async start() {
1234
+ if (this._connection) {
1235
+ return;
1236
+ }
1237
+ const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(""));
1238
+ const topologyConfig = {
1239
+ originateConnections: 4,
1240
+ maxPeers: 10,
1241
+ sampleSize: 20
1242
+ };
1243
+ (0, import_log9.log)("starting...", {}, {
1244
+ file: "space-protocol.ts",
1245
+ line: 107,
1246
+ scope: this,
1247
+ callSite: (f, a) => f(...a)
1248
+ });
1249
+ this._connection = await this._networkManager.joinSwarm({
1250
+ protocolProvider: this._createProtocolProvider(credentials),
1251
+ peerId: this._swarmIdentity.peerKey,
1252
+ topic: this._topic,
1253
+ topology: new import_network_manager.MMSTTopology(topologyConfig),
1254
+ label: `Protocol swarm: ${this._topic}`
1255
+ });
1256
+ (0, import_log9.log)("started", {}, {
1257
+ file: "space-protocol.ts",
1258
+ line: 116,
1259
+ scope: this,
1260
+ callSite: (f, a) => f(...a)
1261
+ });
1262
+ }
1263
+ async stop() {
1264
+ if (this._connection) {
1265
+ (0, import_log9.log)("stopping...", {}, {
1266
+ file: "space-protocol.ts",
1267
+ line: 121,
1268
+ scope: this,
1269
+ callSite: (f, a) => f(...a)
1270
+ });
1271
+ await this._connection.close();
1272
+ (0, import_log9.log)("stopped", {}, {
1273
+ file: "space-protocol.ts",
1274
+ line: 123,
1275
+ scope: this,
1276
+ callSite: (f, a) => f(...a)
1277
+ });
1278
+ }
1279
+ }
1280
+ _createProtocolProvider(credentials) {
1281
+ return (wireParams) => {
1282
+ const session = new SpaceProtocolSession({
1283
+ wireParams,
1284
+ swarmIdentity: this._swarmIdentity,
1285
+ onSessionAuth: this._onSessionAuth
1286
+ });
1287
+ this._sessions.set(wireParams.remotePeerId, session);
1288
+ for (const feed of this._feeds) {
1289
+ session.replicator.addFeed(feed);
1290
+ }
1291
+ return session;
1292
+ };
1293
+ }
1294
+ };
1295
+ __decorate5([
1296
+ import_log9.logInfo
1297
+ ], SpaceProtocol.prototype, "_topic", void 0);
1298
+ __decorate5([
1299
+ import_log9.logInfo
1300
+ ], SpaceProtocol.prototype, "_ownPeerKey", null);
1301
+ var AuthStatus;
1302
+ (function(AuthStatus2) {
1303
+ AuthStatus2["INITIAL"] = "INITIAL";
1304
+ AuthStatus2["SUCCESS"] = "SUCCESS";
1305
+ AuthStatus2["FAILURE"] = "FAILURE";
1306
+ })(AuthStatus || (AuthStatus = {}));
1307
+ var SpaceProtocolSession = class {
1308
+ // TODO(dmaretskyi): Allow to pass in extra extensions.
1309
+ constructor({ wireParams, swarmIdentity, onSessionAuth }) {
1310
+ // TODO(dmaretskyi): Start with upload=false when switching it on the fly works.
1311
+ this.replicator = new import_teleport_extension_replicator.ReplicatorExtension().setOptions({
1312
+ upload: true
1313
+ });
1314
+ this._authStatus = AuthStatus.INITIAL;
1315
+ this._wireParams = wireParams;
1316
+ this._swarmIdentity = swarmIdentity;
1317
+ this._onSessionAuth = onSessionAuth;
1318
+ this._teleport = new import_teleport2.Teleport(wireParams);
1319
+ }
1320
+ get authStatus() {
1321
+ return this._authStatus;
1322
+ }
1323
+ get stream() {
1324
+ return this._teleport.stream;
1325
+ }
1326
+ async initialize() {
1327
+ await this._teleport.open();
1328
+ this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
1329
+ provider: this._swarmIdentity.credentialProvider,
1330
+ verifier: this._swarmIdentity.credentialAuthenticator,
1331
+ onAuthSuccess: () => {
1332
+ var _a;
1333
+ this._authStatus = AuthStatus.SUCCESS;
1334
+ (0, import_log9.log)("Peer authenticated", {}, {
1335
+ file: "space-protocol.ts",
1336
+ line: 206,
1337
+ scope: this,
1338
+ callSite: (f, a) => f(...a)
1339
+ });
1340
+ (_a = this._onSessionAuth) == null ? void 0 : _a.call(this, this._teleport);
1341
+ },
1342
+ onAuthFailure: () => {
1343
+ import_log9.log.warn("Auth failed", {}, {
1344
+ file: "space-protocol.ts",
1345
+ line: 211,
1346
+ scope: this,
1347
+ callSite: (f, a) => f(...a)
1348
+ });
1349
+ this._authStatus = AuthStatus.FAILURE;
1350
+ }
1351
+ }));
1352
+ this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
1353
+ }
1354
+ async destroy() {
1355
+ await this._teleport.close();
1356
+ }
1357
+ };
1358
+ __decorate5([
1359
+ import_log9.logInfo
1360
+ ], SpaceProtocolSession.prototype, "_wireParams", void 0);
1361
+ __decorate5([
1362
+ import_log9.logInfo
1363
+ ], SpaceProtocolSession.prototype, "authStatus", null);
1364
+
1365
+ // packages/core/echo/echo-pipeline/src/space/space-manager.ts
1366
+ var __decorate6 = function(decorators, target, key, desc) {
1367
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1368
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1369
+ r = Reflect.decorate(decorators, target, key, desc);
1370
+ else
1371
+ for (var i = decorators.length - 1; i >= 0; i--)
1372
+ if (d = decorators[i])
1373
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1374
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1375
+ };
1376
+ var SpaceManager = class SpaceManager2 {
1377
+ constructor({ feedStore, networkManager }) {
1378
+ this._spaces = new import_util6.ComplexMap(import_keys5.PublicKey.hash);
1379
+ this._feedStore = feedStore;
1380
+ this._networkManager = networkManager;
1381
+ }
1382
+ // TODO(burdon): Remove.
1383
+ get spaces() {
1384
+ return this._spaces;
1385
+ }
1386
+ async open() {
1387
+ }
1388
+ async close() {
1389
+ await Promise.all([
1390
+ ...this._spaces.values()
1391
+ ].map((space) => space.close()));
1392
+ }
1393
+ async constructSpace({ metadata, swarmIdentity, onNetworkConnection }) {
1394
+ var _a;
1395
+ (0, import_log10.log)("constructing space...", {
1396
+ spaceKey: metadata.genesisFeedKey
1397
+ }, {
1398
+ file: "space-manager.ts",
1399
+ line: 70,
1400
+ scope: this,
1401
+ callSite: (f, a) => f(...a)
1402
+ });
1403
+ const genesisFeed = await this._feedStore.openFeed((_a = metadata.genesisFeedKey) != null ? _a : (0, import_debug4.failUndefined)());
1404
+ const spaceKey = metadata.key;
1405
+ const protocol = new SpaceProtocol({
1406
+ topic: spaceKey,
1407
+ swarmIdentity,
1408
+ networkManager: this._networkManager,
1409
+ onSessionAuth: onNetworkConnection
1410
+ });
1411
+ const space = new Space({
1412
+ spaceKey,
1413
+ protocol,
1414
+ genesisFeed,
1415
+ feedProvider: (feedKey) => this._feedStore.openFeed(feedKey)
1416
+ });
1417
+ this._spaces.set(space.key, space);
1418
+ return space;
1419
+ }
1420
+ };
1421
+ __decorate6([
1422
+ import_async7.synchronized
1423
+ ], SpaceManager.prototype, "open", null);
1424
+ __decorate6([
1425
+ import_async7.synchronized
1426
+ ], SpaceManager.prototype, "close", null);
1427
+ SpaceManager = __decorate6([
1428
+ (0, import_async7.trackLeaks)("open", "close")
1429
+ ], SpaceManager);
1430
+
1431
+ // packages/core/echo/echo-pipeline/src/space/genesis.ts
1432
+ var import_credentials4 = require("@dxos/credentials");
1433
+ var import_debug5 = require("@dxos/debug");
1434
+ var import_credentials5 = require("@dxos/protocols/proto/dxos/halo/credentials");
1435
+
1436
+ // packages/core/echo/echo-pipeline/src/space/data-pipeline-controller.ts
1437
+ var import_node_assert9 = __toESM(require("node:assert"));
1438
+ var import_async8 = require("@dxos/async");
1439
+ var import_context2 = require("@dxos/context");
1440
+ var import_debug6 = require("@dxos/debug");
1441
+ var import_echo_db3 = require("@dxos/echo-db");
1442
+ var import_log11 = require("@dxos/log");
1443
+ var import_timeframe3 = require("@dxos/timeframe");
1444
+ var __decorate7 = function(decorators, target, key, desc) {
1445
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1446
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1447
+ r = Reflect.decorate(decorators, target, key, desc);
1448
+ else
1449
+ for (var i = decorators.length - 1; i >= 0; i--)
1450
+ if (d = decorators[i])
1451
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1452
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
1453
+ };
1454
+ var MESSAGES_PER_SNAPSHOT = 10;
1455
+ var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
1456
+ var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
1457
+ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1458
+ constructor(_params) {
1459
+ this._params = _params;
1460
+ this._ctx = new import_context2.Context();
1461
+ this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
1462
+ this.onTimeframeReached = new import_async8.Event();
1463
+ }
1464
+ get pipelineState() {
1465
+ var _a;
1466
+ return (_a = this._pipeline) == null ? void 0 : _a.state;
1467
+ }
1468
+ get snapshotTimeframe() {
1469
+ var _a;
1470
+ return (_a = this._snapshot) == null ? void 0 : _a.timeframe;
1471
+ }
1472
+ getStartTimeframe() {
1473
+ var _a;
1474
+ return snapshotTimeframeToStartingTimeframe((_a = this.snapshotTimeframe) != null ? _a : new import_timeframe3.Timeframe());
1475
+ }
1476
+ async open(spaceContext) {
1477
+ var _a, _b, _c, _d;
1478
+ this._spaceContext = spaceContext;
1479
+ if (this._params.snapshotId) {
1480
+ this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
1481
+ this._lastAutomaticSnapshotTimeframe = (_b = (_a = this._snapshot) == null ? void 0 : _a.timeframe) != null ? _b : new import_timeframe3.Timeframe();
1482
+ }
1483
+ this._pipeline = await this._spaceContext.openPipeline(this.getStartTimeframe());
1484
+ const feedWriter = createMappedFeedWriter((data) => ({
1485
+ data
1486
+ }), (_c = this._pipeline.writer) != null ? _c : (0, import_debug6.failUndefined)());
1487
+ this.databaseBackend = new DatabaseBackendHost(feedWriter, (_d = this._snapshot) == null ? void 0 : _d.database, {
1488
+ snapshots: true
1489
+ // TODO(burdon): Config.
1490
+ });
1491
+ this._itemManager = new import_echo_db3.ItemManager(this._params.modelFactory);
1492
+ await this.databaseBackend.open(this._itemManager, this._params.modelFactory);
1493
+ (0, import_async8.scheduleTask)(this._ctx, async () => {
1494
+ await this._consumePipeline();
1495
+ });
1496
+ this._createPeriodicSnapshots();
1497
+ }
1498
+ _createPeriodicSnapshots() {
1499
+ this.onTimeframeReached.debounce(TIMEFRAME_SAVE_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
1500
+ await this._saveLatestTimeframe();
1501
+ });
1502
+ this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
1503
+ var _a, _b;
1504
+ const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
1505
+ if (!latestTimeframe) {
1506
+ return;
1507
+ }
1508
+ if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
1509
+ const snapshot = await this._saveSnapshot();
1510
+ this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : (0, import_debug6.failUndefined)();
1511
+ (0, import_log11.log)("save", {
1512
+ snapshot
1513
+ }, {
1514
+ file: "data-pipeline-controller.ts",
1515
+ line: 151,
1516
+ scope: this,
1517
+ callSite: (f, a) => f(...a)
1518
+ });
1519
+ }
1520
+ });
1521
+ }
1522
+ async _saveSnapshot() {
1523
+ const snapshot = await this.createSnapshot();
1524
+ const snapshotKey = await this._params.snapshotManager.store(snapshot);
1525
+ await this._params.metadataStore.setSpaceSnapshot(this._params.spaceKey, snapshotKey);
1526
+ return snapshot;
1527
+ }
1528
+ async _saveLatestTimeframe() {
1529
+ var _a;
1530
+ const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
1531
+ if (latestTimeframe) {
1532
+ await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, latestTimeframe);
1533
+ }
1534
+ }
1535
+ async close() {
1536
+ var _a, _b, _c;
1537
+ try {
1538
+ await this._saveLatestTimeframe();
1539
+ await this._saveSnapshot();
1540
+ } catch (err) {
1541
+ import_log11.log.catch(err, {}, {
1542
+ file: "data-pipeline-controller.ts",
1543
+ line: 175,
1544
+ scope: this,
1545
+ callSite: (f, a) => f(...a)
1546
+ });
1547
+ }
1548
+ await this._ctx.dispose();
1549
+ await ((_a = this._pipeline) == null ? void 0 : _a.stop());
1550
+ await ((_b = this.databaseBackend) == null ? void 0 : _b.close());
1551
+ await ((_c = this._itemManager) == null ? void 0 : _c.destroy());
1552
+ await this._params.snapshotManager.close();
1553
+ }
1554
+ createSnapshot() {
1555
+ var _a, _b;
1556
+ return {
1557
+ spaceKey: this._params.spaceKey.asUint8Array(),
1558
+ timeframe: (_b = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe) != null ? _b : new import_timeframe3.Timeframe(),
1559
+ database: this.databaseBackend.createSnapshot()
1560
+ };
1561
+ }
1562
+ async _consumePipeline() {
1563
+ (0, import_node_assert9.default)(this._pipeline, "Pipeline is not initialized.");
1564
+ for await (const msg of this._pipeline.consume()) {
1565
+ const { feedKey, seq, data } = msg;
1566
+ (0, import_log11.log)("processing message", {
1567
+ msg
1568
+ }, {
1569
+ file: "data-pipeline-controller.ts",
1570
+ line: 196,
1571
+ scope: this,
1572
+ callSite: (f, a) => f(...a)
1573
+ });
1574
+ try {
1575
+ if (data.payload.data) {
1576
+ const feedInfo = this._params.feedInfoProvider(feedKey);
1577
+ if (!feedInfo) {
1578
+ import_log11.log.error("Could not find feed.", {
1579
+ feedKey
1580
+ }, {
1581
+ file: "data-pipeline-controller.ts",
1582
+ line: 202,
1583
+ scope: this,
1584
+ callSite: (f, a) => f(...a)
1585
+ });
1586
+ continue;
1587
+ }
1588
+ await this.databaseBackend.echoProcessor({
1589
+ data: data.payload.data.object,
1590
+ meta: {
1591
+ feedKey,
1592
+ seq,
1593
+ timeframe: data.timeframe,
1594
+ memberKey: feedInfo.assertion.identityKey
1595
+ }
1596
+ });
1597
+ this.onTimeframeReached.emit(data.timeframe);
1598
+ }
1599
+ } catch (err) {
1600
+ import_log11.log.catch(err, {}, {
1601
+ file: "data-pipeline-controller.ts",
1602
+ line: 218,
1603
+ scope: this,
1604
+ callSite: (f, a) => f(...a)
1605
+ });
1606
+ }
1607
+ }
1608
+ }
1609
+ async waitUntilTimeframe(timeframe) {
1610
+ (0, import_node_assert9.default)(this._pipeline, "Pipeline is not initialized.");
1611
+ await this._pipeline.state.waitUntilTimeframe(timeframe);
1612
+ }
1613
+ };
1614
+ DataPipelineControllerImpl = __decorate7([
1615
+ (0, import_async8.trackLeaks)("open", "close")
1616
+ ], DataPipelineControllerImpl);
1617
+ var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
1618
+ return snapshotTimeframe.map(([key, seq]) => [
1619
+ key,
1620
+ seq + 1
1621
+ ]);
1622
+ };
1623
+
1624
+ // packages/core/echo/echo-pipeline/src/testing/test-feed-builder.ts
1625
+ var import_testing = require("@dxos/feed-store/testing");
1626
+ var TestFeedBuilder = class extends import_testing.TestBuilder {
1627
+ constructor() {
1628
+ super({
1629
+ valueEncoding
1630
+ });
1631
+ }
1632
+ };
1633
+
1634
+ // packages/core/echo/echo-pipeline/src/testing/test-agent-builder.ts
1635
+ var MemoryNetworkManagerProvider = (signalContext) => () => new import_network_manager2.NetworkManager({
1636
+ signalManager: new import_messaging.MemorySignalManager(signalContext),
1637
+ transportFactory: import_network_manager2.MemoryTransportFactory
1638
+ });
1639
+ var WebsocketNetworkManagerProvider = (signalUrl) => () => new import_network_manager2.NetworkManager({
1640
+ signalManager: new import_messaging.WebsocketSignalManager([
1641
+ signalUrl
1642
+ ]),
1643
+ transportFactory: (0, import_network_manager2.createWebRTCTransportFactory)()
1644
+ });
1645
+ var TestAgentBuilder = class {
1646
+ constructor({ storage, networkManagerProvider } = {}) {
1647
+ this._agents = new import_util7.ComplexMap(import_keys6.PublicKey.hash);
1648
+ this._storage = storage != null ? storage : (0, import_random_access_storage.createStorage)({
1649
+ type: import_random_access_storage.StorageType.RAM
1650
+ });
1651
+ this._networkManagerProvider = networkManagerProvider != null ? networkManagerProvider : MemoryNetworkManagerProvider(new import_messaging.MemorySignalManagerContext());
1652
+ }
1653
+ async close() {
1654
+ return Promise.all(this.agents.map((agent) => agent.close()));
1655
+ }
1656
+ get agents() {
1657
+ return Array.from(this._agents.values());
1658
+ }
1659
+ getAgent(deviceKey) {
1660
+ return this._agents.get(deviceKey);
1661
+ }
1662
+ async createPeer() {
1663
+ const feedBuilder = new TestFeedBuilder().setStorage(this._storage, `agent-${this._agents.size}`);
1664
+ const identityKey = await feedBuilder.keyring.createKey();
1665
+ const deviceKey = await feedBuilder.keyring.createKey();
1666
+ const agent = new TestAgent(this._networkManagerProvider, feedBuilder, identityKey, deviceKey);
1667
+ this._agents.set(deviceKey, agent);
1668
+ return agent;
1669
+ }
1670
+ };
1671
+ var TestAgent = class {
1672
+ constructor(_networkManagerProvider, _feedBuilder, identityKey, deviceKey) {
1673
+ this._networkManagerProvider = _networkManagerProvider;
1674
+ this._feedBuilder = _feedBuilder;
1675
+ this.identityKey = identityKey;
1676
+ this.deviceKey = deviceKey;
1677
+ this._spaces = new import_util7.ComplexMap(import_keys6.PublicKey.hash);
1678
+ this.keyring = this._feedBuilder.keyring;
1679
+ this.feedStore = this._feedBuilder.createFeedStore();
1680
+ }
1681
+ async close() {
1682
+ return Promise.all([
1683
+ ...this.spaces.map((space) => space.close())
1684
+ ]);
1685
+ }
1686
+ get spaces() {
1687
+ return Array.from(this._spaces.values());
1688
+ }
1689
+ getSpace(spaceKey) {
1690
+ return this._spaces.get(spaceKey);
1691
+ }
1692
+ createSpaceManager() {
1693
+ return new SpaceManager({
1694
+ feedStore: this._feedBuilder.createFeedStore(),
1695
+ networkManager: this._networkManagerProvider()
1696
+ });
1697
+ }
1698
+ async createSpace(identityKey = this.identityKey, spaceKey, genesisKey) {
1699
+ if (!spaceKey) {
1700
+ spaceKey = await this.keyring.createKey();
1701
+ }
1702
+ const dataFeedKey = await this.keyring.createKey();
1703
+ const dataFeed = await this.feedStore.openFeed(dataFeedKey, {
1704
+ writable: true
1705
+ });
1706
+ const controlFeedKey = await this.keyring.createKey();
1707
+ const controlFeed = await this.feedStore.openFeed(controlFeedKey, {
1708
+ writable: true
1709
+ });
1710
+ const genesisFeed = genesisKey ? await this.feedStore.openFeed(genesisKey) : controlFeed;
1711
+ const snapshotManager = new SnapshotManager(new SnapshotStore((0, import_random_access_storage.createStorage)().createDirectory("snapshots")));
1712
+ const metadataStore = new MetadataStore((0, import_random_access_storage.createStorage)().createDirectory("metadata"));
1713
+ await metadataStore.addSpace({
1714
+ key: spaceKey
1715
+ });
1716
+ const dataPipelineController = new DataPipelineControllerImpl({
1717
+ modelFactory: new import_model_factory.ModelFactory().registerModel(import_document_model.DocumentModel),
1718
+ metadataStore,
1719
+ snapshotManager,
1720
+ memberKey: identityKey,
1721
+ spaceKey,
1722
+ feedInfoProvider: (feedKey) => space.spaceState.feeds.get(feedKey),
1723
+ snapshotId: void 0
1724
+ });
1725
+ const space = new Space({
1726
+ spaceKey,
1727
+ protocol: this.createSpaceProtocol(spaceKey),
1728
+ genesisFeed,
1729
+ feedProvider: (feedKey) => this.feedStore.openFeed(feedKey)
1730
+ }).setControlFeed(controlFeed).setDataFeed(dataFeed);
1731
+ await space.open();
1732
+ await space.initDataPipeline(dataPipelineController);
1733
+ this._spaces.set(spaceKey, space);
1734
+ return [
1735
+ space,
1736
+ dataPipelineController
1737
+ ];
1738
+ }
1739
+ createSpaceProtocol(topic, presence) {
1740
+ return new SpaceProtocol({
1741
+ topic,
1742
+ swarmIdentity: {
1743
+ peerKey: this.deviceKey,
1744
+ credentialProvider: MOCK_AUTH_PROVIDER,
1745
+ credentialAuthenticator: MOCK_AUTH_VERIFIER
1746
+ },
1747
+ networkManager: this._networkManagerProvider(),
1748
+ onSessionAuth: (session) => {
1749
+ session.addExtension("dxos.mesh.teleport.presence", (presence != null ? presence : this.createPresence()).createExtension({
1750
+ remotePeerId: session.remotePeerId
1751
+ }));
1752
+ }
1753
+ });
1754
+ }
1755
+ createPresence() {
1756
+ return new import_teleport_extension_presence.Presence({
1757
+ localPeerId: this.deviceKey,
1758
+ announceInterval: 30,
1759
+ offlineTimeout: 200,
1760
+ identityKey: this.identityKey
1761
+ });
1762
+ }
1763
+ };
1764
+
1765
+ // packages/core/echo/echo-pipeline/src/testing/util.ts
1766
+ var import_async9 = require("@dxos/async");
1767
+ var import_document_model2 = require("@dxos/document-model");
1768
+ var import_echo_db4 = require("@dxos/echo-db");
1769
+ var import_testing2 = require("@dxos/feed-store/testing");
1770
+ var import_keys7 = require("@dxos/keys");
1771
+ var import_model_factory2 = require("@dxos/model-factory");
1772
+ var import_timeframe4 = require("@dxos/timeframe");
1773
+ var createMemoryDatabase = async (modelFactory) => {
1774
+ const feed = new import_testing2.MockFeedWriter();
1775
+ const backend = new DatabaseBackendHost(feed, void 0, {
1776
+ snapshots: true
1777
+ });
1778
+ feed.written.on(([data, meta]) => backend.echoProcessor({
1779
+ data: data.object,
1780
+ meta: {
1781
+ ...meta,
1782
+ memberKey: import_keys7.PublicKey.random(),
1783
+ timeframe: new import_timeframe4.Timeframe([
1784
+ [
1785
+ meta.feedKey,
1786
+ meta.seq
1787
+ ]
1788
+ ])
1789
+ }
1790
+ }));
1791
+ const itemManager = new import_echo_db4.ItemManager(modelFactory);
1792
+ await backend.open(itemManager, new import_model_factory2.ModelFactory().registerModel(import_document_model2.DocumentModel));
1793
+ return {
1794
+ backend,
1795
+ itemManager
1796
+ };
1797
+ };
1798
+ var createRemoteDatabaseFromDataServiceHost = async (modelFactory, dataServiceHost) => {
1799
+ const dataServiceSubscriptions = new DataServiceSubscriptions();
1800
+ const dataService = new DataServiceImpl(dataServiceSubscriptions);
1801
+ const spaceKey = import_keys7.PublicKey.random();
1802
+ dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
1803
+ const backend = new import_echo_db4.DatabaseBackendProxy(dataService, spaceKey);
1804
+ const itemManager = new import_echo_db4.ItemManager(modelFactory);
1805
+ await backend.open(itemManager, new import_model_factory2.ModelFactory().registerModel(import_document_model2.DocumentModel));
1806
+ return {
1807
+ itemManager,
1808
+ backend
1809
+ };
1810
+ };
1811
+ var testLocalDatabase = async (create, check = create) => {
1812
+ var _a;
1813
+ const objectId = import_keys7.PublicKey.random().toHex();
1814
+ await ((_a = create.databaseBackend.getWriteStream()) == null ? void 0 : _a.write({
1815
+ object: {
1816
+ objectId,
1817
+ genesis: {
1818
+ modelType: import_document_model2.DocumentModel.meta.type
1819
+ }
1820
+ }
1821
+ }));
1822
+ await (0, import_async9.asyncTimeout)(check._itemManager.update.waitForCondition(() => check._itemManager.entities.has(objectId)), 500);
1823
+ };
1824
+ // Annotate the CommonJS export names for ESM import in node:
1825
+ 0 && (module.exports = {
1826
+ MemoryNetworkManagerProvider,
1827
+ TestAgent,
1828
+ TestAgentBuilder,
1829
+ TestFeedBuilder,
1830
+ WebsocketNetworkManagerProvider,
1831
+ createMemoryDatabase,
1832
+ createRemoteDatabaseFromDataServiceHost,
1833
+ testLocalDatabase
1834
+ });
1835
+ //# sourceMappingURL=index.cjs.map