@dxos/echo-pipeline 0.4.8-next.fff1521 → 0.4.9-main.1057b49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/lib/browser/{chunk-3FVT6KX6.mjs → chunk-RTEEJ723.mjs} +289 -1807
  2. package/dist/lib/browser/chunk-RTEEJ723.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +633 -14
  4. package/dist/lib/browser/index.mjs.map +4 -4
  5. package/dist/lib/browser/meta.json +1 -1
  6. package/dist/lib/browser/testing/index.mjs +4 -332
  7. package/dist/lib/browser/testing/index.mjs.map +4 -4
  8. package/dist/lib/node/chunk-7VZVCCNF.cjs +1948 -0
  9. package/dist/lib/node/chunk-7VZVCCNF.cjs.map +7 -0
  10. package/dist/lib/node/index.cjs +643 -34
  11. package/dist/lib/node/index.cjs.map +4 -4
  12. package/dist/lib/node/meta.json +1 -1
  13. package/dist/lib/node/testing/index.cjs +13 -338
  14. package/dist/lib/node/testing/index.cjs.map +4 -4
  15. package/dist/types/src/db-host/data-service.d.ts +3 -13
  16. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  17. package/dist/types/src/db-host/index.d.ts +0 -2
  18. package/dist/types/src/db-host/index.d.ts.map +1 -1
  19. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  20. package/dist/types/src/space/index.d.ts +0 -1
  21. package/dist/types/src/space/index.d.ts.map +1 -1
  22. package/dist/types/src/space/space-manager.d.ts +1 -4
  23. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  24. package/dist/types/src/space/space.d.ts +1 -7
  25. package/dist/types/src/space/space.d.ts.map +1 -1
  26. package/dist/types/src/testing/index.d.ts +0 -2
  27. package/dist/types/src/testing/index.d.ts.map +1 -1
  28. package/dist/types/src/testing/test-agent-builder.d.ts +1 -3
  29. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  30. package/package.json +30 -33
  31. package/src/automerge/automerge-host.ts +1 -1
  32. package/src/db-host/data-service.ts +10 -56
  33. package/src/db-host/index.ts +0 -2
  34. package/src/space/control-pipeline.ts +3 -1
  35. package/src/space/index.ts +0 -1
  36. package/src/space/space-manager.ts +1 -13
  37. package/src/space/space.test.ts +2 -112
  38. package/src/space/space.ts +2 -60
  39. package/src/testing/index.ts +0 -2
  40. package/src/testing/test-agent-builder.ts +3 -8
  41. package/dist/lib/browser/chunk-3FVT6KX6.mjs.map +0 -7
  42. package/dist/lib/node/chunk-WZ4WTAN6.cjs +0 -3454
  43. package/dist/lib/node/chunk-WZ4WTAN6.cjs.map +0 -7
  44. package/dist/types/src/db-host/data-service-host.d.ts +0 -38
  45. package/dist/types/src/db-host/data-service-host.d.ts.map +0 -1
  46. package/dist/types/src/db-host/database-host.d.ts +0 -27
  47. package/dist/types/src/db-host/database-host.d.ts.map +0 -1
  48. package/dist/types/src/space/data-pipeline.d.ts +0 -80
  49. package/dist/types/src/space/data-pipeline.d.ts.map +0 -1
  50. package/dist/types/src/space/data-pipeline.test.d.ts +0 -1
  51. package/dist/types/src/space/data-pipeline.test.d.ts.map +0 -1
  52. package/dist/types/src/testing/database-test-rig.d.ts +0 -67
  53. package/dist/types/src/testing/database-test-rig.d.ts.map +0 -1
  54. package/dist/types/src/testing/util.d.ts +0 -14
  55. package/dist/types/src/testing/util.d.ts.map +0 -1
  56. package/dist/types/src/tests/database.test.d.ts +0 -2
  57. package/dist/types/src/tests/database.test.d.ts.map +0 -1
  58. package/src/db-host/data-service-host.ts +0 -233
  59. package/src/db-host/database-host.ts +0 -63
  60. package/src/space/data-pipeline.test.ts +0 -3
  61. package/src/space/data-pipeline.ts +0 -468
  62. package/src/testing/database-test-rig.ts +0 -289
  63. package/src/testing/util.ts +0 -85
  64. package/src/tests/database.test.ts +0 -100
@@ -1,3454 +0,0 @@
1
- "use strict";
2
- var __create = Object.create;
3
- var __defProp = Object.defineProperty;
4
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
- var __hasOwnProp = Object.prototype.hasOwnProperty;
8
- var __export = (target, all) => {
9
- for (var name in all)
10
- __defProp(target, name, { get: all[name], enumerable: true });
11
- };
12
- var __copyProps = (to, from, except, desc) => {
13
- if (from && typeof from === "object" || typeof from === "function") {
14
- for (let key of __getOwnPropNames(from))
15
- if (!__hasOwnProp.call(to, key) && key !== except)
16
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
- }
18
- return to;
19
- };
20
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
- // If the importer is in node compatibility mode or this is not an ESM
22
- // file that has been converted to a CommonJS file using a Babel-
23
- // compatible transform (i.e. "__esModule" has not been set), then set
24
- // "default" to the CommonJS "module.exports" for node compatibility.
25
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
- mod
27
- ));
28
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
- var chunk_WZ4WTAN6_exports = {};
30
- __export(chunk_WZ4WTAN6_exports, {
31
- AuthExtension: () => AuthExtension,
32
- AuthStatus: () => AuthStatus,
33
- AutomergeHost: () => AutomergeHost,
34
- AutomergeStorageAdapter: () => AutomergeStorageAdapter,
35
- DataPipeline: () => DataPipeline,
36
- DataServiceHost: () => DataServiceHost,
37
- DataServiceImpl: () => DataServiceImpl,
38
- DataServiceSubscriptions: () => DataServiceSubscriptions,
39
- DatabaseHost: () => DatabaseHost,
40
- LocalHostNetworkAdapter: () => LocalHostNetworkAdapter,
41
- MOCK_AUTH_PROVIDER: () => MOCK_AUTH_PROVIDER,
42
- MOCK_AUTH_VERIFIER: () => MOCK_AUTH_VERIFIER,
43
- MeshNetworkAdapter: () => MeshNetworkAdapter,
44
- MetadataStore: () => MetadataStore,
45
- Pipeline: () => Pipeline,
46
- SnapshotManager: () => SnapshotManager,
47
- SnapshotStore: () => SnapshotStore,
48
- Space: () => Space,
49
- SpaceManager: () => SpaceManager,
50
- SpaceProtocol: () => SpaceProtocol,
51
- SpaceProtocolSession: () => SpaceProtocolSession,
52
- TimeframeClock: () => TimeframeClock,
53
- codec: () => codec,
54
- createMappedFeedWriter: () => createMappedFeedWriter,
55
- getSpaceKeyFromDoc: () => getSpaceKeyFromDoc,
56
- mapFeedIndexesToTimeframe: () => mapFeedIndexesToTimeframe,
57
- mapTimeframeToFeedIndexes: () => mapTimeframeToFeedIndexes,
58
- startAfter: () => startAfter,
59
- valueEncoding: () => valueEncoding
60
- });
61
- module.exports = __toCommonJS(chunk_WZ4WTAN6_exports);
62
- var import_hypercore = require("@dxos/hypercore");
63
- var import_protocols = require("@dxos/protocols");
64
- var import_invariant = require("@dxos/invariant");
65
- var import_async = require("@dxos/async");
66
- var import_codec_protobuf = require("@dxos/codec-protobuf");
67
- var import_context = require("@dxos/context");
68
- var import_echo_db = require("@dxos/echo-db");
69
- var import_invariant2 = require("@dxos/invariant");
70
- var import_log = require("@dxos/log");
71
- var import_service = require("@dxos/protocols/proto/dxos/echo/service");
72
- var import_util = require("@dxos/util");
73
- var import_echo_db2 = require("@dxos/echo-db");
74
- var import_context2 = require("@dxos/context");
75
- var import_keys = require("@dxos/keys");
76
- var import_protocols2 = require("@dxos/protocols");
77
- var import_blob = require("@dxos/protocols/proto/dxos/echo/blob");
78
- var import_crypto = require("@dxos/crypto");
79
- var import_protocols3 = require("@dxos/protocols");
80
- var import_debug = require("@dxos/debug");
81
- var import_invariant3 = require("@dxos/invariant");
82
- var import_keys2 = require("@dxos/keys");
83
- var import_log2 = require("@dxos/log");
84
- var import_util2 = require("@dxos/util");
85
- var import_crc_32 = __toESM(require("crc-32"));
86
- var import_async2 = require("@dxos/async");
87
- var import_context3 = require("@dxos/context");
88
- var import_invariant4 = require("@dxos/invariant");
89
- var import_keys3 = require("@dxos/keys");
90
- var import_log3 = require("@dxos/log");
91
- var import_protocols4 = require("@dxos/protocols");
92
- var import_services = require("@dxos/protocols/proto/dxos/client/services");
93
- var import_util3 = require("@dxos/util");
94
- var import_async3 = require("@dxos/async");
95
- var import_debug2 = require("@dxos/debug");
96
- var import_log4 = require("@dxos/log");
97
- var import_timeframe = require("@dxos/timeframe");
98
- var import_async4 = require("@dxos/async");
99
- var import_context4 = require("@dxos/context");
100
- var import_debug3 = require("@dxos/debug");
101
- var import_feed_store = require("@dxos/feed-store");
102
- var import_invariant5 = require("@dxos/invariant");
103
- var import_keys4 = require("@dxos/keys");
104
- var import_log5 = require("@dxos/log");
105
- var import_timeframe2 = require("@dxos/timeframe");
106
- var import_util4 = require("@dxos/util");
107
- var import_invariant6 = require("@dxos/invariant");
108
- var import_log6 = require("@dxos/log");
109
- var import_async5 = require("@dxos/async");
110
- var import_context5 = require("@dxos/context");
111
- var import_crypto2 = require("@dxos/crypto");
112
- var import_invariant7 = require("@dxos/invariant");
113
- var import_log7 = require("@dxos/log");
114
- var import_protocols5 = require("@dxos/protocols");
115
- var import_teleport = require("@dxos/teleport");
116
- var import_async6 = require("@dxos/async");
117
- var import_context6 = require("@dxos/context");
118
- var import_credentials = require("@dxos/credentials");
119
- var import_echo_db3 = require("@dxos/echo-db");
120
- var import_invariant8 = require("@dxos/invariant");
121
- var import_log8 = require("@dxos/log");
122
- var import_protocols6 = require("@dxos/protocols");
123
- var import_timeframe3 = require("@dxos/timeframe");
124
- var import_tracing = require("@dxos/tracing");
125
- var import_util5 = require("@dxos/util");
126
- var import_async7 = require("@dxos/async");
127
- var import_invariant9 = require("@dxos/invariant");
128
- var import_log9 = require("@dxos/log");
129
- var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
130
- var import_tracing2 = require("@dxos/tracing");
131
- var import_util6 = require("@dxos/util");
132
- var import_async8 = require("@dxos/async");
133
- var import_context7 = require("@dxos/context");
134
- var import_credentials3 = require("@dxos/credentials");
135
- var import_keys5 = require("@dxos/keys");
136
- var import_log10 = require("@dxos/log");
137
- var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
138
- var import_timeframe4 = require("@dxos/timeframe");
139
- var import_tracing3 = require("@dxos/tracing");
140
- var import_util7 = require("@dxos/util");
141
- var import_crypto3 = require("@dxos/crypto");
142
- var import_keys6 = require("@dxos/keys");
143
- var import_log11 = require("@dxos/log");
144
- var import_network_manager = require("@dxos/network-manager");
145
- var import_teleport2 = require("@dxos/teleport");
146
- var import_teleport_extension_object_sync = require("@dxos/teleport-extension-object-sync");
147
- var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
148
- var import_tracing4 = require("@dxos/tracing");
149
- var import_util8 = require("@dxos/util");
150
- var import_async9 = require("@dxos/async");
151
- var import_debug4 = require("@dxos/debug");
152
- var import_keys7 = require("@dxos/keys");
153
- var import_log12 = require("@dxos/log");
154
- var import_protocols7 = require("@dxos/protocols");
155
- var import_util9 = require("@dxos/util");
156
- var import_util10 = require("@dxos/util");
157
- var import_async10 = require("@dxos/async");
158
- var import_automerge_repo = require("@dxos/automerge/automerge-repo");
159
- var import_codec_protobuf2 = require("@dxos/codec-protobuf");
160
- var import_invariant10 = require("@dxos/invariant");
161
- var import_log13 = require("@dxos/log");
162
- var import_async11 = require("@dxos/async");
163
- var import_automerge_repo2 = require("@dxos/automerge/automerge-repo");
164
- var import_invariant11 = require("@dxos/invariant");
165
- var import_log14 = require("@dxos/log");
166
- var import_teleport_extension_automerge_replicator = require("@dxos/teleport-extension-automerge-replicator");
167
- var import_automerge = require("@dxos/automerge/automerge");
168
- var import_automerge_repo3 = require("@dxos/automerge/automerge-repo");
169
- var import_automerge_repo_storage_indexeddb = require("@dxos/automerge/automerge-repo-storage-indexeddb");
170
- var import_context8 = require("@dxos/context");
171
- var import_keys8 = require("@dxos/keys");
172
- var import_log15 = require("@dxos/log");
173
- var import_protocols8 = require("@dxos/protocols");
174
- var import_random_access_storage = require("@dxos/random-access-storage");
175
- var import_tracing5 = require("@dxos/tracing");
176
- var import_util11 = require("@dxos/util");
177
- var codec = import_protocols.schema.getCodecForType("dxos.echo.feed.FeedMessage");
178
- var valueEncoding = (0, import_hypercore.createCodecEncoding)(codec);
179
- var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/common/feeds.ts";
180
- var createMappedFeedWriter = (mapper, writer) => {
181
- (0, import_invariant.invariant)(mapper, void 0, {
182
- F: __dxlog_file,
183
- L: 16,
184
- S: void 0,
185
- A: [
186
- "mapper",
187
- ""
188
- ]
189
- });
190
- (0, import_invariant.invariant)(writer, void 0, {
191
- F: __dxlog_file,
192
- L: 17,
193
- S: void 0,
194
- A: [
195
- "writer",
196
- ""
197
- ]
198
- });
199
- return {
200
- write: async (data, options) => await writer.write(await mapper(data), options)
201
- };
202
- };
203
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
204
- var MUTATION_LIMIT_PER_OBJECT = 10;
205
- var DataServiceHost = class {
206
- constructor(_itemManager, _itemDemuxer, _flush, _writeStream, _opts = {}) {
207
- this._itemManager = _itemManager;
208
- this._itemDemuxer = _itemDemuxer;
209
- this._flush = _flush;
210
- this._writeStream = _writeStream;
211
- this._opts = _opts;
212
- this._ctx = new import_context.Context();
213
- this._clientTagMap = new import_util.ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
214
- }
215
- async open() {
216
- }
217
- async close() {
218
- await this._ctx.dispose();
219
- }
220
- get _deferEvents() {
221
- return this._opts.deferEvents ?? true;
222
- }
223
- /**
224
- * Real-time subscription to data objects in a space.
225
- */
226
- subscribe() {
227
- return new import_codec_protobuf.Stream(({ next, close, ctx }) => {
228
- ctx.onDispose(this._ctx.onDispose(close));
229
- const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
230
- next({
231
- batch: {
232
- objects
233
- }
234
- });
235
- const updateScheduler = new import_async.UpdateScheduler(ctx, async () => {
236
- flushPendingUpdate();
237
- }, {
238
- maxFrequency: 10
239
- });
240
- const pendingUpdates = [];
241
- const mutationsPerObject = /* @__PURE__ */ new Map();
242
- const clearPendingUpdates = () => {
243
- pendingUpdates.length = 0;
244
- mutationsPerObject.clear();
245
- };
246
- const flushPendingUpdate = () => {
247
- const stagedEvents = [];
248
- const objectsWithSnapshots = /* @__PURE__ */ new Set();
249
- for (const [id, count] of mutationsPerObject) {
250
- if (count >= MUTATION_LIMIT_PER_OBJECT) {
251
- objectsWithSnapshots.add(id);
252
- const entity = this._itemManager.entities.get(id);
253
- if (entity) {
254
- stagedEvents.push(entity.createSnapshot());
255
- }
256
- }
257
- }
258
- for (const obj of pendingUpdates) {
259
- if (!objectsWithSnapshots.has(obj.objectId)) {
260
- stagedEvents.push(obj);
261
- }
262
- }
263
- next({
264
- batch: {
265
- objects: stagedEvents
266
- }
267
- });
268
- clearPendingUpdates();
269
- };
270
- this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
271
- clearPendingUpdates();
272
- next({
273
- action: import_service.EchoEvent.DatabaseAction.RESET,
274
- batch: {
275
- objects: snapshot.items
276
- }
277
- });
278
- });
279
- this._itemDemuxer.mutation.on(ctx, (message) => {
280
- const { batch, meta } = message;
281
- (0, import_invariant2.invariant)(!meta.clientTag, "Unexpected client tag in mutation message", {
282
- F: __dxlog_file2,
283
- L: 138,
284
- S: this,
285
- A: [
286
- "!(meta as any).clientTag",
287
- "'Unexpected client tag in mutation message'"
288
- ]
289
- });
290
- (0, import_log.log)("message", {
291
- batch: batch.objects?.length,
292
- meta
293
- }, {
294
- F: __dxlog_file2,
295
- L: 139,
296
- S: this,
297
- C: (f, a) => f(...a)
298
- });
299
- const clientTag = this._clientTagMap.get([
300
- message.meta.feedKey,
301
- message.meta.seq
302
- ]);
303
- batch.objects?.forEach((object) => {
304
- (0, import_echo_db.setMetadataOnObject)(object, {
305
- ...meta
306
- });
307
- });
308
- if (clientTag) {
309
- flushPendingUpdate();
310
- (0, import_echo_db.tagMutationsInBatch)(batch, clientTag, 0);
311
- next({
312
- clientTag,
313
- feedKey: message.meta.feedKey,
314
- seq: message.meta.seq,
315
- batch
316
- });
317
- } else {
318
- for (const obj of batch.objects ?? []) {
319
- const newCount = (mutationsPerObject.get(obj.objectId) ?? 0) + 1;
320
- mutationsPerObject.set(obj.objectId, newCount);
321
- }
322
- for (const obj of batch.objects ?? []) {
323
- if ((mutationsPerObject.get(obj.objectId) ?? 0) < MUTATION_LIMIT_PER_OBJECT) {
324
- pendingUpdates.push(obj);
325
- }
326
- }
327
- if (this._deferEvents) {
328
- updateScheduler.trigger();
329
- } else {
330
- flushPendingUpdate();
331
- }
332
- }
333
- });
334
- });
335
- }
336
- async write(request) {
337
- (0, import_invariant2.invariant)(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
338
- F: __dxlog_file2,
339
- L: 184,
340
- S: this,
341
- A: [
342
- "!this._ctx.disposed",
343
- "'Cannot write to closed DataServiceHost'"
344
- ]
345
- });
346
- (0, import_invariant2.invariant)(this._writeStream, "Cannot write mutations in readonly mode", {
347
- F: __dxlog_file2,
348
- L: 185,
349
- S: this,
350
- A: [
351
- "this._writeStream",
352
- "'Cannot write mutations in readonly mode'"
353
- ]
354
- });
355
- (0, import_log.log)("write", {
356
- clientTag: request.clientTag,
357
- objectCount: request.batch.objects?.length ?? 0
358
- }, {
359
- F: __dxlog_file2,
360
- L: 187,
361
- S: this,
362
- C: (f, a) => f(...a)
363
- });
364
- const message = createDataMessage(request.batch);
365
- const receipt = await this._writeStream.write(message, {
366
- afterWrite: async (receipt2) => {
367
- if (request.clientTag) {
368
- (0, import_log.log)("tag", {
369
- clientTag: request.clientTag,
370
- feedKey: receipt2.feedKey,
371
- seq: receipt2.seq
372
- }, {
373
- F: __dxlog_file2,
374
- L: 196,
375
- S: this,
376
- C: (f, a) => f(...a)
377
- });
378
- this._clientTagMap.set([
379
- receipt2.feedKey,
380
- receipt2.seq
381
- ], request.clientTag);
382
- }
383
- }
384
- });
385
- return receipt;
386
- }
387
- async flush() {
388
- await this._flush();
389
- }
390
- getHostInfo() {
391
- throw new Error("Method not implemented.");
392
- }
393
- syncRepo(request) {
394
- throw new Error("Method not implemented.");
395
- }
396
- sendSyncMessage(request) {
397
- throw new Error("Method not implemented.");
398
- }
399
- };
400
- var createDataMessage = (batch) => ({
401
- batch: {
402
- objects: batch.objects?.map((object) => ({
403
- ...object,
404
- mutations: object.mutations?.map((mutation) => ({
405
- ...mutation,
406
- meta: void 0
407
- })),
408
- meta: void 0
409
- }))
410
- }
411
- });
412
- var DatabaseHost = class {
413
- constructor(_outboundStream, _flush) {
414
- this._outboundStream = _outboundStream;
415
- this._flush = _flush;
416
- }
417
- get isReadOnly() {
418
- return !!this._outboundStream;
419
- }
420
- get echoProcessor() {
421
- return this._echoProcessor;
422
- }
423
- async open(itemManager, modelFactory) {
424
- this._itemManager = itemManager;
425
- this._itemManager._debugLabel = "host";
426
- this._itemDemuxer = new import_echo_db2.ItemDemuxer(itemManager, modelFactory);
427
- this._echoProcessor = this._itemDemuxer.open();
428
- }
429
- async close() {
430
- }
431
- getWriteStream() {
432
- return this._outboundStream;
433
- }
434
- createSnapshot() {
435
- return this._itemDemuxer.createSnapshot();
436
- }
437
- createDataServiceHost(opts = {}) {
438
- return new DataServiceHost(this._itemManager, this._itemDemuxer, this._flush, this._outboundStream ?? void 0, opts);
439
- }
440
- };
441
- var SpaceSnapshot = import_protocols2.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
442
- var SnapshotManager = class {
443
- constructor(_snapshotStore, _blobStore, _blobSync) {
444
- this._snapshotStore = _snapshotStore;
445
- this._blobStore = _blobStore;
446
- this._blobSync = _blobSync;
447
- }
448
- async _getBlob(blobId) {
449
- const blob = await this._blobStore.get(blobId);
450
- return SpaceSnapshot.decode(blob);
451
- }
452
- async load(ctx, id) {
453
- const blobId = import_keys.PublicKey.fromHex(id).asUint8Array();
454
- const blobMeta = await this._blobStore.getMeta(blobId);
455
- if (blobMeta && blobMeta.state === import_blob.BlobMeta.State.FULLY_PRESENT) {
456
- return this._getBlob(blobId);
457
- }
458
- const fallbackStore = await (0, import_context2.cancelWithContext)(ctx, this._snapshotStore.loadSnapshot(id));
459
- if (fallbackStore) {
460
- return fallbackStore;
461
- }
462
- await this._blobSync.download(ctx, blobId);
463
- return this._getBlob(blobId);
464
- }
465
- async store(snapshot) {
466
- const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
467
- await this._blobSync.notifyBlobAdded(id);
468
- return import_keys.PublicKey.from(id).toHex();
469
- }
470
- };
471
- var SpaceSnapshot2 = import_protocols3.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
472
- var SnapshotStore = class {
473
- constructor(_directory) {
474
- this._directory = _directory;
475
- }
476
- async saveSnapshot(snapshot) {
477
- const encoded = SpaceSnapshot2.encode(snapshot);
478
- const key = await import_crypto.subtleCrypto.digest("SHA-256", encoded);
479
- const keyString = Buffer.from(key).toString("hex");
480
- const file = await this._directory.getOrCreateFile(keyString);
481
- try {
482
- await file.write(0, Buffer.from(encoded));
483
- } finally {
484
- await file.close();
485
- }
486
- return keyString;
487
- }
488
- async loadSnapshot(key) {
489
- const file = await this._directory.getOrCreateFile(key);
490
- try {
491
- const { size } = await file.stat();
492
- if (size === 0) {
493
- return void 0;
494
- }
495
- const buffer = await file.read(0, size);
496
- return SpaceSnapshot2.decode(buffer);
497
- } finally {
498
- await file.close();
499
- }
500
- }
501
- async listSnapshots() {
502
- const entries = await this._directory.list();
503
- return await Promise.all(entries.map(async (key) => {
504
- const { size } = await this._directory.getOrCreateFile(key).stat();
505
- return {
506
- key,
507
- size
508
- };
509
- }));
510
- }
511
- };
512
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
513
- var DataServiceSubscriptions = class {
514
- constructor() {
515
- this._spaces = new import_util2.ComplexMap(import_keys2.PublicKey.hash);
516
- }
517
- clear() {
518
- this._spaces.clear();
519
- }
520
- async registerSpace(spaceKey, host) {
521
- (0, import_log2.log)("Registering space", {
522
- spaceKey
523
- }, {
524
- F: __dxlog_file3,
525
- L: 35,
526
- S: this,
527
- C: (f, a) => f(...a)
528
- });
529
- (0, import_invariant3.invariant)(!this._spaces.has(spaceKey), void 0, {
530
- F: __dxlog_file3,
531
- L: 36,
532
- S: this,
533
- A: [
534
- "!this._spaces.has(spaceKey)",
535
- ""
536
- ]
537
- });
538
- await host.open();
539
- this._spaces.set(spaceKey, host);
540
- }
541
- async unregisterSpace(spaceKey) {
542
- (0, import_log2.log)("Unregistering space", {
543
- spaceKey
544
- }, {
545
- F: __dxlog_file3,
546
- L: 42,
547
- S: this,
548
- C: (f, a) => f(...a)
549
- });
550
- const host = this._spaces.get(spaceKey);
551
- await host?.close();
552
- this._spaces.delete(spaceKey);
553
- }
554
- getDataService(spaceKey) {
555
- return this._spaces.get(spaceKey);
556
- }
557
- };
558
- var DataServiceImpl = class {
559
- constructor(_subscriptions, _automergeHost) {
560
- this._subscriptions = _subscriptions;
561
- this._automergeHost = _automergeHost;
562
- }
563
- subscribe(request) {
564
- (0, import_invariant3.invariant)(request.spaceKey, void 0, {
565
- F: __dxlog_file3,
566
- L: 64,
567
- S: this,
568
- A: [
569
- "request.spaceKey",
570
- ""
571
- ]
572
- });
573
- const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
574
- return host.subscribe();
575
- }
576
- write(request) {
577
- (0, import_invariant3.invariant)(request.spaceKey, void 0, {
578
- F: __dxlog_file3,
579
- L: 71,
580
- S: this,
581
- A: [
582
- "request.spaceKey",
583
- ""
584
- ]
585
- });
586
- (0, import_invariant3.invariant)(request.batch, void 0, {
587
- F: __dxlog_file3,
588
- L: 72,
589
- S: this,
590
- A: [
591
- "request.batch",
592
- ""
593
- ]
594
- });
595
- const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
596
- return host.write(request);
597
- }
598
- flush(request) {
599
- (0, import_invariant3.invariant)(request.spaceKey, void 0, {
600
- F: __dxlog_file3,
601
- L: 79,
602
- S: this,
603
- A: [
604
- "request.spaceKey",
605
- ""
606
- ]
607
- });
608
- const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
609
- return host.flush();
610
- }
611
- // Automerge specific.
612
- async getHostInfo(request) {
613
- return this._automergeHost.getHostInfo();
614
- }
615
- syncRepo(request) {
616
- return this._automergeHost.syncRepo(request);
617
- }
618
- sendSyncMessage(request) {
619
- return this._automergeHost.sendSyncMessage(request);
620
- }
621
- };
622
- function _ts_decorate(decorators, target, key, desc) {
623
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
624
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
625
- r = Reflect.decorate(decorators, target, key, desc);
626
- else
627
- for (var i = decorators.length - 1; i >= 0; i--)
628
- if (d = decorators[i])
629
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
630
- return c > 3 && r && Object.defineProperty(target, key, r), r;
631
- }
632
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
633
- var EXPIRED_INVITATION_CLEANUP_INTERVAL = 60 * 60 * 1e3;
634
- var emptyEchoMetadata = () => ({
635
- version: import_protocols4.STORAGE_VERSION,
636
- spaces: [],
637
- created: /* @__PURE__ */ new Date(),
638
- updated: /* @__PURE__ */ new Date()
639
- });
640
- var emptyLargeSpaceMetadata = () => ({});
641
- var EchoMetadata = import_protocols4.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
642
- var LargeSpaceMetadata = import_protocols4.schema.getCodecForType("dxos.echo.metadata.LargeSpaceMetadata");
643
- var MetadataStore = class {
644
- constructor(directory) {
645
- this._metadata = emptyEchoMetadata();
646
- this._spaceLargeMetadata = new import_util3.ComplexMap(import_keys3.PublicKey.hash);
647
- this._metadataFile = void 0;
648
- this.update = new import_async2.Event();
649
- this._invitationCleanupCtx = new import_context3.Context();
650
- this._directory = directory;
651
- }
652
- get metadata() {
653
- return this._metadata;
654
- }
655
- get version() {
656
- return this._metadata.version ?? 0;
657
- }
658
- /**
659
- * Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
660
- * addSpaceFeed functions.
661
- */
662
- get spaces() {
663
- return this._metadata.spaces ?? [];
664
- }
665
- async _readFile(file, codec2) {
666
- try {
667
- const { size: fileLength } = await file.stat();
668
- if (fileLength < 8) {
669
- return;
670
- }
671
- const dataSize = fromBytesInt32(await file.read(0, 4));
672
- const checksum = fromBytesInt32(await file.read(4, 4));
673
- (0, import_log3.log)("loaded", {
674
- size: dataSize,
675
- checksum,
676
- name: file.filename
677
- }, {
678
- F: __dxlog_file4,
679
- L: 89,
680
- S: this,
681
- C: (f, a) => f(...a)
682
- });
683
- if (fileLength < dataSize + 8) {
684
- throw new import_protocols4.DataCorruptionError("Metadata size is smaller than expected.", {
685
- fileLength,
686
- dataSize
687
- });
688
- }
689
- const data = await file.read(8, dataSize);
690
- const calculatedChecksum = import_crc_32.default.buf(data);
691
- if (calculatedChecksum !== checksum) {
692
- throw new import_protocols4.DataCorruptionError("Metadata checksum is invalid.");
693
- }
694
- return codec2.decode(data);
695
- } finally {
696
- await file.close();
697
- }
698
- }
699
- /**
700
- * @internal
701
- */
702
- async _writeFile(file, codec2, data) {
703
- const encoded = (0, import_util3.arrayToBuffer)(codec2.encode(data));
704
- const checksum = import_crc_32.default.buf(encoded);
705
- const result = Buffer.alloc(8 + encoded.length);
706
- result.writeInt32LE(encoded.length, 0);
707
- result.writeInt32LE(checksum, 4);
708
- encoded.copy(result, 8);
709
- await file.write(0, result);
710
- (0, import_log3.log)("saved", {
711
- size: encoded.length,
712
- checksum
713
- }, {
714
- F: __dxlog_file4,
715
- L: 124,
716
- S: this,
717
- C: (f, a) => f(...a)
718
- });
719
- }
720
- async close() {
721
- await this._invitationCleanupCtx.dispose();
722
- await this.flush();
723
- await this._metadataFile?.close();
724
- this._metadataFile = void 0;
725
- this._metadata = emptyEchoMetadata();
726
- this._spaceLargeMetadata.clear();
727
- }
728
- /**
729
- * Loads metadata from persistent storage.
730
- */
731
- async load() {
732
- if (!this._metadataFile || this._metadataFile.closed) {
733
- this._metadataFile = this._directory.getOrCreateFile("EchoMetadata");
734
- }
735
- try {
736
- const metadata = await this._readFile(this._metadataFile, EchoMetadata);
737
- if (metadata) {
738
- this._metadata = metadata;
739
- }
740
- this._metadata.spaces?.forEach((space) => {
741
- space.state ??= import_services.SpaceState.ACTIVE;
742
- });
743
- } catch (err) {
744
- import_log3.log.error("failed to load metadata", {
745
- err
746
- }, {
747
- F: __dxlog_file4,
748
- L: 156,
749
- S: this,
750
- C: (f, a) => f(...a)
751
- });
752
- this._metadata = emptyEchoMetadata();
753
- }
754
- await (0, import_util3.forEachAsync)([
755
- this._metadata.identity?.haloSpace.key,
756
- ...this._metadata.spaces?.map((space) => space.key) ?? []
757
- ].filter(import_util3.isNotNullOrUndefined), async (key) => {
758
- try {
759
- await this._loadSpaceLargeMetadata(key);
760
- } catch (err) {
761
- import_log3.log.error("failed to load space large metadata", {
762
- err
763
- }, {
764
- F: __dxlog_file4,
765
- L: 168,
766
- S: this,
767
- C: (f, a) => f(...a)
768
- });
769
- }
770
- });
771
- (0, import_async2.scheduleTaskInterval)(this._invitationCleanupCtx, async () => {
772
- for (const invitation of this.getInvitations()) {
773
- if (invitation.created && invitation.lifetime && invitation.lifetime !== 0 && invitation.created.getTime() + invitation.lifetime * 1e3 < Date.now()) {
774
- await this.removeInvitation(invitation.invitationId);
775
- }
776
- }
777
- }, EXPIRED_INVITATION_CLEANUP_INTERVAL);
778
- }
779
- async _save() {
780
- const data = {
781
- ...this._metadata,
782
- version: import_protocols4.STORAGE_VERSION,
783
- created: this._metadata.created ?? /* @__PURE__ */ new Date(),
784
- updated: /* @__PURE__ */ new Date()
785
- };
786
- this.update.emit(data);
787
- const file = this._directory.getOrCreateFile("EchoMetadata");
788
- await this._writeFile(file, EchoMetadata, data);
789
- }
790
- async _loadSpaceLargeMetadata(key) {
791
- const file = this._directory.getOrCreateFile(`space_${key.toHex()}_large`);
792
- try {
793
- const metadata = await this._readFile(file, LargeSpaceMetadata);
794
- if (metadata) {
795
- this._spaceLargeMetadata.set(key, metadata);
796
- }
797
- } catch (err) {
798
- import_log3.log.error("failed to load space large metadata", {
799
- err
800
- }, {
801
- F: __dxlog_file4,
802
- L: 215,
803
- S: this,
804
- C: (f, a) => f(...a)
805
- });
806
- }
807
- }
808
- async _saveSpaceLargeMetadata(key) {
809
- const data = this._getLargeSpaceMetadata(key);
810
- const file = this._directory.getOrCreateFile(`space_${key.toHex()}_large`);
811
- await this._writeFile(file, LargeSpaceMetadata, data);
812
- }
813
- async flush() {
814
- await this._directory.flush();
815
- }
816
- _getSpace(spaceKey) {
817
- if (this._metadata.identity?.haloSpace.key.equals(spaceKey)) {
818
- return this._metadata.identity.haloSpace;
819
- }
820
- const space = this.spaces.find((space2) => space2.key === spaceKey);
821
- (0, import_invariant4.invariant)(space, "Space not found", {
822
- F: __dxlog_file4,
823
- L: 237,
824
- S: this,
825
- A: [
826
- "space",
827
- "'Space not found'"
828
- ]
829
- });
830
- return space;
831
- }
832
- _getLargeSpaceMetadata(key) {
833
- let entry = this._spaceLargeMetadata.get(key);
834
- if (entry) {
835
- return entry;
836
- }
837
- entry = emptyLargeSpaceMetadata();
838
- this._spaceLargeMetadata.set(key, entry);
839
- return entry;
840
- }
841
- /**
842
- * Clears storage - doesn't work for now.
843
- */
844
- async clear() {
845
- (0, import_log3.log)("clearing all metadata", void 0, {
846
- F: __dxlog_file4,
847
- L: 256,
848
- S: this,
849
- C: (f, a) => f(...a)
850
- });
851
- await this._directory.delete();
852
- this._metadata = emptyEchoMetadata();
853
- }
854
- getIdentityRecord() {
855
- return this._metadata.identity;
856
- }
857
- async setIdentityRecord(record) {
858
- (0, import_invariant4.invariant)(!this._metadata.identity, "Cannot overwrite existing identity in metadata", {
859
- F: __dxlog_file4,
860
- L: 266,
861
- S: this,
862
- A: [
863
- "!this._metadata.identity",
864
- "'Cannot overwrite existing identity in metadata'"
865
- ]
866
- });
867
- this._metadata.identity = record;
868
- await this._save();
869
- await this.flush();
870
- }
871
- getInvitations() {
872
- return this._metadata.invitations ?? [];
873
- }
874
- async addInvitation(invitation) {
875
- if (this._metadata.invitations?.find((i) => i.invitationId === invitation.invitationId)) {
876
- return;
877
- }
878
- (this._metadata.invitations ??= []).push(invitation);
879
- await this._save();
880
- await this.flush();
881
- }
882
- async removeInvitation(invitationId) {
883
- this._metadata.invitations = (this._metadata.invitations ?? []).filter((i) => i.invitationId !== invitationId);
884
- await this._save();
885
- await this.flush();
886
- }
887
- async addSpace(record) {
888
- (0, import_invariant4.invariant)(!(this._metadata.spaces ?? []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata", {
889
- F: __dxlog_file4,
890
- L: 294,
891
- S: this,
892
- A: [
893
- "!(this._metadata.spaces ?? []).find((space) => space.key === record.key)",
894
- "'Cannot overwrite existing space in metadata'"
895
- ]
896
- });
897
- (this._metadata.spaces ??= []).push(record);
898
- await this._save();
899
- await this.flush();
900
- }
901
- async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
902
- this._getSpace(spaceKey).dataTimeframe = timeframe;
903
- await this._save();
904
- }
905
- async setSpaceControlLatestTimeframe(spaceKey, timeframe) {
906
- this._getSpace(spaceKey).controlTimeframe = timeframe;
907
- await this._save();
908
- await this.flush();
909
- }
910
- async setCache(spaceKey, cache) {
911
- this._getSpace(spaceKey).cache = cache;
912
- await this._save();
913
- }
914
- async setWritableFeedKeys(spaceKey, controlFeedKey, dataFeedKey) {
915
- const space = this._getSpace(spaceKey);
916
- space.controlFeedKey = controlFeedKey;
917
- space.dataFeedKey = dataFeedKey;
918
- await this._save();
919
- await this.flush();
920
- }
921
- async setSpaceState(spaceKey, state) {
922
- this._getSpace(spaceKey).state = state;
923
- await this._save();
924
- await this.flush();
925
- }
926
- getSpaceControlPipelineSnapshot(spaceKey) {
927
- return this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot;
928
- }
929
- async setSpaceControlPipelineSnapshot(spaceKey, snapshot) {
930
- this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot = snapshot;
931
- await this._saveSpaceLargeMetadata(spaceKey);
932
- await this.flush();
933
- }
934
- };
935
- _ts_decorate([
936
- import_async2.synchronized
937
- ], MetadataStore.prototype, "load", null);
938
- _ts_decorate([
939
- import_async2.synchronized
940
- ], MetadataStore.prototype, "_save", null);
941
- _ts_decorate([
942
- import_async2.synchronized
943
- ], MetadataStore.prototype, "_saveSpaceLargeMetadata", null);
944
- var fromBytesInt32 = (buf) => buf.readInt32LE(0);
945
- function _ts_decorate2(decorators, target, key, desc) {
946
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
947
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
948
- r = Reflect.decorate(decorators, target, key, desc);
949
- else
950
- for (var i = decorators.length - 1; i >= 0; i--)
951
- if (d = decorators[i])
952
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
953
- return c > 3 && r && Object.defineProperty(target, key, r), r;
954
- }
955
- var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
956
- var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
957
- feedKey,
958
- index
959
- }));
960
- var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
961
- feedKey,
962
- index
963
- ]));
964
- var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
965
- feedKey,
966
- index: index + 1
967
- }));
968
- var TimeframeClock = class {
969
- constructor(_timeframe = new import_timeframe.Timeframe()) {
970
- this._timeframe = _timeframe;
971
- this.update = new import_async3.Event();
972
- this._pendingTimeframe = _timeframe;
973
- }
974
- /**
975
- * Timeframe that was processed by ECHO.
976
- */
977
- get timeframe() {
978
- return this._timeframe;
979
- }
980
- /**
981
- * Timeframe that is currently being processed by ECHO.
982
- * Will be equal to `timeframe` after the processing is complete.
983
- */
984
- get pendingTimeframe() {
985
- return this._pendingTimeframe;
986
- }
987
- setTimeframe(timeframe) {
988
- this._timeframe = timeframe;
989
- this._pendingTimeframe = timeframe;
990
- this.update.emit(this._timeframe);
991
- }
992
- updatePendingTimeframe(key, seq) {
993
- this._pendingTimeframe = import_timeframe.Timeframe.merge(this._pendingTimeframe, new import_timeframe.Timeframe([
994
- [
995
- key,
996
- seq
997
- ]
998
- ]));
999
- }
1000
- updateTimeframe() {
1001
- this._timeframe = this._pendingTimeframe;
1002
- this.update.emit(this._timeframe);
1003
- }
1004
- hasGaps(timeframe) {
1005
- const gaps = import_timeframe.Timeframe.dependencies(timeframe, this._timeframe);
1006
- return !gaps.isEmpty();
1007
- }
1008
- async waitUntilReached(target) {
1009
- (0, import_log4.log)("waitUntilReached", {
1010
- target,
1011
- current: this._timeframe
1012
- }, {
1013
- F: __dxlog_file5,
1014
- L: 70,
1015
- S: this,
1016
- C: (f, a) => f(...a)
1017
- });
1018
- await this.update.waitForCondition(() => {
1019
- (0, import_log4.log)("check if reached", {
1020
- target,
1021
- current: this._timeframe,
1022
- deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
1023
- }, {
1024
- F: __dxlog_file5,
1025
- L: 72,
1026
- S: this,
1027
- C: (f, a) => f(...a)
1028
- });
1029
- return import_timeframe.Timeframe.dependencies(target, this._timeframe).isEmpty();
1030
- });
1031
- }
1032
- };
1033
- _ts_decorate2([
1034
- (0, import_debug2.timed)(5e3)
1035
- ], TimeframeClock.prototype, "waitUntilReached", null);
1036
- var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
1037
- var createMessageSelector = (timeframeClock) => {
1038
- return (messages) => {
1039
- for (let i = 0; i < messages.length; i++) {
1040
- const { data: { timeframe } } = messages[i];
1041
- (0, import_invariant6.invariant)(timeframe, void 0, {
1042
- F: __dxlog_file6,
1043
- L: 25,
1044
- S: void 0,
1045
- A: [
1046
- "timeframe",
1047
- ""
1048
- ]
1049
- });
1050
- if (!timeframeClock.hasGaps(timeframe)) {
1051
- return i;
1052
- }
1053
- }
1054
- (0, import_log6.log)("Skipping...", void 0, {
1055
- F: __dxlog_file6,
1056
- L: 33,
1057
- S: void 0,
1058
- C: (f, a) => f(...a)
1059
- });
1060
- };
1061
- };
1062
- function _ts_decorate3(decorators, target, key, desc) {
1063
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1064
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1065
- r = Reflect.decorate(decorators, target, key, desc);
1066
- else
1067
- for (var i = decorators.length - 1; i >= 0; i--)
1068
- if (d = decorators[i])
1069
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1070
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1071
- }
1072
- var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
1073
- var PipelineState = class {
1074
- constructor(_feeds, _timeframeClock) {
1075
- this._feeds = _feeds;
1076
- this._timeframeClock = _timeframeClock;
1077
- this._ctx = new import_context4.Context();
1078
- this.timeframeUpdate = this._timeframeClock.update;
1079
- this.stalled = new import_async4.Event();
1080
- this._startTimeframe = new import_timeframe2.Timeframe();
1081
- this._reachedTarget = false;
1082
- }
1083
- /**
1084
- * Latest theoretical timeframe based on the last mutation in each feed.
1085
- * NOTE: This might never be reached if the mutation dependencies
1086
- */
1087
- // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
1088
- get endTimeframe() {
1089
- return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
1090
- feedKey: feed.key,
1091
- index: feed.length - 1
1092
- })));
1093
- }
1094
- get startTimeframe() {
1095
- return this._startTimeframe;
1096
- }
1097
- get timeframe() {
1098
- return this._timeframeClock.timeframe;
1099
- }
1100
- get pendingTimeframe() {
1101
- return this._timeframeClock.pendingTimeframe;
1102
- }
1103
- get targetTimeframe() {
1104
- return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
1105
- }
1106
- get reachedTarget() {
1107
- return this._reachedTarget;
1108
- }
1109
- get feeds() {
1110
- return Array.from(this._feeds.values());
1111
- }
1112
- async waitUntilTimeframe(target) {
1113
- await this._timeframeClock.waitUntilReached(target);
1114
- }
1115
- setTargetTimeframe(target) {
1116
- this._targetTimeframe = target;
1117
- }
1118
- /**
1119
- * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
1120
- *
1121
- * This function will resolve immediately if the pipeline is stalled.
1122
- *
1123
- * @param timeout Timeout in milliseconds to specify the maximum wait time.
1124
- */
1125
- async waitUntilReachedTargetTimeframe({ ctx = new import_context4.Context(), timeout, breakOnStall = true } = {}) {
1126
- (0, import_log5.log)("waitUntilReachedTargetTimeframe", {
1127
- timeout,
1128
- current: this.timeframe,
1129
- target: this.targetTimeframe
1130
- }, {
1131
- F: __dxlog_file7,
1132
- L: 133,
1133
- S: this,
1134
- C: (f, a) => f(...a)
1135
- });
1136
- this._reachedTargetPromise ??= Promise.race([
1137
- this._timeframeClock.update.waitForCondition(() => {
1138
- return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
1139
- }),
1140
- ...breakOnStall ? [
1141
- this.stalled.discardParameter().waitForCount(1)
1142
- ] : []
1143
- ]);
1144
- let done = false;
1145
- if (timeout) {
1146
- return Promise.race([
1147
- (0, import_context4.rejectOnDispose)(ctx),
1148
- (0, import_context4.rejectOnDispose)(this._ctx),
1149
- this._reachedTargetPromise.then(() => {
1150
- done = true;
1151
- this._reachedTarget = true;
1152
- }),
1153
- (0, import_async4.sleepWithContext)(this._ctx, timeout).then(() => {
1154
- if (done) {
1155
- return;
1156
- }
1157
- import_log5.log.warn("waitUntilReachedTargetTimeframe timed out", {
1158
- timeout,
1159
- current: this.timeframe,
1160
- target: this.targetTimeframe,
1161
- dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
1162
- }, {
1163
- F: __dxlog_file7,
1164
- L: 161,
1165
- S: this,
1166
- C: (f, a) => f(...a)
1167
- });
1168
- })
1169
- ]);
1170
- } else {
1171
- return this._reachedTargetPromise;
1172
- }
1173
- }
1174
- };
1175
- var Pipeline = class {
1176
- constructor() {
1177
- this._timeframeClock = new TimeframeClock(new import_timeframe2.Timeframe());
1178
- this._feeds = new import_util4.ComplexMap(import_keys4.PublicKey.hash);
1179
- this._state = new PipelineState(this._feeds, this._timeframeClock);
1180
- this._processingTrigger = new import_async4.Trigger().wake();
1181
- this._pauseTrigger = new import_async4.Trigger().wake();
1182
- this._downloads = new import_util4.ComplexMap((value) => import_keys4.PublicKey.hash(value.key));
1183
- this._isStopping = false;
1184
- this._isStarted = false;
1185
- this._isBeingConsumed = false;
1186
- this._isPaused = false;
1187
- }
1188
- get state() {
1189
- return this._state;
1190
- }
1191
- get writer() {
1192
- (0, import_invariant5.invariant)(this._writer, "Writer not set.", {
1193
- F: __dxlog_file7,
1194
- L: 242,
1195
- S: this,
1196
- A: [
1197
- "this._writer",
1198
- "'Writer not set.'"
1199
- ]
1200
- });
1201
- return this._writer;
1202
- }
1203
- hasFeed(feedKey) {
1204
- return this._feeds.has(feedKey);
1205
- }
1206
- getFeeds() {
1207
- return this._feedSetIterator.feeds;
1208
- }
1209
- // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
1210
- // which might be opening feeds during the mutation processing, which w
1211
- async addFeed(feed) {
1212
- this._feeds.set(feed.key, feed);
1213
- if (this._feedSetIterator) {
1214
- await this._feedSetIterator.addFeed(feed);
1215
- }
1216
- if (this._isStarted && !this._isPaused) {
1217
- this._setFeedDownloadState(feed);
1218
- }
1219
- }
1220
- setWriteFeed(feed) {
1221
- (0, import_invariant5.invariant)(!this._writer, "Writer already set.", {
1222
- F: __dxlog_file7,
1223
- L: 269,
1224
- S: this,
1225
- A: [
1226
- "!this._writer",
1227
- "'Writer already set.'"
1228
- ]
1229
- });
1230
- (0, import_invariant5.invariant)(feed.properties.writable, "Feed must be writable.", {
1231
- F: __dxlog_file7,
1232
- L: 270,
1233
- S: this,
1234
- A: [
1235
- "feed.properties.writable",
1236
- "'Feed must be writable.'"
1237
- ]
1238
- });
1239
- this._writer = createMappedFeedWriter((payload) => ({
1240
- timeframe: this._timeframeClock.timeframe,
1241
- payload
1242
- }), feed.createFeedWriter());
1243
- }
1244
- async start() {
1245
- (0, import_invariant5.invariant)(!this._isStarted, "Pipeline is already started.", {
1246
- F: __dxlog_file7,
1247
- L: 283,
1248
- S: this,
1249
- A: [
1250
- "!this._isStarted",
1251
- "'Pipeline is already started.'"
1252
- ]
1253
- });
1254
- (0, import_log5.log)("starting...", void 0, {
1255
- F: __dxlog_file7,
1256
- L: 284,
1257
- S: this,
1258
- C: (f, a) => f(...a)
1259
- });
1260
- await this._initIterator();
1261
- await this._feedSetIterator.open();
1262
- this._isStarted = true;
1263
- (0, import_log5.log)("started", void 0, {
1264
- F: __dxlog_file7,
1265
- L: 288,
1266
- S: this,
1267
- C: (f, a) => f(...a)
1268
- });
1269
- if (!this._isPaused) {
1270
- for (const feed of this._feeds.values()) {
1271
- this._setFeedDownloadState(feed);
1272
- }
1273
- }
1274
- }
1275
- async stop() {
1276
- (0, import_log5.log)("stopping...", void 0, {
1277
- F: __dxlog_file7,
1278
- L: 299,
1279
- S: this,
1280
- C: (f, a) => f(...a)
1281
- });
1282
- this._isStopping = true;
1283
- for (const [feed, handle] of this._downloads.entries()) {
1284
- feed.undownload(handle);
1285
- }
1286
- this._downloads.clear();
1287
- await this._feedSetIterator?.close();
1288
- await this._processingTrigger.wait();
1289
- await this._state._ctx.dispose();
1290
- this._state._ctx = new import_context4.Context();
1291
- this._state._reachedTargetPromise = void 0;
1292
- this._state._reachedTarget = false;
1293
- this._isStarted = false;
1294
- (0, import_log5.log)("stopped", void 0, {
1295
- F: __dxlog_file7,
1296
- L: 312,
1297
- S: this,
1298
- C: (f, a) => f(...a)
1299
- });
1300
- }
1301
- /**
1302
- * @param timeframe Timeframe of already processed messages.
1303
- * The pipeline will start processing messages AFTER this timeframe.
1304
- */
1305
- async setCursor(timeframe) {
1306
- (0, import_invariant5.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
1307
- F: __dxlog_file7,
1308
- L: 321,
1309
- S: this,
1310
- A: [
1311
- "!this._isStarted || this._isPaused",
1312
- "'Invalid state.'"
1313
- ]
1314
- });
1315
- this._state._startTimeframe = timeframe;
1316
- this._timeframeClock.setTimeframe(timeframe);
1317
- if (this._feedSetIterator) {
1318
- await this._feedSetIterator.close();
1319
- await this._initIterator();
1320
- await this._feedSetIterator.open();
1321
- }
1322
- }
1323
- /**
1324
- * Calling pause while processing will cause a deadlock.
1325
- */
1326
- async pause() {
1327
- if (this._isPaused) {
1328
- return;
1329
- }
1330
- this._pauseTrigger.reset();
1331
- await this._processingTrigger.wait();
1332
- this._isPaused = true;
1333
- }
1334
- async unpause() {
1335
- (0, import_invariant5.invariant)(this._isPaused, "Pipeline is not paused.", {
1336
- F: __dxlog_file7,
1337
- L: 350,
1338
- S: this,
1339
- A: [
1340
- "this._isPaused",
1341
- "'Pipeline is not paused.'"
1342
- ]
1343
- });
1344
- this._pauseTrigger.wake();
1345
- this._isPaused = false;
1346
- for (const feed of this._feeds.values()) {
1347
- this._setFeedDownloadState(feed);
1348
- }
1349
- }
1350
- /**
1351
- * Starts to iterate over the ordered messages from the added feeds.
1352
- * Updates the timeframe clock after the message has bee processed.
1353
- */
1354
- async *consume() {
1355
- (0, import_invariant5.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
1356
- F: __dxlog_file7,
1357
- L: 365,
1358
- S: this,
1359
- A: [
1360
- "!this._isBeingConsumed",
1361
- "'Pipeline is already being consumed.'"
1362
- ]
1363
- });
1364
- this._isBeingConsumed = true;
1365
- (0, import_invariant5.invariant)(this._feedSetIterator, "Iterator not initialized.", {
1366
- F: __dxlog_file7,
1367
- L: 368,
1368
- S: this,
1369
- A: [
1370
- "this._feedSetIterator",
1371
- "'Iterator not initialized.'"
1372
- ]
1373
- });
1374
- let lastFeedSetIterator = this._feedSetIterator;
1375
- let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1376
- while (!this._isStopping) {
1377
- await this._pauseTrigger.wait();
1378
- if (lastFeedSetIterator !== this._feedSetIterator) {
1379
- (0, import_invariant5.invariant)(this._feedSetIterator, "Iterator not initialized.", {
1380
- F: __dxlog_file7,
1381
- L: 377,
1382
- S: this,
1383
- A: [
1384
- "this._feedSetIterator",
1385
- "'Iterator not initialized.'"
1386
- ]
1387
- });
1388
- lastFeedSetIterator = this._feedSetIterator;
1389
- iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1390
- }
1391
- const { done, value } = await iterable.next();
1392
- if (!done) {
1393
- const block = value ?? (0, import_debug3.failUndefined)();
1394
- this._processingTrigger.reset();
1395
- this._timeframeClock.updatePendingTimeframe(import_keys4.PublicKey.from(block.feedKey), block.seq);
1396
- yield block;
1397
- this._processingTrigger.wake();
1398
- this._timeframeClock.updateTimeframe();
1399
- }
1400
- }
1401
- this._isBeingConsumed = false;
1402
- }
1403
- _setFeedDownloadState(feed) {
1404
- let handle = this._downloads.get(feed);
1405
- if (handle) {
1406
- feed.undownload(handle);
1407
- }
1408
- const timeframe = this._state._startTimeframe;
1409
- const seq = timeframe.get(feed.key) ?? -1;
1410
- (0, import_log5.log)("download", {
1411
- feed: feed.key.truncate(),
1412
- seq,
1413
- length: feed.length
1414
- }, {
1415
- F: __dxlog_file7,
1416
- L: 406,
1417
- S: this,
1418
- C: (f, a) => f(...a)
1419
- });
1420
- handle = feed.download({
1421
- start: seq + 1,
1422
- linear: true
1423
- }, (err, data) => {
1424
- if (err) {
1425
- } else {
1426
- import_log5.log.info("downloaded", {
1427
- data
1428
- }, {
1429
- F: __dxlog_file7,
1430
- L: 411,
1431
- S: this,
1432
- C: (f, a) => f(...a)
1433
- });
1434
- }
1435
- });
1436
- this._downloads.set(feed, handle);
1437
- }
1438
- async _initIterator() {
1439
- this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
1440
- start: startAfter(this._timeframeClock.timeframe),
1441
- stallTimeout: 1e3
1442
- });
1443
- this._feedSetIterator.stalled.on((iterator) => {
1444
- import_log5.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
1445
- F: __dxlog_file7,
1446
- L: 425,
1447
- S: this,
1448
- C: (f, a) => f(...a)
1449
- });
1450
- this._state.stalled.emit();
1451
- });
1452
- for (const feed of this._feeds.values()) {
1453
- await this._feedSetIterator.addFeed(feed);
1454
- }
1455
- }
1456
- };
1457
- _ts_decorate3([
1458
- import_async4.synchronized
1459
- ], Pipeline.prototype, "start", null);
1460
- _ts_decorate3([
1461
- import_async4.synchronized
1462
- ], Pipeline.prototype, "stop", null);
1463
- _ts_decorate3([
1464
- import_async4.synchronized
1465
- ], Pipeline.prototype, "setCursor", null);
1466
- _ts_decorate3([
1467
- import_async4.synchronized
1468
- ], Pipeline.prototype, "pause", null);
1469
- _ts_decorate3([
1470
- import_async4.synchronized
1471
- ], Pipeline.prototype, "unpause", null);
1472
- var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/auth.ts";
1473
- var AuthExtension = class extends import_teleport.RpcExtension {
1474
- constructor(_authParams) {
1475
- super({
1476
- requested: {
1477
- AuthService: import_protocols5.schema.getService("dxos.mesh.teleport.auth.AuthService")
1478
- },
1479
- exposed: {
1480
- AuthService: import_protocols5.schema.getService("dxos.mesh.teleport.auth.AuthService")
1481
- },
1482
- timeout: 60 * 1e3
1483
- });
1484
- this._authParams = _authParams;
1485
- this._ctx = new import_context5.Context({
1486
- onError: (err) => {
1487
- import_log7.log.catch(err, void 0, {
1488
- F: __dxlog_file8,
1489
- L: 28,
1490
- S: this,
1491
- C: (f, a) => f(...a)
1492
- });
1493
- }
1494
- });
1495
- }
1496
- async getHandlers() {
1497
- return {
1498
- AuthService: {
1499
- authenticate: async ({ challenge }) => {
1500
- try {
1501
- const credential = await this._authParams.provider(challenge);
1502
- if (!credential) {
1503
- throw new Error("auth rejected");
1504
- }
1505
- return {
1506
- credential
1507
- };
1508
- } catch (err) {
1509
- import_log7.log.error("failed to generate auth credentials", err, {
1510
- F: __dxlog_file8,
1511
- L: 55,
1512
- S: this,
1513
- C: (f, a) => f(...a)
1514
- });
1515
- throw new Error("auth rejected");
1516
- }
1517
- }
1518
- }
1519
- };
1520
- }
1521
- async onOpen(context) {
1522
- await super.onOpen(context);
1523
- (0, import_async5.scheduleTask)(this._ctx, async () => {
1524
- try {
1525
- const challenge = (0, import_crypto2.randomBytes)(32);
1526
- const { credential } = await this.rpc.AuthService.authenticate({
1527
- challenge
1528
- });
1529
- (0, import_invariant7.invariant)(credential?.length > 0, "invalid credential", {
1530
- F: __dxlog_file8,
1531
- L: 69,
1532
- S: this,
1533
- A: [
1534
- "credential?.length > 0",
1535
- "'invalid credential'"
1536
- ]
1537
- });
1538
- const success = await this._authParams.verifier(challenge, credential);
1539
- (0, import_invariant7.invariant)(success, "credential not verified", {
1540
- F: __dxlog_file8,
1541
- L: 71,
1542
- S: this,
1543
- A: [
1544
- "success",
1545
- "'credential not verified'"
1546
- ]
1547
- });
1548
- (0, import_async5.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
1549
- } catch (err) {
1550
- (0, import_log7.log)("auth failed", err, {
1551
- F: __dxlog_file8,
1552
- L: 74,
1553
- S: this,
1554
- C: (f, a) => f(...a)
1555
- });
1556
- this.close();
1557
- this._authParams.onAuthFailure();
1558
- }
1559
- });
1560
- }
1561
- async onClose() {
1562
- await this._ctx.dispose();
1563
- await super.onClose();
1564
- }
1565
- async onAbort() {
1566
- await this._ctx.dispose();
1567
- await super.onAbort();
1568
- }
1569
- };
1570
- function _ts_decorate4(decorators, target, key, desc) {
1571
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1572
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1573
- r = Reflect.decorate(decorators, target, key, desc);
1574
- else
1575
- for (var i = decorators.length - 1; i >= 0; i--)
1576
- if (d = decorators[i])
1577
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1578
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1579
- }
1580
- var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
1581
- var MESSAGES_PER_SNAPSHOT = 10;
1582
- var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
1583
- var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 5e3;
1584
- var DataPipeline = class {
1585
- constructor(_params) {
1586
- this._params = _params;
1587
- this._ctx = new import_context6.Context();
1588
- this._pipeline = void 0;
1589
- this._targetTimeframe = void 0;
1590
- this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
1591
- this._isOpen = false;
1592
- this._lastTimeframeSaveTime = 0;
1593
- this._lastSnapshotSaveTime = 0;
1594
- this._lastProcessedEpoch = -1;
1595
- this._usage = new import_tracing.TimeUsageCounter();
1596
- this._mutations = new import_tracing.TimeSeriesCounter();
1597
- this.currentEpoch = void 0;
1598
- this.appliedEpoch = void 0;
1599
- this.onNewEpoch = new import_async6.Event();
1600
- }
1601
- get isOpen() {
1602
- return this._isOpen;
1603
- }
1604
- get pipeline() {
1605
- return this._pipeline;
1606
- }
1607
- get pipelineState() {
1608
- return this._pipeline?.state;
1609
- }
1610
- setTargetTimeframe(timeframe) {
1611
- this._targetTimeframe = timeframe;
1612
- this._pipeline?.state.setTargetTimeframe(timeframe);
1613
- }
1614
- async processCredential(credential) {
1615
- if (!(0, import_credentials.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
1616
- return;
1617
- }
1618
- this.currentEpoch = credential;
1619
- if (this._isOpen) {
1620
- await this._processEpochInSeparateTask(credential);
1621
- }
1622
- }
1623
- async open() {
1624
- if (this._isOpen) {
1625
- return;
1626
- }
1627
- this._pipeline = new Pipeline();
1628
- await this._params.onPipelineCreated(this._pipeline);
1629
- await this._pipeline.pause();
1630
- await this._pipeline.start();
1631
- if (this._targetTimeframe) {
1632
- this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
1633
- }
1634
- const feedWriter = {
1635
- write: (data, options) => {
1636
- (0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
1637
- F: __dxlog_file9,
1638
- L: 164,
1639
- S: this,
1640
- A: [
1641
- "this._pipeline",
1642
- "'Pipeline is not initialized.'"
1643
- ]
1644
- });
1645
- (0, import_invariant8.invariant)(this.currentEpoch, "Epoch is not initialized.", {
1646
- F: __dxlog_file9,
1647
- L: 165,
1648
- S: this,
1649
- A: [
1650
- "this.currentEpoch",
1651
- "'Epoch is not initialized.'"
1652
- ]
1653
- });
1654
- return this._pipeline.writer.write({
1655
- data
1656
- }, options);
1657
- }
1658
- };
1659
- this.databaseHost = new DatabaseHost(feedWriter, () => this._flush());
1660
- this.itemManager = new import_echo_db3.ItemManager(this._params.modelFactory);
1661
- await this.databaseHost.open(this.itemManager, this._params.modelFactory);
1662
- (0, import_async6.scheduleTask)(this._ctx, async () => {
1663
- await this._consumePipeline();
1664
- });
1665
- this._isOpen = true;
1666
- }
1667
- async close() {
1668
- if (!this._isOpen) {
1669
- return;
1670
- }
1671
- (0, import_log8.log)("close", void 0, {
1672
- F: __dxlog_file9,
1673
- L: 189,
1674
- S: this,
1675
- C: (f, a) => f(...a)
1676
- });
1677
- this._isOpen = false;
1678
- await this._ctx.dispose();
1679
- await this._pipeline?.stop();
1680
- try {
1681
- await this._saveCache();
1682
- if (this._pipeline) {
1683
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1684
- }
1685
- } catch (err) {
1686
- import_log8.log.catch(err, void 0, {
1687
- F: __dxlog_file9,
1688
- L: 202,
1689
- S: this,
1690
- C: (f, a) => f(...a)
1691
- });
1692
- }
1693
- await this.databaseHost?.close();
1694
- await this.itemManager?.destroy();
1695
- this._ctx = new import_context6.Context();
1696
- this._pipeline = void 0;
1697
- this._targetTimeframe = void 0;
1698
- this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
1699
- this.currentEpoch = void 0;
1700
- this.appliedEpoch = void 0;
1701
- this._lastProcessedEpoch = -1;
1702
- this._epochCtx = void 0;
1703
- }
1704
- async _consumePipeline() {
1705
- const pipeline = this._pipeline;
1706
- if (this.currentEpoch) {
1707
- const waitForOneEpoch = this.onNewEpoch.waitForCount(1);
1708
- await this._processEpochInSeparateTask(this.currentEpoch);
1709
- await waitForOneEpoch;
1710
- }
1711
- let messageCounter = 0;
1712
- (0, import_invariant8.invariant)(pipeline, "Pipeline is not initialized.", {
1713
- F: __dxlog_file9,
1714
- L: 229,
1715
- S: this,
1716
- A: [
1717
- "pipeline",
1718
- "'Pipeline is not initialized.'"
1719
- ]
1720
- });
1721
- for await (const msg of pipeline.consume()) {
1722
- const span = this._usage.beginRecording();
1723
- this._mutations.inc();
1724
- const { feedKey, seq, data } = msg;
1725
- (0, import_log8.log)("processing message", {
1726
- feedKey,
1727
- seq
1728
- }, {
1729
- F: __dxlog_file9,
1730
- L: 235,
1731
- S: this,
1732
- C: (f, a) => f(...a)
1733
- });
1734
- try {
1735
- if (data.payload.data) {
1736
- const feedInfo = this._params.feedInfoProvider(feedKey);
1737
- if (!feedInfo) {
1738
- import_log8.log.warn("Could not find feed", {
1739
- feedKey
1740
- }, {
1741
- F: __dxlog_file9,
1742
- L: 241,
1743
- S: this,
1744
- C: (f, a) => f(...a)
1745
- });
1746
- continue;
1747
- }
1748
- const timer = import_util5.tracer.mark("dxos.echo.pipeline.data");
1749
- this.databaseHost.echoProcessor({
1750
- batch: data.payload.data.batch,
1751
- meta: {
1752
- feedKey,
1753
- seq,
1754
- timeframe: data.timeframe,
1755
- memberKey: feedInfo.assertion.identityKey
1756
- }
1757
- });
1758
- timer.end();
1759
- import_log8.log.trace("dxos.echo.data-pipeline.processed", {
1760
- feedKey: feedKey.toHex(),
1761
- seq,
1762
- spaceKey: this._params.spaceKey.toHex()
1763
- }, {
1764
- F: __dxlog_file9,
1765
- L: 258,
1766
- S: this,
1767
- C: (f, a) => f(...a)
1768
- });
1769
- await this._noteTargetStateIfNeeded(pipeline.state.pendingTimeframe);
1770
- }
1771
- } catch (err) {
1772
- import_log8.log.catch(err, void 0, {
1773
- F: __dxlog_file9,
1774
- L: 268,
1775
- S: this,
1776
- C: (f, a) => f(...a)
1777
- });
1778
- }
1779
- span.end();
1780
- if (++messageCounter > 100) {
1781
- messageCounter = 0;
1782
- await idle(1e3);
1783
- }
1784
- }
1785
- }
1786
- _createSnapshot() {
1787
- (0, import_invariant8.invariant)(this.databaseHost, "Database backend is not initialized.", {
1788
- F: __dxlog_file9,
1789
- L: 282,
1790
- S: this,
1791
- A: [
1792
- "this.databaseHost",
1793
- "'Database backend is not initialized.'"
1794
- ]
1795
- });
1796
- return {
1797
- spaceKey: this._params.spaceKey.asUint8Array(),
1798
- timeframe: this._pipeline.state.timeframe,
1799
- database: this.databaseHost.createSnapshot()
1800
- };
1801
- }
1802
- async _saveTargetTimeframe(timeframe) {
1803
- const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
1804
- await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
1805
- this._targetTimeframe = newTimeframe;
1806
- }
1807
- async _saveCache() {
1808
- const cache = {};
1809
- try {
1810
- const propertiesItem = this.itemManager.items.find((item) => item.modelMeta?.type === "dxos.org/model/document" && // TODO(burdon): Document?
1811
- ((0, import_echo_db3.getStateMachineFromItem)(item)?.snapshot()).type === import_echo_db3.TYPE_PROPERTIES);
1812
- if (propertiesItem) {
1813
- cache.properties = (0, import_echo_db3.getStateMachineFromItem)(propertiesItem)?.snapshot();
1814
- }
1815
- } catch (err) {
1816
- import_log8.log.warn("Failed to cache properties", err, {
1817
- F: __dxlog_file9,
1818
- L: 311,
1819
- S: this,
1820
- C: (f, a) => f(...a)
1821
- });
1822
- }
1823
- await this._params.metadataStore.setCache(this._params.spaceKey, cache);
1824
- }
1825
- async _noteTargetStateIfNeeded(timeframe) {
1826
- if (!this._pipeline?.state.reachedTarget) {
1827
- return;
1828
- }
1829
- if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
1830
- this._lastTimeframeSaveTime = Date.now();
1831
- await this._saveTargetTimeframe(timeframe);
1832
- }
1833
- if (Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL && timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
1834
- await this._saveCache();
1835
- }
1836
- }
1837
- async _processEpochInSeparateTask(epoch) {
1838
- if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
1839
- return;
1840
- }
1841
- await this._epochCtx?.dispose();
1842
- const ctx = new import_context6.Context({
1843
- onError: (err) => {
1844
- if (err instanceof import_protocols6.CancelledError) {
1845
- (0, import_log8.log)("Epoch processing cancelled.", void 0, {
1846
- F: __dxlog_file9,
1847
- L: 347,
1848
- S: this,
1849
- C: (f, a) => f(...a)
1850
- });
1851
- } else {
1852
- import_log8.log.catch(err, void 0, {
1853
- F: __dxlog_file9,
1854
- L: 349,
1855
- S: this,
1856
- C: (f, a) => f(...a)
1857
- });
1858
- }
1859
- }
1860
- });
1861
- this._epochCtx = ctx;
1862
- (0, import_async6.scheduleTask)(ctx, async () => {
1863
- if (!this._isOpen) {
1864
- return;
1865
- }
1866
- await this._processEpoch(ctx, epoch.subject.assertion);
1867
- if (epoch.subject.assertion.snapshotCid === void 0) {
1868
- epoch.subject.assertion.snapshotCid = this.appliedEpoch?.subject.assertion.snapshotCid;
1869
- }
1870
- this.appliedEpoch = epoch;
1871
- this.onNewEpoch.emit(epoch);
1872
- });
1873
- }
1874
- async _processEpoch(ctx, epoch) {
1875
- (0, import_invariant8.invariant)(this._isOpen, "Space is closed.", {
1876
- F: __dxlog_file9,
1877
- L: 373,
1878
- S: this,
1879
- A: [
1880
- "this._isOpen",
1881
- "'Space is closed.'"
1882
- ]
1883
- });
1884
- (0, import_invariant8.invariant)(this._pipeline, void 0, {
1885
- F: __dxlog_file9,
1886
- L: 374,
1887
- S: this,
1888
- A: [
1889
- "this._pipeline",
1890
- ""
1891
- ]
1892
- });
1893
- this._lastProcessedEpoch = epoch.number;
1894
- (0, import_log8.log)("processing", {
1895
- epoch: (0, import_log8.omit)(epoch, "proof")
1896
- }, {
1897
- F: __dxlog_file9,
1898
- L: 377,
1899
- S: this,
1900
- C: (f, a) => f(...a)
1901
- });
1902
- if (epoch.snapshotCid) {
1903
- const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
1904
- this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
1905
- }
1906
- (0, import_log8.log)("restarting pipeline from epoch", void 0, {
1907
- F: __dxlog_file9,
1908
- L: 383,
1909
- S: this,
1910
- C: (f, a) => f(...a)
1911
- });
1912
- await this._pipeline.pause();
1913
- await this._pipeline.setCursor(epoch.timeframe);
1914
- await this._pipeline.unpause();
1915
- }
1916
- async waitUntilTimeframe(timeframe) {
1917
- (0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
1918
- F: __dxlog_file9,
1919
- L: 390,
1920
- S: this,
1921
- A: [
1922
- "this._pipeline",
1923
- "'Pipeline is not initialized.'"
1924
- ]
1925
- });
1926
- await this._pipeline.state.waitUntilTimeframe(timeframe);
1927
- }
1928
- async createEpoch() {
1929
- (0, import_invariant8.invariant)(this._pipeline, void 0, {
1930
- F: __dxlog_file9,
1931
- L: 396,
1932
- S: this,
1933
- A: [
1934
- "this._pipeline",
1935
- ""
1936
- ]
1937
- });
1938
- (0, import_invariant8.invariant)(this.currentEpoch, void 0, {
1939
- F: __dxlog_file9,
1940
- L: 397,
1941
- S: this,
1942
- A: [
1943
- "this.currentEpoch",
1944
- ""
1945
- ]
1946
- });
1947
- await this._pipeline.pause();
1948
- const snapshot = await this._createSnapshot();
1949
- const snapshotCid = await this._params.snapshotManager.store(snapshot);
1950
- const epoch = {
1951
- previousId: this.currentEpoch.id,
1952
- timeframe: this._pipeline.state.timeframe,
1953
- number: this.currentEpoch.subject.assertion.number + 1,
1954
- snapshotCid
1955
- };
1956
- await this._pipeline.unpause();
1957
- return epoch;
1958
- }
1959
- async ensureEpochInitialized() {
1960
- await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
1961
- }
1962
- async _flush() {
1963
- try {
1964
- await this._saveCache();
1965
- if (this._pipeline) {
1966
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1967
- }
1968
- } catch (err) {
1969
- import_log8.log.catch(err, void 0, {
1970
- F: __dxlog_file9,
1971
- L: 427,
1972
- S: this,
1973
- C: (f, a) => f(...a)
1974
- });
1975
- }
1976
- await this._params.metadataStore.flush();
1977
- }
1978
- };
1979
- _ts_decorate4([
1980
- import_tracing.trace.metricsCounter()
1981
- ], DataPipeline.prototype, "_usage", void 0);
1982
- _ts_decorate4([
1983
- import_tracing.trace.metricsCounter()
1984
- ], DataPipeline.prototype, "_mutations", void 0);
1985
- _ts_decorate4([
1986
- import_async6.synchronized
1987
- ], DataPipeline.prototype, "open", null);
1988
- _ts_decorate4([
1989
- import_async6.synchronized
1990
- ], DataPipeline.prototype, "close", null);
1991
- _ts_decorate4([
1992
- import_async6.synchronized
1993
- ], DataPipeline.prototype, "_processEpoch", null);
1994
- _ts_decorate4([
1995
- import_async6.synchronized
1996
- ], DataPipeline.prototype, "createEpoch", null);
1997
- DataPipeline = _ts_decorate4([
1998
- (0, import_async6.trackLeaks)("open", "close"),
1999
- import_tracing.trace.resource()
2000
- ], DataPipeline);
2001
- var idle = async (timeout) => {
2002
- if (!("scheduler" in globalThis && typeof globalThis.scheduler.postTask === "function")) {
2003
- await (0, import_async6.sleep)(1);
2004
- return;
2005
- }
2006
- await new Promise((resolve) => {
2007
- const cleanup = () => {
2008
- clearTimeout(timer);
2009
- controller.abort();
2010
- };
2011
- const controller = new AbortController();
2012
- void globalThis.scheduler.postTask(() => {
2013
- cleanup();
2014
- resolve();
2015
- }, {
2016
- priority: "background",
2017
- signal: controller.signal
2018
- }).catch(() => {
2019
- });
2020
- const timer = setTimeout(() => {
2021
- cleanup();
2022
- resolve();
2023
- }, timeout);
2024
- });
2025
- };
2026
- function _ts_decorate5(decorators, target, key, desc) {
2027
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2028
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2029
- r = Reflect.decorate(decorators, target, key, desc);
2030
- else
2031
- for (var i = decorators.length - 1; i >= 0; i--)
2032
- if (d = decorators[i])
2033
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2034
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2035
- }
2036
- var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
2037
- var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
2038
- var CONTROL_PIPELINE_SNAPSHOT_DELAY = 1e4;
2039
- var USE_SNAPSHOTS = true;
2040
- var ControlPipeline = class {
2041
- constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
2042
- this._ctx = new import_context7.Context();
2043
- this._lastTimeframeSaveTime = Date.now();
2044
- this.onFeedAdmitted = new import_util7.Callback();
2045
- this._usage = new import_tracing3.TimeUsageCounter();
2046
- this._mutations = new import_tracing3.TimeSeriesCounter();
2047
- this._snapshotTask = new import_async8.DeferredTask(this._ctx, async () => {
2048
- await (0, import_async8.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
2049
- await this._saveSnapshot();
2050
- });
2051
- this._spaceKey = spaceKey;
2052
- this._metadata = metadataStore;
2053
- this._pipeline = new Pipeline();
2054
- void this._pipeline.addFeed(genesisFeed);
2055
- this._spaceStateMachine = new import_credentials3.SpaceStateMachine(spaceKey);
2056
- this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
2057
- (0, import_log10.log)("feed admitted", {
2058
- key: info.key
2059
- }, {
2060
- F: __dxlog_file10,
2061
- L: 74,
2062
- S: this,
2063
- C: (f, a) => f(...a)
2064
- });
2065
- if (info.assertion.designation === import_credentials4.AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
2066
- queueMicrotask(async () => {
2067
- try {
2068
- const feed = await feedProvider(info.key);
2069
- await this._pipeline.addFeed(feed);
2070
- } catch (err) {
2071
- import_log10.log.catch(err, void 0, {
2072
- F: __dxlog_file10,
2073
- L: 83,
2074
- S: this,
2075
- C: (f, a) => f(...a)
2076
- });
2077
- }
2078
- });
2079
- }
2080
- await this.onFeedAdmitted.callIfSet(info);
2081
- });
2082
- this.onMemberAdmitted = this._spaceStateMachine.onMemberAdmitted;
2083
- this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
2084
- }
2085
- get spaceState() {
2086
- return this._spaceStateMachine;
2087
- }
2088
- get pipeline() {
2089
- return this._pipeline;
2090
- }
2091
- async setWriteFeed(feed) {
2092
- await this._pipeline.addFeed(feed);
2093
- this._pipeline.setWriteFeed(feed);
2094
- }
2095
- async start() {
2096
- const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
2097
- (0, import_log10.log)("load snapshot", {
2098
- key: this._spaceKey,
2099
- present: !!snapshot,
2100
- tf: snapshot?.timeframe
2101
- }, {
2102
- F: __dxlog_file10,
2103
- L: 111,
2104
- S: this,
2105
- C: (f, a) => f(...a)
2106
- });
2107
- if (USE_SNAPSHOTS && snapshot) {
2108
- await this._processSnapshot(snapshot);
2109
- }
2110
- (0, import_log10.log)("starting...", void 0, {
2111
- F: __dxlog_file10,
2112
- L: 116,
2113
- S: this,
2114
- C: (f, a) => f(...a)
2115
- });
2116
- setTimeout(async () => {
2117
- void this._consumePipeline(new import_context7.Context());
2118
- });
2119
- await this._pipeline.start();
2120
- (0, import_log10.log)("started", void 0, {
2121
- F: __dxlog_file10,
2122
- L: 122,
2123
- S: this,
2124
- C: (f, a) => f(...a)
2125
- });
2126
- }
2127
- async _processSnapshot(snapshot) {
2128
- await this._pipeline.setCursor(snapshot.timeframe);
2129
- for (const message of snapshot.messages ?? []) {
2130
- const result = await this._spaceStateMachine.process(message.credential, {
2131
- sourceFeed: message.feedKey,
2132
- skipVerification: true
2133
- });
2134
- if (!result) {
2135
- import_log10.log.warn("credential processing failed from snapshot", {
2136
- message
2137
- }, {
2138
- F: __dxlog_file10,
2139
- L: 135,
2140
- S: this,
2141
- C: (f, a) => f(...a)
2142
- });
2143
- }
2144
- }
2145
- }
2146
- async _saveSnapshot() {
2147
- await this._pipeline.pause();
2148
- const snapshot = {
2149
- timeframe: this._pipeline.state.timeframe,
2150
- messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
2151
- feedKey: entry.sourceFeed,
2152
- credential: entry.credential
2153
- }))
2154
- };
2155
- await this._pipeline.unpause();
2156
- (0, import_log10.log)("save snapshot", {
2157
- key: this._spaceKey,
2158
- snapshot
2159
- }, {
2160
- F: __dxlog_file10,
2161
- L: 151,
2162
- S: this,
2163
- C: (f, a) => f(...a)
2164
- });
2165
- await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
2166
- }
2167
- async _consumePipeline(ctx) {
2168
- for await (const msg of this._pipeline.consume()) {
2169
- const span = this._usage.beginRecording();
2170
- this._mutations.inc();
2171
- try {
2172
- await this._processMessage(ctx, msg);
2173
- } catch (err) {
2174
- import_log10.log.catch(err, void 0, {
2175
- F: __dxlog_file10,
2176
- L: 164,
2177
- S: this,
2178
- C: (f, a) => f(...a)
2179
- });
2180
- }
2181
- span.end();
2182
- }
2183
- }
2184
- async _processMessage(ctx, msg) {
2185
- (0, import_log10.log)("processing", {
2186
- key: msg.feedKey,
2187
- seq: msg.seq
2188
- }, {
2189
- F: __dxlog_file10,
2190
- L: 174,
2191
- S: this,
2192
- C: (f, a) => f(...a)
2193
- });
2194
- if (msg.data.payload.credential) {
2195
- const timer = import_util7.tracer.mark("dxos.echo.pipeline.control");
2196
- const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
2197
- sourceFeed: import_keys5.PublicKey.from(msg.feedKey)
2198
- });
2199
- timer.end();
2200
- if (!result) {
2201
- import_log10.log.warn("processing failed", {
2202
- msg
2203
- }, {
2204
- F: __dxlog_file10,
2205
- L: 183,
2206
- S: this,
2207
- C: (f, a) => f(...a)
2208
- });
2209
- } else {
2210
- await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
2211
- }
2212
- this._snapshotTask.schedule();
2213
- }
2214
- }
2215
- async _noteTargetStateIfNeeded(timeframe) {
2216
- if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2) {
2217
- this._lastTimeframeSaveTime = Date.now();
2218
- await this._saveTargetTimeframe(timeframe);
2219
- }
2220
- }
2221
- async stop() {
2222
- (0, import_log10.log)("stopping...", void 0, {
2223
- F: __dxlog_file10,
2224
- L: 203,
2225
- S: this,
2226
- C: (f, a) => f(...a)
2227
- });
2228
- await this._ctx.dispose();
2229
- await this._pipeline.stop();
2230
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
2231
- (0, import_log10.log)("stopped", void 0, {
2232
- F: __dxlog_file10,
2233
- L: 207,
2234
- S: this,
2235
- C: (f, a) => f(...a)
2236
- });
2237
- }
2238
- async _saveTargetTimeframe(timeframe) {
2239
- try {
2240
- const newTimeframe = import_timeframe4.Timeframe.merge(this._targetTimeframe ?? new import_timeframe4.Timeframe(), timeframe);
2241
- await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
2242
- this._targetTimeframe = newTimeframe;
2243
- } catch (err) {
2244
- (0, import_log10.log)(err, void 0, {
2245
- F: __dxlog_file10,
2246
- L: 216,
2247
- S: this,
2248
- C: (f, a) => f(...a)
2249
- });
2250
- }
2251
- }
2252
- };
2253
- _ts_decorate5([
2254
- import_tracing3.trace.metricsCounter()
2255
- ], ControlPipeline.prototype, "_usage", void 0);
2256
- _ts_decorate5([
2257
- import_tracing3.trace.metricsCounter()
2258
- ], ControlPipeline.prototype, "_mutations", void 0);
2259
- _ts_decorate5([
2260
- import_tracing3.trace.span({
2261
- showInBrowserTimeline: true
2262
- })
2263
- ], ControlPipeline.prototype, "start", null);
2264
- _ts_decorate5([
2265
- import_tracing3.trace.span()
2266
- ], ControlPipeline.prototype, "_consumePipeline", null);
2267
- _ts_decorate5([
2268
- import_tracing3.trace.span()
2269
- ], ControlPipeline.prototype, "_processMessage", null);
2270
- ControlPipeline = _ts_decorate5([
2271
- import_tracing3.trace.resource(),
2272
- (0, import_async8.trackLeaks)("start", "stop")
2273
- ], ControlPipeline);
2274
- function _ts_decorate6(decorators, target, key, desc) {
2275
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2276
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2277
- r = Reflect.decorate(decorators, target, key, desc);
2278
- else
2279
- for (var i = decorators.length - 1; i >= 0; i--)
2280
- if (d = decorators[i])
2281
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2282
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2283
- }
2284
- var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
2285
- var Space = class {
2286
- constructor(params) {
2287
- this._addFeedMutex = new import_async7.Mutex();
2288
- this.onCredentialProcessed = new import_util6.Callback();
2289
- this.stateUpdate = new import_async7.Event();
2290
- this._isOpen = false;
2291
- (0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
2292
- F: __dxlog_file11,
2293
- L: 78,
2294
- S: this,
2295
- A: [
2296
- "params.spaceKey && params.feedProvider",
2297
- ""
2298
- ]
2299
- });
2300
- this._key = params.spaceKey;
2301
- this._genesisFeedKey = params.genesisFeed.key;
2302
- this._feedProvider = params.feedProvider;
2303
- this._snapshotManager = params.snapshotManager;
2304
- this._controlPipeline = new ControlPipeline({
2305
- spaceKey: params.spaceKey,
2306
- genesisFeed: params.genesisFeed,
2307
- feedProvider: params.feedProvider,
2308
- metadataStore: params.metadataStore
2309
- });
2310
- this._controlPipeline.onFeedAdmitted.set(async (info) => {
2311
- const sparse = info.assertion.designation === import_credentials2.AdmittedFeed.Designation.DATA;
2312
- if (info.assertion.designation === import_credentials2.AdmittedFeed.Designation.DATA) {
2313
- queueMicrotask(async () => {
2314
- if (this._dataPipeline.pipeline) {
2315
- if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
2316
- return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key, {
2317
- sparse
2318
- }));
2319
- }
2320
- }
2321
- });
2322
- }
2323
- if (!info.key.equals(params.genesisFeed.key)) {
2324
- queueMicrotask(async () => {
2325
- this.protocol.addFeed(await params.feedProvider(info.key, {
2326
- sparse
2327
- }));
2328
- });
2329
- }
2330
- });
2331
- this._controlPipeline.onCredentialProcessed.set(async (credential) => {
2332
- await this.onCredentialProcessed.callIfSet(credential);
2333
- (0, import_log9.log)("onCredentialProcessed", {
2334
- credential
2335
- }, {
2336
- F: __dxlog_file11,
2337
- L: 116,
2338
- S: this,
2339
- C: (f, a) => f(...a)
2340
- });
2341
- this.stateUpdate.emit();
2342
- });
2343
- this.protocol = params.protocol;
2344
- this.protocol.addFeed(params.genesisFeed);
2345
- this._dataPipeline = new DataPipeline({
2346
- modelFactory: params.modelFactory,
2347
- metadataStore: params.metadataStore,
2348
- snapshotManager: params.snapshotManager,
2349
- memberKey: params.memberKey,
2350
- spaceKey: this._key,
2351
- feedInfoProvider: (feedKey) => this._controlPipeline.spaceState.feeds.get(feedKey),
2352
- snapshotId: params.snapshotId,
2353
- onPipelineCreated: async (pipeline) => {
2354
- if (this._dataFeed) {
2355
- pipeline.setWriteFeed(this._dataFeed);
2356
- }
2357
- await this._addFeedMutex.executeSynchronized(async () => {
2358
- for (const feed of this._controlPipeline.spaceState.feeds.values()) {
2359
- if (feed.assertion.designation === import_credentials2.AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
2360
- await pipeline.addFeed(await this._feedProvider(feed.key, {
2361
- sparse: true
2362
- }));
2363
- }
2364
- }
2365
- });
2366
- }
2367
- });
2368
- }
2369
- get key() {
2370
- return this._key;
2371
- }
2372
- get isOpen() {
2373
- return this._isOpen;
2374
- }
2375
- get genesisFeedKey() {
2376
- return this._genesisFeedKey;
2377
- }
2378
- get controlFeedKey() {
2379
- return this._controlFeed?.key;
2380
- }
2381
- get dataFeedKey() {
2382
- return this._dataFeed?.key;
2383
- }
2384
- get spaceState() {
2385
- return this._controlPipeline.spaceState;
2386
- }
2387
- /**
2388
- * @test-only
2389
- */
2390
- get controlPipeline() {
2391
- return this._controlPipeline.pipeline;
2392
- }
2393
- get dataPipeline() {
2394
- return this._dataPipeline;
2395
- }
2396
- get snapshotManager() {
2397
- return this._snapshotManager;
2398
- }
2399
- async setControlFeed(feed) {
2400
- (0, import_invariant9.invariant)(!this._controlFeed, "Control feed already set.", {
2401
- F: __dxlog_file11,
2402
- L: 191,
2403
- S: this,
2404
- A: [
2405
- "!this._controlFeed",
2406
- "'Control feed already set.'"
2407
- ]
2408
- });
2409
- this._controlFeed = feed;
2410
- await this._controlPipeline.setWriteFeed(feed);
2411
- return this;
2412
- }
2413
- async setDataFeed(feed) {
2414
- (0, import_invariant9.invariant)(!this._dataFeed, "Data feed already set.", {
2415
- F: __dxlog_file11,
2416
- L: 198,
2417
- S: this,
2418
- A: [
2419
- "!this._dataFeed",
2420
- "'Data feed already set.'"
2421
- ]
2422
- });
2423
- this._dataFeed = feed;
2424
- await this._dataPipeline.pipeline?.addFeed(feed);
2425
- this._dataPipeline.pipeline?.setWriteFeed(feed);
2426
- return this;
2427
- }
2428
- /**
2429
- * Use for diagnostics.
2430
- */
2431
- getControlFeeds() {
2432
- return Array.from(this._controlPipeline.spaceState.feeds.values());
2433
- }
2434
- /**
2435
- * Use for diagnostics.
2436
- */
2437
- // getDataFeeds(): FeedInfo[] {
2438
- // return this._dataPipeline?.getFeeds();
2439
- // }
2440
- async open(ctx) {
2441
- (0, import_log9.log)("opening...", void 0, {
2442
- F: __dxlog_file11,
2443
- L: 221,
2444
- S: this,
2445
- C: (f, a) => f(...a)
2446
- });
2447
- if (this._isOpen) {
2448
- return;
2449
- }
2450
- await this._controlPipeline.start();
2451
- await this.protocol.start();
2452
- await this._controlPipeline.spaceState.addCredentialProcessor(this._dataPipeline);
2453
- this._isOpen = true;
2454
- (0, import_log9.log)("opened", void 0, {
2455
- F: __dxlog_file11,
2456
- L: 232,
2457
- S: this,
2458
- C: (f, a) => f(...a)
2459
- });
2460
- }
2461
- async close() {
2462
- (0, import_log9.log)("closing...", {
2463
- key: this._key
2464
- }, {
2465
- F: __dxlog_file11,
2466
- L: 237,
2467
- S: this,
2468
- C: (f, a) => f(...a)
2469
- });
2470
- if (!this._isOpen) {
2471
- return;
2472
- }
2473
- await this._controlPipeline.spaceState.removeCredentialProcessor(this._dataPipeline);
2474
- await this._dataPipeline.close();
2475
- await this.protocol.stop();
2476
- await this._controlPipeline.stop();
2477
- this._isOpen = false;
2478
- (0, import_log9.log)("closed", void 0, {
2479
- F: __dxlog_file11,
2480
- L: 250,
2481
- S: this,
2482
- C: (f, a) => f(...a)
2483
- });
2484
- }
2485
- async initializeDataPipeline() {
2486
- (0, import_log9.log)("initializeDataPipeline", void 0, {
2487
- F: __dxlog_file11,
2488
- L: 255,
2489
- S: this,
2490
- C: (f, a) => f(...a)
2491
- });
2492
- (0, import_invariant9.invariant)(this._isOpen, "Space must be open to initialize data pipeline.", {
2493
- F: __dxlog_file11,
2494
- L: 256,
2495
- S: this,
2496
- A: [
2497
- "this._isOpen",
2498
- "'Space must be open to initialize data pipeline.'"
2499
- ]
2500
- });
2501
- await this._dataPipeline.open();
2502
- }
2503
- };
2504
- _ts_decorate6([
2505
- import_tracing2.trace.info()
2506
- ], Space.prototype, "protocol", void 0);
2507
- _ts_decorate6([
2508
- import_tracing2.trace.info()
2509
- ], Space.prototype, "_controlPipeline", void 0);
2510
- _ts_decorate6([
2511
- import_tracing2.trace.info()
2512
- ], Space.prototype, "_dataPipeline", void 0);
2513
- _ts_decorate6([
2514
- import_log9.logInfo,
2515
- import_tracing2.trace.info()
2516
- ], Space.prototype, "key", null);
2517
- _ts_decorate6([
2518
- import_async7.synchronized,
2519
- import_tracing2.trace.span()
2520
- ], Space.prototype, "open", null);
2521
- _ts_decorate6([
2522
- import_async7.synchronized
2523
- ], Space.prototype, "close", null);
2524
- _ts_decorate6([
2525
- import_async7.synchronized
2526
- ], Space.prototype, "initializeDataPipeline", null);
2527
- Space = _ts_decorate6([
2528
- (0, import_async7.trackLeaks)("open", "close"),
2529
- import_tracing2.trace.resource()
2530
- ], Space);
2531
- function _ts_decorate7(decorators, target, key, desc) {
2532
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2533
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2534
- r = Reflect.decorate(decorators, target, key, desc);
2535
- else
2536
- for (var i = decorators.length - 1; i >= 0; i--)
2537
- if (d = decorators[i])
2538
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2539
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2540
- }
2541
- var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
2542
- var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
2543
- var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
2544
- var SpaceProtocol = class {
2545
- constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore }) {
2546
- this._feeds = /* @__PURE__ */ new Set();
2547
- this._sessions = new import_util8.ComplexMap(import_keys6.PublicKey.hash);
2548
- this._spaceKey = topic;
2549
- this._networkManager = networkManager;
2550
- this._swarmIdentity = swarmIdentity;
2551
- this._onSessionAuth = onSessionAuth;
2552
- this._onAuthFailure = onAuthFailure;
2553
- this.blobSync = new import_teleport_extension_object_sync.BlobSync({
2554
- blobStore
2555
- });
2556
- this._topic = import_crypto3.subtleCrypto.digest("SHA-256", topic.asBuffer()).then(import_crypto3.discoveryKey).then(import_keys6.PublicKey.from);
2557
- }
2558
- get sessions() {
2559
- return this._sessions;
2560
- }
2561
- get feeds() {
2562
- return this._feeds;
2563
- }
2564
- get _ownPeerKey() {
2565
- return this._swarmIdentity.peerKey;
2566
- }
2567
- // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
2568
- addFeed(feed) {
2569
- (0, import_log11.log)("addFeed", {
2570
- key: feed.key
2571
- }, {
2572
- F: __dxlog_file12,
2573
- L: 103,
2574
- S: this,
2575
- C: (f, a) => f(...a)
2576
- });
2577
- this._feeds.add(feed);
2578
- for (const session of this._sessions.values()) {
2579
- session.replicator.addFeed(feed);
2580
- }
2581
- }
2582
- // TODO(burdon): Rename open? Common open/close interfaces for all services?
2583
- async start() {
2584
- if (this._connection) {
2585
- return;
2586
- }
2587
- const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(""));
2588
- const topologyConfig = {
2589
- originateConnections: 4,
2590
- maxPeers: 10,
2591
- sampleSize: 20
2592
- };
2593
- await this.blobSync.open();
2594
- (0, import_log11.log)("starting...", void 0, {
2595
- F: __dxlog_file12,
2596
- L: 129,
2597
- S: this,
2598
- C: (f, a) => f(...a)
2599
- });
2600
- const topic = await this._topic;
2601
- this._connection = await this._networkManager.joinSwarm({
2602
- protocolProvider: this._createProtocolProvider(credentials),
2603
- peerId: this._swarmIdentity.peerKey,
2604
- topic,
2605
- topology: new import_network_manager.MMSTTopology(topologyConfig),
2606
- label: `swarm ${topic.truncate()} for space ${this._spaceKey.truncate()}`
2607
- });
2608
- (0, import_log11.log)("started", void 0, {
2609
- F: __dxlog_file12,
2610
- L: 139,
2611
- S: this,
2612
- C: (f, a) => f(...a)
2613
- });
2614
- }
2615
- async stop() {
2616
- await this.blobSync.close();
2617
- if (this._connection) {
2618
- (0, import_log11.log)("stopping...", void 0, {
2619
- F: __dxlog_file12,
2620
- L: 146,
2621
- S: this,
2622
- C: (f, a) => f(...a)
2623
- });
2624
- await this._connection.close();
2625
- (0, import_log11.log)("stopped", void 0, {
2626
- F: __dxlog_file12,
2627
- L: 148,
2628
- S: this,
2629
- C: (f, a) => f(...a)
2630
- });
2631
- }
2632
- }
2633
- _createProtocolProvider(credentials) {
2634
- return (wireParams) => {
2635
- const session = new SpaceProtocolSession({
2636
- wireParams,
2637
- swarmIdentity: this._swarmIdentity,
2638
- onSessionAuth: this._onSessionAuth,
2639
- onAuthFailure: this._onAuthFailure,
2640
- blobSync: this.blobSync
2641
- });
2642
- this._sessions.set(wireParams.remotePeerId, session);
2643
- for (const feed of this._feeds) {
2644
- session.replicator.addFeed(feed);
2645
- }
2646
- return session;
2647
- };
2648
- }
2649
- };
2650
- _ts_decorate7([
2651
- import_log11.logInfo,
2652
- import_tracing4.trace.info()
2653
- ], SpaceProtocol.prototype, "_topic", void 0);
2654
- _ts_decorate7([
2655
- import_tracing4.trace.info()
2656
- ], SpaceProtocol.prototype, "_spaceKey", void 0);
2657
- _ts_decorate7([
2658
- import_log11.logInfo
2659
- ], SpaceProtocol.prototype, "_ownPeerKey", null);
2660
- SpaceProtocol = _ts_decorate7([
2661
- import_tracing4.trace.resource()
2662
- ], SpaceProtocol);
2663
- var AuthStatus;
2664
- (function(AuthStatus2) {
2665
- AuthStatus2["INITIAL"] = "INITIAL";
2666
- AuthStatus2["SUCCESS"] = "SUCCESS";
2667
- AuthStatus2["FAILURE"] = "FAILURE";
2668
- })(AuthStatus || (AuthStatus = {}));
2669
- var SpaceProtocolSession = class {
2670
- // TODO(dmaretskyi): Allow to pass in extra extensions.
2671
- constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync }) {
2672
- this.replicator = new import_teleport_extension_replicator.ReplicatorExtension().setOptions({
2673
- upload: true
2674
- });
2675
- this._authStatus = "INITIAL";
2676
- this._wireParams = wireParams;
2677
- this._swarmIdentity = swarmIdentity;
2678
- this._onSessionAuth = onSessionAuth;
2679
- this._onAuthFailure = onAuthFailure;
2680
- this._blobSync = blobSync;
2681
- this._teleport = new import_teleport2.Teleport(wireParams);
2682
- }
2683
- get authStatus() {
2684
- return this._authStatus;
2685
- }
2686
- get stats() {
2687
- return this._teleport.stats;
2688
- }
2689
- get stream() {
2690
- return this._teleport.stream;
2691
- }
2692
- async open(sessionId) {
2693
- await this._teleport.open(sessionId);
2694
- this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
2695
- provider: this._swarmIdentity.credentialProvider,
2696
- verifier: this._swarmIdentity.credentialAuthenticator,
2697
- onAuthSuccess: () => {
2698
- (0, import_log11.log)("Peer authenticated", void 0, {
2699
- F: __dxlog_file12,
2700
- L: 245,
2701
- S: this,
2702
- C: (f, a) => f(...a)
2703
- });
2704
- this._authStatus = "SUCCESS";
2705
- this._onSessionAuth?.(this._teleport);
2706
- },
2707
- onAuthFailure: () => {
2708
- this._authStatus = "FAILURE";
2709
- this._onAuthFailure?.(this._teleport);
2710
- }
2711
- }));
2712
- this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
2713
- this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
2714
- }
2715
- async close() {
2716
- (0, import_log11.log)("close", void 0, {
2717
- F: __dxlog_file12,
2718
- L: 261,
2719
- S: this,
2720
- C: (f, a) => f(...a)
2721
- });
2722
- await this._teleport.close();
2723
- }
2724
- async abort() {
2725
- await this._teleport.abort();
2726
- }
2727
- };
2728
- _ts_decorate7([
2729
- import_log11.logInfo
2730
- ], SpaceProtocolSession.prototype, "_wireParams", void 0);
2731
- _ts_decorate7([
2732
- import_log11.logInfo
2733
- ], SpaceProtocolSession.prototype, "authStatus", null);
2734
- function _ts_decorate8(decorators, target, key, desc) {
2735
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2736
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2737
- r = Reflect.decorate(decorators, target, key, desc);
2738
- else
2739
- for (var i = decorators.length - 1; i >= 0; i--)
2740
- if (d = decorators[i])
2741
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2742
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2743
- }
2744
- var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
2745
- var SpaceManager = class {
2746
- constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
2747
- this._spaces = new import_util9.ComplexMap(import_keys7.PublicKey.hash);
2748
- this._instanceId = import_keys7.PublicKey.random().toHex();
2749
- this._feedStore = feedStore;
2750
- this._networkManager = networkManager;
2751
- this._modelFactory = modelFactory;
2752
- this._metadataStore = metadataStore;
2753
- this._snapshotStore = snapshotStore;
2754
- this._blobStore = blobStore;
2755
- }
2756
- // TODO(burdon): Remove.
2757
- get spaces() {
2758
- return this._spaces;
2759
- }
2760
- async open() {
2761
- }
2762
- async close() {
2763
- await Promise.all([
2764
- ...this._spaces.values()
2765
- ].map((space) => space.close()));
2766
- }
2767
- async constructSpace({ metadata, swarmIdentity, onAuthorizedConnection, onAuthFailure, memberKey }) {
2768
- import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols7.trace.begin({
2769
- id: this._instanceId
2770
- }), {
2771
- F: __dxlog_file13,
2772
- L: 100,
2773
- S: this,
2774
- C: (f, a) => f(...a)
2775
- });
2776
- (0, import_log12.log)("constructing space...", {
2777
- spaceKey: metadata.genesisFeedKey
2778
- }, {
2779
- F: __dxlog_file13,
2780
- L: 101,
2781
- S: this,
2782
- C: (f, a) => f(...a)
2783
- });
2784
- const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? (0, import_debug4.failUndefined)());
2785
- const spaceKey = metadata.key;
2786
- const protocol = new SpaceProtocol({
2787
- topic: spaceKey,
2788
- swarmIdentity,
2789
- networkManager: this._networkManager,
2790
- onSessionAuth: onAuthorizedConnection,
2791
- onAuthFailure,
2792
- blobStore: this._blobStore
2793
- });
2794
- const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
2795
- const space = new Space({
2796
- spaceKey,
2797
- protocol,
2798
- genesisFeed,
2799
- feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
2800
- modelFactory: this._modelFactory,
2801
- metadataStore: this._metadataStore,
2802
- snapshotManager,
2803
- memberKey
2804
- });
2805
- this._spaces.set(space.key, space);
2806
- import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols7.trace.end({
2807
- id: this._instanceId
2808
- }), {
2809
- F: __dxlog_file13,
2810
- L: 129,
2811
- S: this,
2812
- C: (f, a) => f(...a)
2813
- });
2814
- return space;
2815
- }
2816
- };
2817
- _ts_decorate8([
2818
- import_async9.synchronized
2819
- ], SpaceManager.prototype, "open", null);
2820
- _ts_decorate8([
2821
- import_async9.synchronized
2822
- ], SpaceManager.prototype, "close", null);
2823
- SpaceManager = _ts_decorate8([
2824
- (0, import_async9.trackLeaks)("open", "close")
2825
- ], SpaceManager);
2826
- var AutomergeStorageAdapter = class {
2827
- constructor(_directory) {
2828
- this._directory = _directory;
2829
- this._state = "opened";
2830
- }
2831
- async load(key) {
2832
- if (this._state !== "opened") {
2833
- return void 0;
2834
- }
2835
- const filename = this._getFilename(key);
2836
- const file = this._directory.getOrCreateFile(filename);
2837
- const { size } = await file.stat();
2838
- if (!size || size === 0) {
2839
- return void 0;
2840
- }
2841
- const buffer = await file.read(0, size);
2842
- return (0, import_util10.bufferToArray)(buffer);
2843
- }
2844
- async save(key, data) {
2845
- if (this._state !== "opened") {
2846
- return void 0;
2847
- }
2848
- const filename = this._getFilename(key);
2849
- const file = this._directory.getOrCreateFile(filename);
2850
- await file.write(0, (0, import_util10.arrayToBuffer)(data));
2851
- await file.truncate?.(data.length);
2852
- await file.flush?.();
2853
- }
2854
- async remove(key) {
2855
- if (this._state !== "opened") {
2856
- return void 0;
2857
- }
2858
- const filename = this._getFilename(key);
2859
- const file = this._directory.getOrCreateFile(filename);
2860
- await file.destroy();
2861
- }
2862
- async loadRange(keyPrefix) {
2863
- if (this._state !== "opened") {
2864
- return [];
2865
- }
2866
- const filename = this._getFilename(keyPrefix);
2867
- const entries = await this._directory.list();
2868
- return Promise.all(entries.filter((entry) => entry.startsWith(filename)).map(async (entry) => {
2869
- const file = this._directory.getOrCreateFile(entry);
2870
- const { size } = await file.stat();
2871
- const buffer = await file.read(0, size);
2872
- return {
2873
- key: this._getKeyFromFilename(entry),
2874
- data: (0, import_util10.bufferToArray)(buffer)
2875
- };
2876
- }));
2877
- }
2878
- async removeRange(keyPrefix) {
2879
- if (this._state !== "opened") {
2880
- return void 0;
2881
- }
2882
- const filename = this._getFilename(keyPrefix);
2883
- const entries = await this._directory.list();
2884
- await Promise.all(entries.filter((entry) => entry.startsWith(filename)).map(async (entry) => {
2885
- const file = this._directory.getOrCreateFile(entry);
2886
- await file.destroy();
2887
- }));
2888
- }
2889
- async close() {
2890
- this._state = "closed";
2891
- }
2892
- _getFilename(key) {
2893
- return key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-");
2894
- }
2895
- _getKeyFromFilename(filename) {
2896
- return filename.split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%"));
2897
- }
2898
- };
2899
- var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/local-host-network-adapter.ts";
2900
- var LocalHostNetworkAdapter = class extends import_automerge_repo.NetworkAdapter {
2901
- constructor() {
2902
- super(...arguments);
2903
- this._peers = /* @__PURE__ */ new Map();
2904
- this._connected = new import_async10.Trigger();
2905
- }
2906
- /**
2907
- * Emits `ready` event. That signals to `Repo` that it can start using the adapter.
2908
- */
2909
- ready() {
2910
- this.emit("ready", {
2911
- network: this
2912
- });
2913
- }
2914
- connect(peerId) {
2915
- this.peerId = peerId;
2916
- this._connected.wake();
2917
- }
2918
- send(message) {
2919
- const peer = this._peers.get(message.targetId);
2920
- (0, import_invariant10.invariant)(peer, "Peer not found.", {
2921
- F: __dxlog_file14,
2922
- L: 45,
2923
- S: this,
2924
- A: [
2925
- "peer",
2926
- "'Peer not found.'"
2927
- ]
2928
- });
2929
- peer.send(message);
2930
- }
2931
- async close() {
2932
- this._peers.forEach((peer) => peer.disconnect());
2933
- this.emit("close");
2934
- }
2935
- disconnect() {
2936
- }
2937
- syncRepo({ id, syncMessage }) {
2938
- const peerId = this._getPeerId(id);
2939
- return new import_codec_protobuf2.Stream(({ next, close }) => {
2940
- (0, import_invariant10.invariant)(!this._peers.has(peerId), "Peer already connected.", {
2941
- F: __dxlog_file14,
2942
- L: 63,
2943
- S: this,
2944
- A: [
2945
- "!this._peers.has(peerId)",
2946
- "'Peer already connected.'"
2947
- ]
2948
- });
2949
- this._peers.set(peerId, {
2950
- connected: true,
2951
- send: (message) => {
2952
- next({
2953
- syncMessage: import_automerge_repo.cbor.encode(message)
2954
- });
2955
- },
2956
- disconnect: () => {
2957
- this._peers.delete(peerId);
2958
- close();
2959
- this.emit("peer-disconnected", {
2960
- peerId
2961
- });
2962
- }
2963
- });
2964
- this._connected.wait({
2965
- timeout: 1e3
2966
- }).then(() => {
2967
- this.emit("peer-candidate", {
2968
- peerMetadata: {},
2969
- peerId
2970
- });
2971
- }).catch((err) => import_log13.log.catch(err, void 0, {
2972
- F: __dxlog_file14,
2973
- L: 88,
2974
- S: this,
2975
- C: (f, a) => f(...a)
2976
- }));
2977
- });
2978
- }
2979
- async sendSyncMessage({ id, syncMessage }) {
2980
- await this._connected.wait({
2981
- timeout: 1e3
2982
- });
2983
- const message = import_automerge_repo.cbor.decode(syncMessage);
2984
- this.emit("message", message);
2985
- }
2986
- async getHostInfo() {
2987
- await this._connected.wait({
2988
- timeout: 1e3
2989
- });
2990
- (0, import_invariant10.invariant)(this.peerId, "Peer id not set.", {
2991
- F: __dxlog_file14,
2992
- L: 100,
2993
- S: this,
2994
- A: [
2995
- "this.peerId",
2996
- "'Peer id not set.'"
2997
- ]
2998
- });
2999
- return {
3000
- peerId: this.peerId
3001
- };
3002
- }
3003
- _getPeerId(id) {
3004
- return id;
3005
- }
3006
- };
3007
- var __dxlog_file15 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-network-adapter.ts";
3008
- var MeshNetworkAdapter = class extends import_automerge_repo2.NetworkAdapter {
3009
- constructor() {
3010
- super(...arguments);
3011
- this._extensions = /* @__PURE__ */ new Map();
3012
- this._connected = new import_async11.Trigger();
3013
- }
3014
- /**
3015
- * Emits `ready` event. That signals to `Repo` that it can start using the adapter.
3016
- */
3017
- ready() {
3018
- this.emit("ready", {
3019
- network: this
3020
- });
3021
- }
3022
- connect(peerId) {
3023
- this.peerId = peerId;
3024
- this._connected.wake();
3025
- }
3026
- send(message) {
3027
- const receiverId = message.targetId;
3028
- const extension = this._extensions.get(receiverId);
3029
- (0, import_invariant11.invariant)(extension, "Extension not found.", {
3030
- F: __dxlog_file15,
3031
- L: 38,
3032
- S: this,
3033
- A: [
3034
- "extension",
3035
- "'Extension not found.'"
3036
- ]
3037
- });
3038
- extension.sendSyncMessage({
3039
- payload: import_automerge_repo2.cbor.encode(message)
3040
- }).catch((err) => import_log14.log.catch(err, void 0, {
3041
- F: __dxlog_file15,
3042
- L: 39,
3043
- S: this,
3044
- C: (f, a) => f(...a)
3045
- }));
3046
- }
3047
- disconnect() {
3048
- }
3049
- createExtension() {
3050
- (0, import_invariant11.invariant)(this.peerId, "Peer id not set.", {
3051
- F: __dxlog_file15,
3052
- L: 47,
3053
- S: this,
3054
- A: [
3055
- "this.peerId",
3056
- "'Peer id not set.'"
3057
- ]
3058
- });
3059
- let peerInfo;
3060
- const extension = new import_teleport_extension_automerge_replicator.AutomergeReplicator({
3061
- peerId: this.peerId
3062
- }, {
3063
- onStartReplication: async (info, remotePeerId) => {
3064
- await this._connected.wait();
3065
- (0, import_log14.log)("onStartReplication", {
3066
- id: info.id,
3067
- thisPeerId: this.peerId,
3068
- remotePeerId: remotePeerId.toHex()
3069
- }, {
3070
- F: __dxlog_file15,
3071
- L: 70,
3072
- S: this,
3073
- C: (f, a) => f(...a)
3074
- });
3075
- if (!this._extensions.has(info.id)) {
3076
- peerInfo = info;
3077
- this._extensions.set(info.id, extension);
3078
- (0, import_log14.log)("peer-candidate", {
3079
- id: info.id,
3080
- thisPeerId: this.peerId,
3081
- remotePeerId: remotePeerId.toHex()
3082
- }, {
3083
- F: __dxlog_file15,
3084
- L: 76,
3085
- S: this,
3086
- C: (f, a) => f(...a)
3087
- });
3088
- this.emit("peer-candidate", {
3089
- // TODO(mykola): Hack, stop abusing `peerMetadata` field.
3090
- peerMetadata: {
3091
- dxos_deviceKey: remotePeerId.toHex()
3092
- },
3093
- peerId: info.id
3094
- });
3095
- }
3096
- },
3097
- onSyncMessage: async ({ payload }) => {
3098
- if (!peerInfo) {
3099
- return;
3100
- }
3101
- const message = import_automerge_repo2.cbor.decode(payload);
3102
- this.emit("message", message);
3103
- },
3104
- onClose: async () => {
3105
- if (!peerInfo) {
3106
- return;
3107
- }
3108
- this.emit("peer-disconnected", {
3109
- peerId: peerInfo.id
3110
- });
3111
- this._extensions.delete(peerInfo.id);
3112
- }
3113
- });
3114
- return extension;
3115
- }
3116
- };
3117
- var AutomergeStorageWrapper = class {
3118
- constructor({ storage, callbacks }) {
3119
- this._storage = storage;
3120
- this._callbacks = callbacks;
3121
- }
3122
- async load(key) {
3123
- return this._storage.load(key);
3124
- }
3125
- async save(key, value) {
3126
- await this._callbacks.beforeSave?.(key);
3127
- await this._storage.save(key, value);
3128
- await this._callbacks.afterSave?.(key);
3129
- }
3130
- async remove(key) {
3131
- return this._storage.remove(key);
3132
- }
3133
- async loadRange(keyPrefix) {
3134
- return this._storage.loadRange(keyPrefix);
3135
- }
3136
- async removeRange(keyPrefix) {
3137
- return this._storage.removeRange(keyPrefix);
3138
- }
3139
- async close() {
3140
- if (this._storage instanceof AutomergeStorageAdapter) {
3141
- return this._storage.close();
3142
- }
3143
- }
3144
- };
3145
- function _ts_decorate9(decorators, target, key, desc) {
3146
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3147
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
3148
- r = Reflect.decorate(decorators, target, key, desc);
3149
- else
3150
- for (var i = decorators.length - 1; i >= 0; i--)
3151
- if (d = decorators[i])
3152
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
3153
- return c > 3 && r && Object.defineProperty(target, key, r), r;
3154
- }
3155
- var __dxlog_file16 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
3156
- var AutomergeHost = class {
3157
- constructor({ directory, metadata }) {
3158
- this._ctx = new import_context8.Context();
3159
- this._authorizedDevices = new import_util11.ComplexMap(import_keys8.PublicKey.hash);
3160
- this._updatingMetadata = /* @__PURE__ */ new Map();
3161
- this._requestedDocs = /* @__PURE__ */ new Set();
3162
- this._metadata = metadata;
3163
- this._meshNetwork = new MeshNetworkAdapter();
3164
- this._clientNetwork = new LocalHostNetworkAdapter();
3165
- this._storage = new AutomergeStorageWrapper({
3166
- storage: (
3167
- // TODO(mykola): Delete specific handling of IDB storage.
3168
- directory.type === import_random_access_storage.StorageType.IDB ? new import_automerge_repo_storage_indexeddb.IndexedDBStorageAdapter(directory.path, "data") : new AutomergeStorageAdapter(directory)
3169
- ),
3170
- callbacks: {
3171
- beforeSave: (params) => this._beforeSave(params)
3172
- }
3173
- });
3174
- this._peerId = `host-${import_keys8.PublicKey.random().toHex()}`;
3175
- this._repo = new import_automerge_repo3.Repo({
3176
- peerId: this._peerId,
3177
- network: [
3178
- this._clientNetwork,
3179
- this._meshNetwork
3180
- ],
3181
- storage: this._storage,
3182
- // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
3183
- // Hosts, running in the worker, don't share documents unless requested by other peers.
3184
- sharePolicy: async (peerId, documentId) => {
3185
- if (peerId.startsWith("client-")) {
3186
- return false;
3187
- }
3188
- if (!documentId) {
3189
- return false;
3190
- }
3191
- const doc = this._repo.handles[documentId]?.docSync();
3192
- if (!doc) {
3193
- const isRequested = this._requestedDocs.has(`automerge:${documentId}`);
3194
- (0, import_log15.log)("doc share policy check", {
3195
- peerId,
3196
- documentId,
3197
- isRequested
3198
- }, {
3199
- F: __dxlog_file16,
3200
- L: 96,
3201
- S: this,
3202
- C: (f, a) => f(...a)
3203
- });
3204
- return isRequested;
3205
- }
3206
- try {
3207
- const spaceKey = getSpaceKeyFromDoc(doc);
3208
- if (!spaceKey) {
3209
- (0, import_log15.log)("space key not found for share policy check", {
3210
- peerId,
3211
- documentId
3212
- }, {
3213
- F: __dxlog_file16,
3214
- L: 103,
3215
- S: this,
3216
- C: (f, a) => f(...a)
3217
- });
3218
- return false;
3219
- }
3220
- const authorizedDevices = this._authorizedDevices.get(import_keys8.PublicKey.from(spaceKey));
3221
- const deviceKeyHex = this.repo.peerMetadataByPeerId[peerId]?.dxos_deviceKey;
3222
- if (!deviceKeyHex) {
3223
- (0, import_log15.log)("device key not found for share policy check", {
3224
- peerId,
3225
- documentId
3226
- }, {
3227
- F: __dxlog_file16,
3228
- L: 112,
3229
- S: this,
3230
- C: (f, a) => f(...a)
3231
- });
3232
- return false;
3233
- }
3234
- const deviceKey = import_keys8.PublicKey.from(deviceKeyHex);
3235
- const isAuthorized = authorizedDevices?.has(deviceKey) ?? false;
3236
- (0, import_log15.log)("share policy check", {
3237
- localPeer: this._peerId,
3238
- remotePeer: peerId,
3239
- documentId,
3240
- deviceKey,
3241
- spaceKey,
3242
- isAuthorized
3243
- }, {
3244
- F: __dxlog_file16,
3245
- L: 118,
3246
- S: this,
3247
- C: (f, a) => f(...a)
3248
- });
3249
- return isAuthorized;
3250
- } catch (err) {
3251
- import_log15.log.catch(err, void 0, {
3252
- F: __dxlog_file16,
3253
- L: 128,
3254
- S: this,
3255
- C: (f, a) => f(...a)
3256
- });
3257
- return false;
3258
- }
3259
- }
3260
- });
3261
- this._clientNetwork.ready();
3262
- this._meshNetwork.ready();
3263
- {
3264
- const listener = ({ handle }) => this._onDocument(handle);
3265
- this._repo.on("document", listener);
3266
- this._ctx.onDispose(() => {
3267
- this._repo.off("document", listener);
3268
- });
3269
- }
3270
- }
3271
- get repo() {
3272
- return this._repo;
3273
- }
3274
- async _beforeSave(path) {
3275
- const id = path[0];
3276
- if (this._updatingMetadata.has(id)) {
3277
- return this._updatingMetadata.get(id);
3278
- }
3279
- }
3280
- _onDocument(handle) {
3281
- const listener = (event) => this._onUpdate(event);
3282
- handle.on("change", listener);
3283
- this._ctx.onDispose(() => {
3284
- handle.off("change", listener);
3285
- });
3286
- }
3287
- _onUpdate(event) {
3288
- if (this._metadata == null) {
3289
- return;
3290
- }
3291
- const objectIds = getInlineChanges(event);
3292
- if (objectIds.length === 0) {
3293
- return;
3294
- }
3295
- const heads = (0, import_automerge.getHeads)(event.doc);
3296
- const lastAvailableHash = heads.at(-1);
3297
- if (!lastAvailableHash) {
3298
- return;
3299
- }
3300
- const encodedIds = objectIds.map((objectId) => import_protocols8.idCodec.encode({
3301
- documentId: event.handle.documentId,
3302
- objectId
3303
- }));
3304
- const idToLastHash = new Map(encodedIds.map((id) => [
3305
- id,
3306
- lastAvailableHash
3307
- ]));
3308
- const markingDirtyPromise = this._metadata.markDirty(idToLastHash).then(() => {
3309
- this._updatingMetadata.delete(event.handle.documentId);
3310
- }).catch((err) => {
3311
- this._ctx.disposed && import_log15.log.catch(err, void 0, {
3312
- F: __dxlog_file16,
3313
- L: 188,
3314
- S: this,
3315
- C: (f, a) => f(...a)
3316
- });
3317
- });
3318
- this._updatingMetadata.set(event.handle.documentId, markingDirtyPromise);
3319
- }
3320
- _automergeDocs() {
3321
- return (0, import_util11.mapValues)(this._repo.handles, (handle) => ({
3322
- state: handle.state,
3323
- hasDoc: !!handle.docSync(),
3324
- heads: handle.docSync() ? import_automerge.next.getHeads(handle.docSync()) : null,
3325
- data: handle.docSync()?.doc && (0, import_util11.mapValues)(handle.docSync()?.doc, (value, key) => {
3326
- try {
3327
- switch (key) {
3328
- case "access":
3329
- case "links":
3330
- return value;
3331
- case "objects":
3332
- return Object.keys(value);
3333
- default:
3334
- return `${value}`;
3335
- }
3336
- } catch (err) {
3337
- return `${err}`;
3338
- }
3339
- })
3340
- }));
3341
- }
3342
- _automergePeers() {
3343
- return this._repo.peers;
3344
- }
3345
- async close() {
3346
- await this._storage.close();
3347
- await this._clientNetwork.close();
3348
- await this._ctx.dispose();
3349
- }
3350
- //
3351
- // Methods for client-services.
3352
- //
3353
- syncRepo(request) {
3354
- return this._clientNetwork.syncRepo(request);
3355
- }
3356
- sendSyncMessage(request) {
3357
- return this._clientNetwork.sendSyncMessage(request);
3358
- }
3359
- async getHostInfo() {
3360
- return this._clientNetwork.getHostInfo();
3361
- }
3362
- //
3363
- // Mesh replication.
3364
- //
3365
- createExtension() {
3366
- return this._meshNetwork.createExtension();
3367
- }
3368
- authorizeDevice(spaceKey, deviceKey) {
3369
- (0, import_log15.log)("authorizeDevice", {
3370
- spaceKey,
3371
- deviceKey
3372
- }, {
3373
- F: __dxlog_file16,
3374
- L: 255,
3375
- S: this,
3376
- C: (f, a) => f(...a)
3377
- });
3378
- (0, import_util11.defaultMap)(this._authorizedDevices, spaceKey, () => new import_util11.ComplexSet(import_keys8.PublicKey.hash)).add(deviceKey);
3379
- }
3380
- };
3381
- _ts_decorate9([
3382
- import_tracing5.trace.info()
3383
- ], AutomergeHost.prototype, "_peerId", void 0);
3384
- _ts_decorate9([
3385
- import_tracing5.trace.info({
3386
- depth: null
3387
- })
3388
- ], AutomergeHost.prototype, "_automergeDocs", null);
3389
- _ts_decorate9([
3390
- import_tracing5.trace.info({
3391
- depth: null
3392
- })
3393
- ], AutomergeHost.prototype, "_automergePeers", null);
3394
- AutomergeHost = _ts_decorate9([
3395
- import_tracing5.trace.resource()
3396
- ], AutomergeHost);
3397
- var getInlineChanges = (event) => {
3398
- const inlineChangedObjectIds = /* @__PURE__ */ new Set();
3399
- for (const { path } of event.patches) {
3400
- if (path.length < 2) {
3401
- continue;
3402
- }
3403
- switch (path[0]) {
3404
- case "objects":
3405
- if (path.length >= 2) {
3406
- inlineChangedObjectIds.add(path[1]);
3407
- }
3408
- break;
3409
- }
3410
- }
3411
- return [
3412
- ...inlineChangedObjectIds
3413
- ];
3414
- };
3415
- var getSpaceKeyFromDoc = (doc) => {
3416
- const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
3417
- if (rawSpaceKey == null) {
3418
- return null;
3419
- }
3420
- return String(rawSpaceKey);
3421
- };
3422
- // Annotate the CommonJS export names for ESM import in node:
3423
- 0 && (module.exports = {
3424
- AuthExtension,
3425
- AuthStatus,
3426
- AutomergeHost,
3427
- AutomergeStorageAdapter,
3428
- DataPipeline,
3429
- DataServiceHost,
3430
- DataServiceImpl,
3431
- DataServiceSubscriptions,
3432
- DatabaseHost,
3433
- LocalHostNetworkAdapter,
3434
- MOCK_AUTH_PROVIDER,
3435
- MOCK_AUTH_VERIFIER,
3436
- MeshNetworkAdapter,
3437
- MetadataStore,
3438
- Pipeline,
3439
- SnapshotManager,
3440
- SnapshotStore,
3441
- Space,
3442
- SpaceManager,
3443
- SpaceProtocol,
3444
- SpaceProtocolSession,
3445
- TimeframeClock,
3446
- codec,
3447
- createMappedFeedWriter,
3448
- getSpaceKeyFromDoc,
3449
- mapFeedIndexesToTimeframe,
3450
- mapTimeframeToFeedIndexes,
3451
- startAfter,
3452
- valueEncoding
3453
- });
3454
- //# sourceMappingURL=chunk-WZ4WTAN6.cjs.map