@dxos/echo-pipeline 0.6.7 → 0.6.8-main.3be982f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/lib/browser/chunk-NE5LORNQ.mjs +2028 -0
  2. package/dist/lib/browser/chunk-NE5LORNQ.mjs.map +7 -0
  3. package/dist/lib/browser/chunk-Q4B5JN6L.mjs +2150 -0
  4. package/dist/lib/browser/chunk-Q4B5JN6L.mjs.map +7 -0
  5. package/dist/lib/browser/chunk-XPCF2V5U.mjs +31 -0
  6. package/dist/lib/browser/chunk-XPCF2V5U.mjs.map +7 -0
  7. package/dist/lib/browser/index.mjs +16 -16
  8. package/dist/lib/browser/light.mjs +32 -0
  9. package/dist/lib/browser/light.mjs.map +7 -0
  10. package/dist/lib/browser/meta.json +1 -1
  11. package/dist/lib/browser/testing/index.mjs +3 -7
  12. package/dist/lib/browser/testing/index.mjs.map +3 -3
  13. package/dist/lib/node/chunk-5KNTTBQK.cjs +2146 -0
  14. package/dist/lib/node/chunk-5KNTTBQK.cjs.map +7 -0
  15. package/dist/lib/node/chunk-DZVH7HDD.cjs +43 -0
  16. package/dist/lib/node/chunk-DZVH7HDD.cjs.map +7 -0
  17. package/dist/lib/node/chunk-IHR4UMVA.cjs +2043 -0
  18. package/dist/lib/node/chunk-IHR4UMVA.cjs.map +7 -0
  19. package/dist/lib/node/index.cjs +35 -37
  20. package/dist/lib/node/index.cjs.map +2 -2
  21. package/dist/lib/node/light.cjs +52 -0
  22. package/dist/lib/node/light.cjs.map +7 -0
  23. package/dist/lib/node/meta.json +1 -1
  24. package/dist/lib/node/testing/index.cjs +12 -15
  25. package/dist/lib/node/testing/index.cjs.map +3 -3
  26. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  27. package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
  28. package/dist/types/src/common/codec.d.ts +0 -1
  29. package/dist/types/src/common/codec.d.ts.map +1 -1
  30. package/dist/types/src/common/feeds.d.ts.map +1 -1
  31. package/dist/types/src/common/index.d.ts +1 -0
  32. package/dist/types/src/common/index.d.ts.map +1 -1
  33. package/dist/types/src/common/space-id.d.ts +7 -0
  34. package/dist/types/src/common/space-id.d.ts.map +1 -0
  35. package/dist/types/src/db-host/index.d.ts +0 -2
  36. package/dist/types/src/db-host/index.d.ts.map +1 -1
  37. package/dist/types/src/light.d.ts +4 -0
  38. package/dist/types/src/light.d.ts.map +1 -0
  39. package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
  40. package/dist/types/src/space/space-manager.d.ts +1 -7
  41. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  42. package/dist/types/src/space/space-protocol.d.ts +0 -1
  43. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  44. package/dist/types/src/space/space.d.ts +1 -10
  45. package/dist/types/src/space/space.d.ts.map +1 -1
  46. package/dist/types/src/testing/test-agent-builder.d.ts +0 -3
  47. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  48. package/package.json +43 -33
  49. package/src/automerge/echo-network-adapter.ts +1 -1
  50. package/src/automerge/mesh-echo-replicator.ts +1 -1
  51. package/src/common/index.ts +1 -0
  52. package/src/common/space-id.ts +27 -0
  53. package/src/db-host/index.ts +0 -2
  54. package/src/light.ts +7 -0
  55. package/src/space/space-manager.ts +3 -19
  56. package/src/space/space.ts +2 -32
  57. package/src/testing/test-agent-builder.ts +0 -7
  58. package/dist/lib/browser/chunk-P6XSIJKM.mjs +0 -4281
  59. package/dist/lib/browser/chunk-P6XSIJKM.mjs.map +0 -7
  60. package/dist/lib/node/chunk-IYTGTZ7D.cjs +0 -4255
  61. package/dist/lib/node/chunk-IYTGTZ7D.cjs.map +0 -7
  62. package/dist/types/src/db-host/snapshot-manager.d.ts +0 -19
  63. package/dist/types/src/db-host/snapshot-manager.d.ts.map +0 -1
  64. package/dist/types/src/db-host/snapshot-store.d.ts +0 -16
  65. package/dist/types/src/db-host/snapshot-store.d.ts.map +0 -1
  66. package/dist/types/src/db-host/snapshot-store.test.d.ts +0 -2
  67. package/dist/types/src/db-host/snapshot-store.test.d.ts.map +0 -1
  68. package/src/db-host/snapshot-manager.ts +0 -54
  69. package/src/db-host/snapshot-store.test.ts +0 -31
  70. package/src/db-host/snapshot-store.ts +0 -61
@@ -1,4281 +0,0 @@
1
- import "@dxos/node-std/globals";
2
-
3
- // inject-globals:@inject-globals
4
- import {
5
- global,
6
- Buffer,
7
- process
8
- } from "@dxos/node-std/inject-globals";
9
-
10
- // packages/core/echo/echo-pipeline/src/common/codec.ts
11
- import { createCodecEncoding } from "@dxos/hypercore";
12
- import { schema } from "@dxos/protocols";
13
- var codec = schema.getCodecForType("dxos.echo.feed.FeedMessage");
14
- var valueEncoding = createCodecEncoding(codec);
15
-
16
- // packages/core/echo/echo-pipeline/src/common/feeds.ts
17
- import { invariant } from "@dxos/invariant";
18
- var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/common/feeds.ts";
19
- var createMappedFeedWriter = (mapper, writer) => {
20
- invariant(mapper, void 0, {
21
- F: __dxlog_file,
22
- L: 16,
23
- S: void 0,
24
- A: [
25
- "mapper",
26
- ""
27
- ]
28
- });
29
- invariant(writer, void 0, {
30
- F: __dxlog_file,
31
- L: 17,
32
- S: void 0,
33
- A: [
34
- "writer",
35
- ""
36
- ]
37
- });
38
- return {
39
- write: async (data, options) => await writer.write(await mapper(data), options)
40
- };
41
- };
42
-
43
- // packages/core/echo/echo-pipeline/src/db-host/snapshot-manager.ts
44
- import { cancelWithContext } from "@dxos/context";
45
- import { PublicKey } from "@dxos/keys";
46
- import { schema as schema2 } from "@dxos/protocols";
47
- import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
48
- var SpaceSnapshot = schema2.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
49
- var SnapshotManager = class {
50
- constructor(_snapshotStore, _blobStore, _blobSync) {
51
- this._snapshotStore = _snapshotStore;
52
- this._blobStore = _blobStore;
53
- this._blobSync = _blobSync;
54
- }
55
- async _getBlob(blobId) {
56
- const blob = await this._blobStore.get(blobId);
57
- return SpaceSnapshot.decode(blob);
58
- }
59
- async load(ctx, id) {
60
- const blobId = PublicKey.fromHex(id).asUint8Array();
61
- const blobMeta = await this._blobStore.getMeta(blobId);
62
- if (blobMeta && blobMeta.state === BlobMeta.State.FULLY_PRESENT) {
63
- return this._getBlob(blobId);
64
- }
65
- const fallbackStore = await cancelWithContext(ctx, this._snapshotStore.loadSnapshot(id));
66
- if (fallbackStore) {
67
- return fallbackStore;
68
- }
69
- await this._blobSync.download(ctx, blobId);
70
- return this._getBlob(blobId);
71
- }
72
- async store(snapshot) {
73
- const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
74
- await this._blobSync.notifyBlobAdded(id);
75
- return PublicKey.from(id).toHex();
76
- }
77
- };
78
-
79
- // packages/core/echo/echo-pipeline/src/db-host/snapshot-store.ts
80
- import { subtleCrypto } from "@dxos/crypto";
81
- import { schema as schema3 } from "@dxos/protocols";
82
- var SpaceSnapshot2 = schema3.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
83
- var SnapshotStore = class {
84
- constructor(_directory) {
85
- this._directory = _directory;
86
- }
87
- async saveSnapshot(snapshot) {
88
- const encoded = SpaceSnapshot2.encode(snapshot);
89
- const key = await subtleCrypto.digest("SHA-256", encoded);
90
- const keyString = Buffer.from(key).toString("hex");
91
- const file = await this._directory.getOrCreateFile(keyString);
92
- try {
93
- await file.write(0, Buffer.from(encoded));
94
- } finally {
95
- await file.close();
96
- }
97
- return keyString;
98
- }
99
- async loadSnapshot(key) {
100
- const file = await this._directory.getOrCreateFile(key);
101
- try {
102
- const { size } = await file.stat();
103
- if (size === 0) {
104
- return void 0;
105
- }
106
- const buffer = await file.read(0, size);
107
- return SpaceSnapshot2.decode(buffer);
108
- } finally {
109
- await file.close();
110
- }
111
- }
112
- async listSnapshots() {
113
- const entries = await this._directory.list();
114
- return await Promise.all(entries.map(async (key) => {
115
- const { size } = await this._directory.getOrCreateFile(key).stat();
116
- return {
117
- key,
118
- size
119
- };
120
- }));
121
- }
122
- };
123
-
124
- // packages/core/echo/echo-pipeline/src/db-host/documents-synchronizer.ts
125
- import { UpdateScheduler } from "@dxos/async";
126
- import { next as A } from "@dxos/automerge/automerge";
127
- import { Resource } from "@dxos/context";
128
- import { invariant as invariant2 } from "@dxos/invariant";
129
- import { log } from "@dxos/log";
130
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/documents-synchronizer.ts";
131
- var MAX_UPDATE_FREQ = 10;
132
- var DocumentsSynchronizer = class extends Resource {
133
- constructor(_params) {
134
- super();
135
- this._params = _params;
136
- this._syncStates = /* @__PURE__ */ new Map();
137
- this._pendingUpdates = /* @__PURE__ */ new Set();
138
- this._sendUpdatesJob = void 0;
139
- }
140
- addDocuments(documentIds, retryCounter = 0) {
141
- if (retryCounter > 3) {
142
- log.warn("Failed to load document, retry limit reached", {
143
- documentIds
144
- }, {
145
- F: __dxlog_file2,
146
- L: 49,
147
- S: this,
148
- C: (f, a) => f(...a)
149
- });
150
- return;
151
- }
152
- for (const documentId of documentIds) {
153
- const doc = this._params.repo.find(documentId);
154
- doc.whenReady().then(() => {
155
- this._startSync(doc);
156
- this._pendingUpdates.add(doc.documentId);
157
- this._sendUpdatesJob.trigger();
158
- }).catch((error) => {
159
- log.warn("Failed to load document, wraparound", {
160
- documentId,
161
- error
162
- }, {
163
- F: __dxlog_file2,
164
- L: 63,
165
- S: this,
166
- C: (f, a) => f(...a)
167
- });
168
- this.addDocuments([
169
- documentId
170
- ], retryCounter + 1);
171
- });
172
- }
173
- }
174
- removeDocuments(documentIds) {
175
- for (const documentId of documentIds) {
176
- this._syncStates.get(documentId)?.clearSubscriptions?.();
177
- this._syncStates.delete(documentId);
178
- this._pendingUpdates.delete(documentId);
179
- }
180
- }
181
- async _open() {
182
- this._sendUpdatesJob = new UpdateScheduler(this._ctx, this._checkAndSendUpdates.bind(this), {
183
- maxFrequency: MAX_UPDATE_FREQ
184
- });
185
- }
186
- async _close() {
187
- await this._sendUpdatesJob.join();
188
- this._syncStates.clear();
189
- }
190
- update(updates) {
191
- for (const { documentId, mutation, isNew } of updates) {
192
- if (isNew) {
193
- const doc = this._params.repo.find(documentId);
194
- doc.update((doc2) => A.loadIncremental(doc2, mutation));
195
- this._startSync(doc);
196
- } else {
197
- this._writeMutation(documentId, mutation);
198
- }
199
- }
200
- }
201
- _startSync(doc) {
202
- if (this._syncStates.has(doc.documentId)) {
203
- log.info("Document already being synced", {
204
- documentId: doc.documentId
205
- }, {
206
- F: __dxlog_file2,
207
- L: 102,
208
- S: this,
209
- C: (f, a) => f(...a)
210
- });
211
- return;
212
- }
213
- const syncState = {
214
- handle: doc
215
- };
216
- this._subscribeForChanges(syncState);
217
- this._syncStates.set(doc.documentId, syncState);
218
- }
219
- _subscribeForChanges(syncState) {
220
- const handler = () => {
221
- this._pendingUpdates.add(syncState.handle.documentId);
222
- this._sendUpdatesJob.trigger();
223
- };
224
- syncState.handle.on("heads-changed", handler);
225
- syncState.clearSubscriptions = () => syncState.handle.off("heads-changed", handler);
226
- }
227
- async _checkAndSendUpdates() {
228
- const updates = [];
229
- const docsWithPendingUpdates = Array.from(this._pendingUpdates);
230
- this._pendingUpdates.clear();
231
- for (const documentId of docsWithPendingUpdates) {
232
- const update = this._getPendingChanges(documentId);
233
- if (update) {
234
- updates.push({
235
- documentId,
236
- mutation: update
237
- });
238
- }
239
- }
240
- if (updates.length > 0) {
241
- this._params.sendUpdates({
242
- updates
243
- });
244
- }
245
- }
246
- _getPendingChanges(documentId) {
247
- const syncState = this._syncStates.get(documentId);
248
- invariant2(syncState, "Sync state for document not found", {
249
- F: __dxlog_file2,
250
- L: 143,
251
- S: this,
252
- A: [
253
- "syncState",
254
- "'Sync state for document not found'"
255
- ]
256
- });
257
- const doc = syncState.handle.docSync();
258
- if (!doc) {
259
- return;
260
- }
261
- const mutation = syncState.lastSentHead ? A.saveSince(doc, syncState.lastSentHead) : A.save(doc);
262
- if (mutation.length === 0) {
263
- return;
264
- }
265
- syncState.lastSentHead = A.getHeads(doc);
266
- return mutation;
267
- }
268
- _writeMutation(documentId, mutation) {
269
- const syncState = this._syncStates.get(documentId);
270
- invariant2(syncState, "Sync state for document not found", {
271
- F: __dxlog_file2,
272
- L: 158,
273
- S: this,
274
- A: [
275
- "syncState",
276
- "'Sync state for document not found'"
277
- ]
278
- });
279
- syncState.handle.update((doc) => {
280
- const headsBefore = A.getHeads(doc);
281
- const newDoc = A.loadIncremental(doc, mutation);
282
- if (A.equals(headsBefore, syncState.lastSentHead)) {
283
- syncState.lastSentHead = A.getHeads(newDoc);
284
- }
285
- return newDoc;
286
- });
287
- }
288
- };
289
-
290
- // packages/core/echo/echo-pipeline/src/automerge/collection-synchronizer.ts
291
- import { asyncReturn, Event, scheduleTask, scheduleTaskInterval } from "@dxos/async";
292
- import { next as am } from "@dxos/automerge/automerge";
293
- import { Resource as Resource2 } from "@dxos/context";
294
- import { defaultMap } from "@dxos/util";
295
- var MIN_QUERY_INTERVAL = 5e3;
296
- var POLL_INTERVAL = 3e4;
297
- var CollectionSynchronizer = class extends Resource2 {
298
- constructor(params) {
299
- super();
300
- /**
301
- * CollectionId -> State.
302
- */
303
- this._perCollectionStates = /* @__PURE__ */ new Map();
304
- this._connectedPeers = /* @__PURE__ */ new Set();
305
- this.remoteStateUpdated = new Event();
306
- this._sendCollectionState = params.sendCollectionState;
307
- this._queryCollectionState = params.queryCollectionState;
308
- this._shouldSyncCollection = params.shouldSyncCollection;
309
- }
310
- async _open(ctx) {
311
- scheduleTaskInterval(this._ctx, async () => {
312
- for (const collectionId of this._perCollectionStates.keys()) {
313
- this.refreshCollection(collectionId);
314
- await asyncReturn();
315
- }
316
- }, POLL_INTERVAL);
317
- }
318
- getRegisteredCollectionIds() {
319
- return [
320
- ...this._perCollectionStates.keys()
321
- ];
322
- }
323
- getLocalCollectionState(collectionId) {
324
- return this._getPerCollectionState(collectionId).localState;
325
- }
326
- setLocalCollectionState(collectionId, state) {
327
- this._getPerCollectionState(collectionId).localState = state;
328
- queueMicrotask(async () => {
329
- if (!this._ctx.disposed) {
330
- this._refreshInterestedPeers(collectionId);
331
- this.refreshCollection(collectionId);
332
- }
333
- });
334
- }
335
- getRemoteCollectionStates(collectionId) {
336
- return this._getPerCollectionState(collectionId).remoteStates;
337
- }
338
- refreshCollection(collectionId) {
339
- let scheduleAnotherRefresh = false;
340
- const state = this._getPerCollectionState(collectionId);
341
- for (const peerId of this._connectedPeers) {
342
- if (state.interestedPeers.has(peerId)) {
343
- const lastQueried = state.lastQueried.get(peerId) ?? 0;
344
- if (Date.now() - lastQueried > MIN_QUERY_INTERVAL) {
345
- state.lastQueried.set(peerId, Date.now());
346
- this._queryCollectionState(collectionId, peerId);
347
- } else {
348
- scheduleAnotherRefresh = true;
349
- }
350
- }
351
- }
352
- if (scheduleAnotherRefresh) {
353
- scheduleTask(this._ctx, () => this.refreshCollection(collectionId), MIN_QUERY_INTERVAL);
354
- }
355
- }
356
- /**
357
- * Callback when a connection to a peer is established.
358
- */
359
- onConnectionOpen(peerId) {
360
- this._connectedPeers.add(peerId);
361
- queueMicrotask(async () => {
362
- if (this._ctx.disposed) {
363
- return;
364
- }
365
- for (const [collectionId, state] of this._perCollectionStates.entries()) {
366
- if (this._shouldSyncCollection(collectionId, peerId)) {
367
- state.interestedPeers.add(peerId);
368
- state.lastQueried.set(peerId, Date.now());
369
- this._queryCollectionState(collectionId, peerId);
370
- }
371
- }
372
- });
373
- }
374
- /**
375
- * Callback when a connection to a peer is closed.
376
- */
377
- onConnectionClosed(peerId) {
378
- this._connectedPeers.delete(peerId);
379
- for (const perCollectionState of this._perCollectionStates.values()) {
380
- perCollectionState.remoteStates.delete(peerId);
381
- }
382
- }
383
- /**
384
- * Callback when a peer queries the state of a collection.
385
- */
386
- onCollectionStateQueried(collectionId, peerId) {
387
- const perCollectionState = this._getPerCollectionState(collectionId);
388
- if (perCollectionState.localState) {
389
- this._sendCollectionState(collectionId, peerId, perCollectionState.localState);
390
- }
391
- }
392
- /**
393
- * Callback when a peer sends the state of a collection.
394
- */
395
- onRemoteStateReceived(collectionId, peerId, state) {
396
- const perCollectionState = this._getPerCollectionState(collectionId);
397
- perCollectionState.remoteStates.set(peerId, state);
398
- this.remoteStateUpdated.emit({
399
- peerId,
400
- collectionId
401
- });
402
- }
403
- _getPerCollectionState(collectionId) {
404
- return defaultMap(this._perCollectionStates, collectionId, () => ({
405
- localState: void 0,
406
- remoteStates: /* @__PURE__ */ new Map(),
407
- interestedPeers: /* @__PURE__ */ new Set(),
408
- lastQueried: /* @__PURE__ */ new Map()
409
- }));
410
- }
411
- _refreshInterestedPeers(collectionId) {
412
- for (const peerId of this._connectedPeers) {
413
- if (this._shouldSyncCollection(collectionId, peerId)) {
414
- this._getPerCollectionState(collectionId).interestedPeers.add(peerId);
415
- } else {
416
- this._getPerCollectionState(collectionId).interestedPeers.delete(peerId);
417
- }
418
- }
419
- }
420
- };
421
- var diffCollectionState = (local, remote) => {
422
- const allDocuments = /* @__PURE__ */ new Set([
423
- ...Object.keys(local.documents),
424
- ...Object.keys(remote.documents)
425
- ]);
426
- const different = [];
427
- for (const documentId of allDocuments) {
428
- if (!local.documents[documentId] || !remote.documents[documentId] || !am.equals(local.documents[documentId], remote.documents[documentId])) {
429
- different.push(documentId);
430
- }
431
- }
432
- return {
433
- different
434
- };
435
- };
436
-
437
- // packages/core/echo/echo-pipeline/src/space/auth.ts
438
- import { runInContext, scheduleTask as scheduleTask2 } from "@dxos/async";
439
- import { Context } from "@dxos/context";
440
- import { randomBytes } from "@dxos/crypto";
441
- import { invariant as invariant3 } from "@dxos/invariant";
442
- import { log as log2 } from "@dxos/log";
443
- import { schema as schema4 } from "@dxos/protocols";
444
- import { RpcExtension } from "@dxos/teleport";
445
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/auth.ts";
446
- var AuthExtension = class extends RpcExtension {
447
- constructor(_authParams) {
448
- super({
449
- requested: {
450
- AuthService: schema4.getService("dxos.mesh.teleport.auth.AuthService")
451
- },
452
- exposed: {
453
- AuthService: schema4.getService("dxos.mesh.teleport.auth.AuthService")
454
- },
455
- timeout: 60 * 1e3
456
- });
457
- this._authParams = _authParams;
458
- this._ctx = new Context({
459
- onError: (err) => {
460
- log2.catch(err, void 0, {
461
- F: __dxlog_file3,
462
- L: 28,
463
- S: this,
464
- C: (f, a) => f(...a)
465
- });
466
- }
467
- }, {
468
- F: __dxlog_file3,
469
- L: 26
470
- });
471
- }
472
- async getHandlers() {
473
- return {
474
- AuthService: {
475
- authenticate: async ({ challenge }) => {
476
- try {
477
- const credential = await this._authParams.provider(challenge);
478
- if (!credential) {
479
- throw new Error("auth rejected");
480
- }
481
- return {
482
- credential
483
- };
484
- } catch (err) {
485
- log2.error("failed to generate auth credentials", err, {
486
- F: __dxlog_file3,
487
- L: 55,
488
- S: this,
489
- C: (f, a) => f(...a)
490
- });
491
- throw new Error("auth rejected");
492
- }
493
- }
494
- }
495
- };
496
- }
497
- async onOpen(context) {
498
- await super.onOpen(context);
499
- scheduleTask2(this._ctx, async () => {
500
- try {
501
- const challenge = randomBytes(32);
502
- const { credential } = await this.rpc.AuthService.authenticate({
503
- challenge
504
- });
505
- invariant3(credential?.length > 0, "invalid credential", {
506
- F: __dxlog_file3,
507
- L: 69,
508
- S: this,
509
- A: [
510
- "credential?.length > 0",
511
- "'invalid credential'"
512
- ]
513
- });
514
- const success = await this._authParams.verifier(challenge, credential);
515
- invariant3(success, "credential not verified", {
516
- F: __dxlog_file3,
517
- L: 71,
518
- S: this,
519
- A: [
520
- "success",
521
- "'credential not verified'"
522
- ]
523
- });
524
- runInContext(this._ctx, () => this._authParams.onAuthSuccess());
525
- } catch (err) {
526
- log2("auth failed", err, {
527
- F: __dxlog_file3,
528
- L: 74,
529
- S: this,
530
- C: (f, a) => f(...a)
531
- });
532
- this.close();
533
- this._authParams.onAuthFailure();
534
- }
535
- });
536
- }
537
- async onClose() {
538
- await this._ctx.dispose();
539
- await super.onClose();
540
- }
541
- async onAbort() {
542
- await this._ctx.dispose();
543
- await super.onAbort();
544
- }
545
- };
546
-
547
- // packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
548
- import { Event as Event2 } from "@dxos/async";
549
- import { timed } from "@dxos/debug";
550
- import { log as log3 } from "@dxos/log";
551
- import { Timeframe } from "@dxos/timeframe";
552
- function _ts_decorate(decorators, target, key, desc) {
553
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
554
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
555
- r = Reflect.decorate(decorators, target, key, desc);
556
- else
557
- for (var i = decorators.length - 1; i >= 0; i--)
558
- if (d = decorators[i])
559
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
560
- return c > 3 && r && Object.defineProperty(target, key, r), r;
561
- }
562
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
563
- var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
564
- feedKey,
565
- index
566
- }));
567
- var mapFeedIndexesToTimeframe = (indexes) => new Timeframe(indexes.map(({ feedKey, index }) => [
568
- feedKey,
569
- index
570
- ]));
571
- var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
572
- feedKey,
573
- index: index + 1
574
- }));
575
- var TimeframeClock = class {
576
- constructor(_timeframe = new Timeframe()) {
577
- this._timeframe = _timeframe;
578
- this.update = new Event2();
579
- this._pendingTimeframe = _timeframe;
580
- }
581
- /**
582
- * Timeframe that was processed by ECHO.
583
- */
584
- get timeframe() {
585
- return this._timeframe;
586
- }
587
- /**
588
- * Timeframe that is currently being processed by ECHO.
589
- * Will be equal to `timeframe` after the processing is complete.
590
- */
591
- get pendingTimeframe() {
592
- return this._pendingTimeframe;
593
- }
594
- setTimeframe(timeframe) {
595
- this._timeframe = timeframe;
596
- this._pendingTimeframe = timeframe;
597
- this.update.emit(this._timeframe);
598
- }
599
- updatePendingTimeframe(key, seq) {
600
- this._pendingTimeframe = Timeframe.merge(this._pendingTimeframe, new Timeframe([
601
- [
602
- key,
603
- seq
604
- ]
605
- ]));
606
- }
607
- updateTimeframe() {
608
- this._timeframe = this._pendingTimeframe;
609
- this.update.emit(this._timeframe);
610
- }
611
- hasGaps(timeframe) {
612
- const gaps = Timeframe.dependencies(timeframe, this._timeframe);
613
- return !gaps.isEmpty();
614
- }
615
- async waitUntilReached(target) {
616
- log3("waitUntilReached", {
617
- target,
618
- current: this._timeframe
619
- }, {
620
- F: __dxlog_file4,
621
- L: 70,
622
- S: this,
623
- C: (f, a) => f(...a)
624
- });
625
- await this.update.waitForCondition(() => {
626
- log3("check if reached", {
627
- target,
628
- current: this._timeframe,
629
- deps: Timeframe.dependencies(target, this._timeframe)
630
- }, {
631
- F: __dxlog_file4,
632
- L: 72,
633
- S: this,
634
- C: (f, a) => f(...a)
635
- });
636
- return Timeframe.dependencies(target, this._timeframe).isEmpty();
637
- });
638
- }
639
- };
640
- _ts_decorate([
641
- timed(5e3)
642
- ], TimeframeClock.prototype, "waitUntilReached", null);
643
-
644
- // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
645
- import { Event as Event3, sleepWithContext, synchronized, Trigger } from "@dxos/async";
646
- import { Context as Context2, rejectOnDispose } from "@dxos/context";
647
- import { failUndefined } from "@dxos/debug";
648
- import { FeedSetIterator } from "@dxos/feed-store";
649
- import { invariant as invariant5 } from "@dxos/invariant";
650
- import { PublicKey as PublicKey2 } from "@dxos/keys";
651
- import { log as log5 } from "@dxos/log";
652
- import { Timeframe as Timeframe2 } from "@dxos/timeframe";
653
- import { ComplexMap } from "@dxos/util";
654
-
655
- // packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
656
- import { invariant as invariant4 } from "@dxos/invariant";
657
- import { log as log4 } from "@dxos/log";
658
- var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
659
- var createMessageSelector = (timeframeClock) => {
660
- return (messages) => {
661
- for (let i = 0; i < messages.length; i++) {
662
- const { data: { timeframe } } = messages[i];
663
- invariant4(timeframe, void 0, {
664
- F: __dxlog_file5,
665
- L: 25,
666
- S: void 0,
667
- A: [
668
- "timeframe",
669
- ""
670
- ]
671
- });
672
- if (!timeframeClock.hasGaps(timeframe)) {
673
- return i;
674
- }
675
- }
676
- log4("Skipping...", void 0, {
677
- F: __dxlog_file5,
678
- L: 33,
679
- S: void 0,
680
- C: (f, a) => f(...a)
681
- });
682
- };
683
- };
684
-
685
- // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
686
- function _ts_decorate2(decorators, target, key, desc) {
687
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
688
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
689
- r = Reflect.decorate(decorators, target, key, desc);
690
- else
691
- for (var i = decorators.length - 1; i >= 0; i--)
692
- if (d = decorators[i])
693
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
694
- return c > 3 && r && Object.defineProperty(target, key, r), r;
695
- }
696
- var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
697
- var PipelineState = class {
698
- constructor(_feeds, _timeframeClock) {
699
- this._feeds = _feeds;
700
- this._timeframeClock = _timeframeClock;
701
- this._ctx = new Context2(void 0, {
702
- F: __dxlog_file6,
703
- L: 41
704
- });
705
- this.timeframeUpdate = this._timeframeClock.update;
706
- this.stalled = new Event3();
707
- this._startTimeframe = new Timeframe2();
708
- this._reachedTarget = false;
709
- }
710
- /**
711
- * Latest theoretical timeframe based on the last mutation in each feed.
712
- * NOTE: This might never be reached if the mutation dependencies
713
- */
714
- // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
715
- get endTimeframe() {
716
- return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
717
- feedKey: feed.key,
718
- index: feed.length - 1
719
- })));
720
- }
721
- get startTimeframe() {
722
- return this._startTimeframe;
723
- }
724
- get timeframe() {
725
- return this._timeframeClock.timeframe;
726
- }
727
- get pendingTimeframe() {
728
- return this._timeframeClock.pendingTimeframe;
729
- }
730
- get targetTimeframe() {
731
- return this._targetTimeframe ? this._targetTimeframe : new Timeframe2();
732
- }
733
- get reachedTarget() {
734
- return this._reachedTarget;
735
- }
736
- get feeds() {
737
- return Array.from(this._feeds.values());
738
- }
739
- async waitUntilTimeframe(target) {
740
- await this._timeframeClock.waitUntilReached(target);
741
- }
742
- setTargetTimeframe(target) {
743
- this._targetTimeframe = target;
744
- }
745
- /**
746
- * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
747
- *
748
- * This function will resolve immediately if the pipeline is stalled.
749
- *
750
- * @param timeout Timeout in milliseconds to specify the maximum wait time.
751
- */
752
- async waitUntilReachedTargetTimeframe({ ctx = new Context2(void 0, {
753
- F: __dxlog_file6,
754
- L: 129
755
- }), timeout, breakOnStall = true } = {}) {
756
- log5("waitUntilReachedTargetTimeframe", {
757
- timeout,
758
- current: this.timeframe,
759
- target: this.targetTimeframe
760
- }, {
761
- F: __dxlog_file6,
762
- L: 133,
763
- S: this,
764
- C: (f, a) => f(...a)
765
- });
766
- this._reachedTargetPromise ??= Promise.race([
767
- this._timeframeClock.update.waitForCondition(() => {
768
- return Timeframe2.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
769
- }),
770
- ...breakOnStall ? [
771
- this.stalled.discardParameter().waitForCount(1)
772
- ] : []
773
- ]);
774
- let done = false;
775
- if (timeout) {
776
- return Promise.race([
777
- rejectOnDispose(ctx),
778
- rejectOnDispose(this._ctx),
779
- this._reachedTargetPromise.then(() => {
780
- done = true;
781
- this._reachedTarget = true;
782
- }),
783
- sleepWithContext(this._ctx, timeout).then(() => {
784
- if (done) {
785
- return;
786
- }
787
- log5.warn("waitUntilReachedTargetTimeframe timed out", {
788
- timeout,
789
- current: this.timeframe,
790
- target: this.targetTimeframe,
791
- dependencies: Timeframe2.dependencies(this.targetTimeframe, this.timeframe)
792
- }, {
793
- F: __dxlog_file6,
794
- L: 161,
795
- S: this,
796
- C: (f, a) => f(...a)
797
- });
798
- })
799
- ]);
800
- } else {
801
- return this._reachedTargetPromise;
802
- }
803
- }
804
- };
805
- var Pipeline = class {
806
- constructor() {
807
- this._timeframeClock = new TimeframeClock(new Timeframe2());
808
- this._feeds = new ComplexMap(PublicKey2.hash);
809
- // External state accessor.
810
- this._state = new PipelineState(this._feeds, this._timeframeClock);
811
- // Waits for the message consumer to process the message and yield control back to the pipeline.
812
- this._processingTrigger = new Trigger().wake();
813
- this._pauseTrigger = new Trigger().wake();
814
- // Pending downloads.
815
- this._downloads = new ComplexMap((value) => PublicKey2.hash(value.key));
816
- this._isStopping = false;
817
- this._isStarted = false;
818
- this._isBeingConsumed = false;
819
- this._isPaused = false;
820
- }
821
- get state() {
822
- return this._state;
823
- }
824
- get writer() {
825
- invariant5(this._writer, "Writer not set.", {
826
- F: __dxlog_file6,
827
- L: 243,
828
- S: this,
829
- A: [
830
- "this._writer",
831
- "'Writer not set.'"
832
- ]
833
- });
834
- return this._writer;
835
- }
836
- hasFeed(feedKey) {
837
- return this._feeds.has(feedKey);
838
- }
839
- getFeeds() {
840
- return this._feedSetIterator.feeds;
841
- }
842
- // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
843
- // which might be opening feeds during the mutation processing, which w
844
- async addFeed(feed) {
845
- this._feeds.set(feed.key, feed);
846
- if (this._feedSetIterator) {
847
- await this._feedSetIterator.addFeed(feed);
848
- }
849
- if (this._isStarted && !this._isPaused) {
850
- this._setFeedDownloadState(feed);
851
- }
852
- }
853
- setWriteFeed(feed) {
854
- invariant5(!this._writer, "Writer already set.", {
855
- F: __dxlog_file6,
856
- L: 270,
857
- S: this,
858
- A: [
859
- "!this._writer",
860
- "'Writer already set.'"
861
- ]
862
- });
863
- invariant5(feed.properties.writable, "Feed must be writable.", {
864
- F: __dxlog_file6,
865
- L: 271,
866
- S: this,
867
- A: [
868
- "feed.properties.writable",
869
- "'Feed must be writable.'"
870
- ]
871
- });
872
- this._writer = createMappedFeedWriter((payload) => ({
873
- timeframe: this._timeframeClock.timeframe,
874
- payload
875
- }), feed.createFeedWriter());
876
- }
877
- async start() {
878
- invariant5(!this._isStarted, "Pipeline is already started.", {
879
- F: __dxlog_file6,
880
- L: 284,
881
- S: this,
882
- A: [
883
- "!this._isStarted",
884
- "'Pipeline is already started.'"
885
- ]
886
- });
887
- log5("starting...", void 0, {
888
- F: __dxlog_file6,
889
- L: 285,
890
- S: this,
891
- C: (f, a) => f(...a)
892
- });
893
- await this._initIterator();
894
- await this._feedSetIterator.open();
895
- this._isStarted = true;
896
- log5("started", void 0, {
897
- F: __dxlog_file6,
898
- L: 289,
899
- S: this,
900
- C: (f, a) => f(...a)
901
- });
902
- if (!this._isPaused) {
903
- for (const feed of this._feeds.values()) {
904
- this._setFeedDownloadState(feed);
905
- }
906
- }
907
- }
908
- async stop() {
909
- log5("stopping...", void 0, {
910
- F: __dxlog_file6,
911
- L: 300,
912
- S: this,
913
- C: (f, a) => f(...a)
914
- });
915
- this._isStopping = true;
916
- for (const [feed, handle] of this._downloads.entries()) {
917
- feed.undownload(handle);
918
- }
919
- this._downloads.clear();
920
- await this._feedSetIterator?.close();
921
- await this._processingTrigger.wait();
922
- await this._state._ctx.dispose();
923
- this._state._ctx = new Context2(void 0, {
924
- F: __dxlog_file6,
925
- L: 309
926
- });
927
- this._state._reachedTargetPromise = void 0;
928
- this._state._reachedTarget = false;
929
- this._isStarted = false;
930
- log5("stopped", void 0, {
931
- F: __dxlog_file6,
932
- L: 313,
933
- S: this,
934
- C: (f, a) => f(...a)
935
- });
936
- }
937
- /**
938
- * @param timeframe Timeframe of already processed messages.
939
- * The pipeline will start processing messages AFTER this timeframe.
940
- */
941
- async setCursor(timeframe) {
942
- invariant5(!this._isStarted || this._isPaused, "Invalid state.", {
943
- F: __dxlog_file6,
944
- L: 322,
945
- S: this,
946
- A: [
947
- "!this._isStarted || this._isPaused",
948
- "'Invalid state.'"
949
- ]
950
- });
951
- this._state._startTimeframe = timeframe;
952
- this._timeframeClock.setTimeframe(timeframe);
953
- if (this._feedSetIterator) {
954
- await this._feedSetIterator.close();
955
- await this._initIterator();
956
- await this._feedSetIterator.open();
957
- }
958
- }
959
- /**
960
- * Calling pause while processing will cause a deadlock.
961
- */
962
- async pause() {
963
- if (this._isPaused) {
964
- return;
965
- }
966
- this._pauseTrigger.reset();
967
- await this._processingTrigger.wait();
968
- this._isPaused = true;
969
- }
970
- async unpause() {
971
- invariant5(this._isPaused, "Pipeline is not paused.", {
972
- F: __dxlog_file6,
973
- L: 351,
974
- S: this,
975
- A: [
976
- "this._isPaused",
977
- "'Pipeline is not paused.'"
978
- ]
979
- });
980
- this._pauseTrigger.wake();
981
- this._isPaused = false;
982
- for (const feed of this._feeds.values()) {
983
- this._setFeedDownloadState(feed);
984
- }
985
- }
986
- /**
987
- * Starts to iterate over the ordered messages from the added feeds.
988
- * Updates the timeframe clock after the message has bee processed.
989
- */
990
- async *consume() {
991
- invariant5(!this._isBeingConsumed, "Pipeline is already being consumed.", {
992
- F: __dxlog_file6,
993
- L: 366,
994
- S: this,
995
- A: [
996
- "!this._isBeingConsumed",
997
- "'Pipeline is already being consumed.'"
998
- ]
999
- });
1000
- this._isBeingConsumed = true;
1001
- invariant5(this._feedSetIterator, "Iterator not initialized.", {
1002
- F: __dxlog_file6,
1003
- L: 369,
1004
- S: this,
1005
- A: [
1006
- "this._feedSetIterator",
1007
- "'Iterator not initialized.'"
1008
- ]
1009
- });
1010
- let lastFeedSetIterator = this._feedSetIterator;
1011
- let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1012
- while (!this._isStopping) {
1013
- await this._pauseTrigger.wait();
1014
- if (lastFeedSetIterator !== this._feedSetIterator) {
1015
- invariant5(this._feedSetIterator, "Iterator not initialized.", {
1016
- F: __dxlog_file6,
1017
- L: 378,
1018
- S: this,
1019
- A: [
1020
- "this._feedSetIterator",
1021
- "'Iterator not initialized.'"
1022
- ]
1023
- });
1024
- lastFeedSetIterator = this._feedSetIterator;
1025
- iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1026
- }
1027
- const { done, value } = await iterable.next();
1028
- if (!done) {
1029
- const block = value ?? failUndefined();
1030
- this._processingTrigger.reset();
1031
- this._timeframeClock.updatePendingTimeframe(PublicKey2.from(block.feedKey), block.seq);
1032
- yield block;
1033
- this._processingTrigger.wake();
1034
- this._timeframeClock.updateTimeframe();
1035
- }
1036
- }
1037
- this._isBeingConsumed = false;
1038
- }
1039
- _setFeedDownloadState(feed) {
1040
- let handle = this._downloads.get(feed);
1041
- if (handle) {
1042
- feed.undownload(handle);
1043
- }
1044
- const timeframe = this._state._startTimeframe;
1045
- const seq = timeframe.get(feed.key) ?? -1;
1046
- log5("download", {
1047
- feed: feed.key.truncate(),
1048
- seq,
1049
- length: feed.length
1050
- }, {
1051
- F: __dxlog_file6,
1052
- L: 407,
1053
- S: this,
1054
- C: (f, a) => f(...a)
1055
- });
1056
- handle = feed.download({
1057
- start: seq + 1,
1058
- linear: true
1059
- }, (err, data) => {
1060
- if (err) {
1061
- } else {
1062
- log5.info("downloaded", {
1063
- data
1064
- }, {
1065
- F: __dxlog_file6,
1066
- L: 412,
1067
- S: this,
1068
- C: (f, a) => f(...a)
1069
- });
1070
- }
1071
- });
1072
- this._downloads.set(feed, handle);
1073
- }
1074
- async _initIterator() {
1075
- this._feedSetIterator = new FeedSetIterator(createMessageSelector(this._timeframeClock), {
1076
- start: startAfter(this._timeframeClock.timeframe),
1077
- stallTimeout: 1e3
1078
- });
1079
- this._feedSetIterator.stalled.on((iterator) => {
1080
- log5.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
1081
- F: __dxlog_file6,
1082
- L: 426,
1083
- S: this,
1084
- C: (f, a) => f(...a)
1085
- });
1086
- this._state.stalled.emit();
1087
- });
1088
- for (const feed of this._feeds.values()) {
1089
- await this._feedSetIterator.addFeed(feed);
1090
- }
1091
- }
1092
- };
1093
- _ts_decorate2([
1094
- synchronized
1095
- ], Pipeline.prototype, "start", null);
1096
- _ts_decorate2([
1097
- synchronized
1098
- ], Pipeline.prototype, "stop", null);
1099
- _ts_decorate2([
1100
- synchronized
1101
- ], Pipeline.prototype, "setCursor", null);
1102
- _ts_decorate2([
1103
- synchronized
1104
- ], Pipeline.prototype, "pause", null);
1105
- _ts_decorate2([
1106
- synchronized
1107
- ], Pipeline.prototype, "unpause", null);
1108
-
1109
- // packages/core/echo/echo-pipeline/src/space/space.ts
1110
- import { Event as Event4, scheduleMicroTask, synchronized as synchronized2, trackLeaks as trackLeaks2 } from "@dxos/async";
1111
- import { LifecycleState, Resource as Resource3 } from "@dxos/context";
1112
- import { subtleCrypto as subtleCrypto2 } from "@dxos/crypto";
1113
- import { invariant as invariant6 } from "@dxos/invariant";
1114
- import { PublicKey as PublicKey4, SpaceId } from "@dxos/keys";
1115
- import { log as log7, logInfo } from "@dxos/log";
1116
- import { AdmittedFeed as AdmittedFeed2 } from "@dxos/protocols/proto/dxos/halo/credentials";
1117
- import { trace as trace2 } from "@dxos/tracing";
1118
- import { Callback as Callback2, ComplexMap as ComplexMap2 } from "@dxos/util";
1119
-
1120
- // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
1121
- import { DeferredTask, sleepWithContext as sleepWithContext2, trackLeaks } from "@dxos/async";
1122
- import { Context as Context3 } from "@dxos/context";
1123
- import { SpaceStateMachine } from "@dxos/credentials";
1124
- import { PublicKey as PublicKey3 } from "@dxos/keys";
1125
- import { log as log6 } from "@dxos/log";
1126
- import { AdmittedFeed } from "@dxos/protocols/proto/dxos/halo/credentials";
1127
- import { Timeframe as Timeframe3 } from "@dxos/timeframe";
1128
- import { TimeSeriesCounter, TimeUsageCounter, trace } from "@dxos/tracing";
1129
- import { Callback, tracer } from "@dxos/util";
1130
- function _ts_decorate3(decorators, target, key, desc) {
1131
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1132
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1133
- r = Reflect.decorate(decorators, target, key, desc);
1134
- else
1135
- for (var i = decorators.length - 1; i >= 0; i--)
1136
- if (d = decorators[i])
1137
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1138
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1139
- }
1140
- var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
1141
- var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
1142
- var CONTROL_PIPELINE_SNAPSHOT_DELAY = 1e4;
1143
- var USE_SNAPSHOTS = true;
1144
- var ControlPipeline = class {
1145
- constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
1146
- this._ctx = new Context3(void 0, {
1147
- F: __dxlog_file7,
1148
- L: 47
1149
- });
1150
- this._lastTimeframeSaveTime = Date.now();
1151
- this.onFeedAdmitted = new Callback();
1152
- this._usage = new TimeUsageCounter();
1153
- this._mutations = new TimeSeriesCounter();
1154
- this._snapshotTask = new DeferredTask(this._ctx, async () => {
1155
- await sleepWithContext2(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
1156
- await this._saveSnapshot();
1157
- });
1158
- this._spaceKey = spaceKey;
1159
- this._metadata = metadataStore;
1160
- this._pipeline = new Pipeline();
1161
- void this._pipeline.addFeed(genesisFeed);
1162
- this._spaceStateMachine = new SpaceStateMachine(spaceKey);
1163
- this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
1164
- log6("feed admitted", {
1165
- key: info.key
1166
- }, {
1167
- F: __dxlog_file7,
1168
- L: 82,
1169
- S: this,
1170
- C: (f, a) => f(...a)
1171
- });
1172
- if (info.assertion.designation === AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
1173
- queueMicrotask(async () => {
1174
- try {
1175
- const feed = await feedProvider(info.key);
1176
- if (!this._pipeline.hasFeed(feed.key)) {
1177
- await this._pipeline.addFeed(feed);
1178
- }
1179
- } catch (err) {
1180
- log6.catch(err, void 0, {
1181
- F: __dxlog_file7,
1182
- L: 93,
1183
- S: this,
1184
- C: (f, a) => f(...a)
1185
- });
1186
- }
1187
- });
1188
- }
1189
- await this.onFeedAdmitted.callIfSet(info);
1190
- });
1191
- this.onMemberRoleChanged = this._spaceStateMachine.onMemberRoleChanged;
1192
- this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
1193
- this.onDelegatedInvitation = this._spaceStateMachine.onDelegatedInvitation;
1194
- this.onDelegatedInvitationRemoved = this._spaceStateMachine.onDelegatedInvitationRemoved;
1195
- }
1196
- get spaceState() {
1197
- return this._spaceStateMachine;
1198
- }
1199
- get pipeline() {
1200
- return this._pipeline;
1201
- }
1202
- async setWriteFeed(feed) {
1203
- await this._pipeline.addFeed(feed);
1204
- this._pipeline.setWriteFeed(feed);
1205
- }
1206
- async start() {
1207
- const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
1208
- log6("load snapshot", {
1209
- key: this._spaceKey,
1210
- present: !!snapshot,
1211
- tf: snapshot?.timeframe
1212
- }, {
1213
- F: __dxlog_file7,
1214
- L: 123,
1215
- S: this,
1216
- C: (f, a) => f(...a)
1217
- });
1218
- if (USE_SNAPSHOTS && snapshot) {
1219
- await this._processSnapshot(snapshot);
1220
- }
1221
- log6("starting...", void 0, {
1222
- F: __dxlog_file7,
1223
- L: 128,
1224
- S: this,
1225
- C: (f, a) => f(...a)
1226
- });
1227
- setTimeout(async () => {
1228
- void this._consumePipeline(new Context3(void 0, {
1229
- F: __dxlog_file7,
1230
- L: 130
1231
- }));
1232
- });
1233
- await this._pipeline.start();
1234
- log6("started", void 0, {
1235
- F: __dxlog_file7,
1236
- L: 134,
1237
- S: this,
1238
- C: (f, a) => f(...a)
1239
- });
1240
- }
1241
- async _processSnapshot(snapshot) {
1242
- await this._pipeline.setCursor(snapshot.timeframe);
1243
- for (const message of snapshot.messages ?? []) {
1244
- const result = await this._spaceStateMachine.process(message.credential, {
1245
- sourceFeed: message.feedKey,
1246
- skipVerification: true
1247
- });
1248
- if (!result) {
1249
- log6.warn("credential processing failed from snapshot", {
1250
- message
1251
- }, {
1252
- F: __dxlog_file7,
1253
- L: 147,
1254
- S: this,
1255
- C: (f, a) => f(...a)
1256
- });
1257
- }
1258
- }
1259
- }
1260
- async _saveSnapshot() {
1261
- await this._pipeline.pause();
1262
- const snapshot = {
1263
- timeframe: this._pipeline.state.timeframe,
1264
- messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
1265
- feedKey: entry.sourceFeed,
1266
- credential: entry.credential
1267
- }))
1268
- };
1269
- await this._pipeline.unpause();
1270
- log6("save snapshot", {
1271
- key: this._spaceKey,
1272
- snapshot
1273
- }, {
1274
- F: __dxlog_file7,
1275
- L: 163,
1276
- S: this,
1277
- C: (f, a) => f(...a)
1278
- });
1279
- await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
1280
- }
1281
- async _consumePipeline(ctx) {
1282
- for await (const msg of this._pipeline.consume()) {
1283
- const span = this._usage.beginRecording();
1284
- this._mutations.inc();
1285
- try {
1286
- await this._processMessage(ctx, msg);
1287
- } catch (err) {
1288
- log6.catch(err, void 0, {
1289
- F: __dxlog_file7,
1290
- L: 176,
1291
- S: this,
1292
- C: (f, a) => f(...a)
1293
- });
1294
- }
1295
- span.end();
1296
- }
1297
- }
1298
- async _processMessage(ctx, msg) {
1299
- log6("processing", {
1300
- key: msg.feedKey,
1301
- seq: msg.seq
1302
- }, {
1303
- F: __dxlog_file7,
1304
- L: 186,
1305
- S: this,
1306
- C: (f, a) => f(...a)
1307
- });
1308
- if (msg.data.payload.credential) {
1309
- const timer = tracer.mark("dxos.echo.pipeline.control");
1310
- const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
1311
- sourceFeed: PublicKey3.from(msg.feedKey)
1312
- });
1313
- timer.end();
1314
- if (!result) {
1315
- log6.warn("processing failed", {
1316
- msg
1317
- }, {
1318
- F: __dxlog_file7,
1319
- L: 195,
1320
- S: this,
1321
- C: (f, a) => f(...a)
1322
- });
1323
- } else {
1324
- await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
1325
- }
1326
- this._snapshotTask.schedule();
1327
- }
1328
- }
1329
- async _noteTargetStateIfNeeded(timeframe) {
1330
- if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
1331
- this._lastTimeframeSaveTime = Date.now();
1332
- await this._saveTargetTimeframe(timeframe);
1333
- }
1334
- }
1335
- async stop() {
1336
- log6("stopping...", void 0, {
1337
- F: __dxlog_file7,
1338
- L: 215,
1339
- S: this,
1340
- C: (f, a) => f(...a)
1341
- });
1342
- await this._ctx.dispose();
1343
- await this._pipeline.stop();
1344
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1345
- log6("stopped", void 0, {
1346
- F: __dxlog_file7,
1347
- L: 219,
1348
- S: this,
1349
- C: (f, a) => f(...a)
1350
- });
1351
- }
1352
- async _saveTargetTimeframe(timeframe) {
1353
- try {
1354
- const newTimeframe = Timeframe3.merge(this._targetTimeframe ?? new Timeframe3(), timeframe);
1355
- await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
1356
- this._targetTimeframe = newTimeframe;
1357
- } catch (err) {
1358
- log6(err, void 0, {
1359
- F: __dxlog_file7,
1360
- L: 228,
1361
- S: this,
1362
- C: (f, a) => f(...a)
1363
- });
1364
- }
1365
- }
1366
- };
1367
- _ts_decorate3([
1368
- trace.metricsCounter()
1369
- ], ControlPipeline.prototype, "_usage", void 0);
1370
- _ts_decorate3([
1371
- trace.metricsCounter()
1372
- ], ControlPipeline.prototype, "_mutations", void 0);
1373
- _ts_decorate3([
1374
- trace.span({
1375
- showInBrowserTimeline: true
1376
- })
1377
- ], ControlPipeline.prototype, "start", null);
1378
- _ts_decorate3([
1379
- trace.span()
1380
- ], ControlPipeline.prototype, "_consumePipeline", null);
1381
- _ts_decorate3([
1382
- trace.span()
1383
- ], ControlPipeline.prototype, "_processMessage", null);
1384
- ControlPipeline = _ts_decorate3([
1385
- trace.resource(),
1386
- trackLeaks("start", "stop")
1387
- ], ControlPipeline);
1388
-
1389
- // packages/core/echo/echo-pipeline/src/space/space.ts
1390
- function _ts_decorate4(decorators, target, key, desc) {
1391
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1392
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1393
- r = Reflect.decorate(decorators, target, key, desc);
1394
- else
1395
- for (var i = decorators.length - 1; i >= 0; i--)
1396
- if (d = decorators[i])
1397
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1398
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1399
- }
1400
- var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
1401
- var Space = class extends Resource3 {
1402
- constructor(params) {
1403
- super();
1404
- this.onCredentialProcessed = new Callback2();
1405
- this.stateUpdate = new Event4();
1406
- invariant6(params.spaceKey && params.feedProvider, void 0, {
1407
- F: __dxlog_file8,
1408
- L: 76,
1409
- S: this,
1410
- A: [
1411
- "params.spaceKey && params.feedProvider",
1412
- ""
1413
- ]
1414
- });
1415
- this._id = params.id;
1416
- this._key = params.spaceKey;
1417
- this._genesisFeedKey = params.genesisFeed.key;
1418
- this._feedProvider = params.feedProvider;
1419
- this._snapshotManager = params.snapshotManager;
1420
- this._controlPipeline = new ControlPipeline({
1421
- spaceKey: params.spaceKey,
1422
- genesisFeed: params.genesisFeed,
1423
- feedProvider: params.feedProvider,
1424
- metadataStore: params.metadataStore
1425
- });
1426
- this._controlPipeline.onFeedAdmitted.set(async (info) => {
1427
- const sparse = info.assertion.designation === AdmittedFeed2.Designation.DATA;
1428
- if (!info.key.equals(params.genesisFeed.key)) {
1429
- scheduleMicroTask(this._ctx, async () => {
1430
- await this.protocol.addFeed(await params.feedProvider(info.key, {
1431
- sparse
1432
- }));
1433
- });
1434
- }
1435
- });
1436
- this._controlPipeline.onCredentialProcessed.set(async (credential) => {
1437
- await this.onCredentialProcessed.callIfSet(credential);
1438
- log7("onCredentialProcessed", {
1439
- credential
1440
- }, {
1441
- F: __dxlog_file8,
1442
- L: 104,
1443
- S: this,
1444
- C: (f, a) => f(...a)
1445
- });
1446
- this.stateUpdate.emit();
1447
- });
1448
- this._controlPipeline.onDelegatedInvitation.set(async (invitation) => {
1449
- log7("onDelegatedInvitation", {
1450
- invitation
1451
- }, {
1452
- F: __dxlog_file8,
1453
- L: 108,
1454
- S: this,
1455
- C: (f, a) => f(...a)
1456
- });
1457
- await params.onDelegatedInvitationStatusChange(invitation, true);
1458
- });
1459
- this._controlPipeline.onDelegatedInvitationRemoved.set(async (invitation) => {
1460
- log7("onDelegatedInvitationRemoved", {
1461
- invitation
1462
- }, {
1463
- F: __dxlog_file8,
1464
- L: 112,
1465
- S: this,
1466
- C: (f, a) => f(...a)
1467
- });
1468
- await params.onDelegatedInvitationStatusChange(invitation, false);
1469
- });
1470
- this._controlPipeline.onMemberRoleChanged.set(async (changedMembers) => {
1471
- log7("onMemberRoleChanged", () => ({
1472
- changedMembers: changedMembers.map((m) => [
1473
- m.key,
1474
- m.role
1475
- ])
1476
- }), {
1477
- F: __dxlog_file8,
1478
- L: 116,
1479
- S: this,
1480
- C: (f, a) => f(...a)
1481
- });
1482
- await params.onMemberRolesChanged(changedMembers);
1483
- });
1484
- this.protocol = params.protocol;
1485
- }
1486
- get id() {
1487
- return this._id;
1488
- }
1489
- get key() {
1490
- return this._key;
1491
- }
1492
- get isOpen() {
1493
- return this._lifecycleState === LifecycleState.OPEN;
1494
- }
1495
- get genesisFeedKey() {
1496
- return this._genesisFeedKey;
1497
- }
1498
- get controlFeedKey() {
1499
- return this._controlFeed?.key;
1500
- }
1501
- get dataFeedKey() {
1502
- return this._dataFeed?.key;
1503
- }
1504
- get spaceState() {
1505
- return this._controlPipeline.spaceState;
1506
- }
1507
- /**
1508
- * @test-only
1509
- */
1510
- get controlPipeline() {
1511
- return this._controlPipeline.pipeline;
1512
- }
1513
- get snapshotManager() {
1514
- return this._snapshotManager;
1515
- }
1516
- async setControlFeed(feed) {
1517
- invariant6(!this._controlFeed, "Control feed already set.", {
1518
- F: __dxlog_file8,
1519
- L: 168,
1520
- S: this,
1521
- A: [
1522
- "!this._controlFeed",
1523
- "'Control feed already set.'"
1524
- ]
1525
- });
1526
- this._controlFeed = feed;
1527
- await this._controlPipeline.setWriteFeed(feed);
1528
- return this;
1529
- }
1530
- async setDataFeed(feed) {
1531
- invariant6(!this._dataFeed, "Data feed already set.", {
1532
- F: __dxlog_file8,
1533
- L: 175,
1534
- S: this,
1535
- A: [
1536
- "!this._dataFeed",
1537
- "'Data feed already set.'"
1538
- ]
1539
- });
1540
- this._dataFeed = feed;
1541
- return this;
1542
- }
1543
- /**
1544
- * Use for diagnostics.
1545
- */
1546
- getControlFeeds() {
1547
- return Array.from(this._controlPipeline.spaceState.feeds.values());
1548
- }
1549
- async _open(ctx) {
1550
- log7("opening...", void 0, {
1551
- F: __dxlog_file8,
1552
- L: 189,
1553
- S: this,
1554
- C: (f, a) => f(...a)
1555
- });
1556
- await this._controlPipeline.start();
1557
- await this.protocol.start();
1558
- await this.protocol.addFeed(await this._feedProvider(this._genesisFeedKey));
1559
- log7("opened", void 0, {
1560
- F: __dxlog_file8,
1561
- L: 196,
1562
- S: this,
1563
- C: (f, a) => f(...a)
1564
- });
1565
- }
1566
- async _close() {
1567
- log7("closing...", {
1568
- key: this._key
1569
- }, {
1570
- F: __dxlog_file8,
1571
- L: 201,
1572
- S: this,
1573
- C: (f, a) => f(...a)
1574
- });
1575
- await this.protocol.stop();
1576
- await this._controlPipeline.stop();
1577
- log7("closed", void 0, {
1578
- F: __dxlog_file8,
1579
- L: 207,
1580
- S: this,
1581
- C: (f, a) => f(...a)
1582
- });
1583
- }
1584
- };
1585
- _ts_decorate4([
1586
- trace2.info()
1587
- ], Space.prototype, "protocol", void 0);
1588
- _ts_decorate4([
1589
- trace2.info()
1590
- ], Space.prototype, "_controlPipeline", void 0);
1591
- _ts_decorate4([
1592
- logInfo,
1593
- trace2.info()
1594
- ], Space.prototype, "id", null);
1595
- _ts_decorate4([
1596
- logInfo,
1597
- trace2.info()
1598
- ], Space.prototype, "key", null);
1599
- _ts_decorate4([
1600
- trace2.span()
1601
- ], Space.prototype, "_open", null);
1602
- _ts_decorate4([
1603
- synchronized2
1604
- ], Space.prototype, "_close", null);
1605
- Space = _ts_decorate4([
1606
- trackLeaks2("open", "close"),
1607
- trace2.resource()
1608
- ], Space);
1609
- var SPACE_IDS_CACHE = new ComplexMap2(PublicKey4.hash);
1610
- var createIdFromSpaceKey = async (spaceKey) => {
1611
- const cachedValue = SPACE_IDS_CACHE.get(spaceKey);
1612
- if (cachedValue !== void 0) {
1613
- return cachedValue;
1614
- }
1615
- const digest = await subtleCrypto2.digest("SHA-256", spaceKey.asUint8Array());
1616
- const bytes = new Uint8Array(digest).slice(0, SpaceId.byteLength);
1617
- const spaceId = SpaceId.encode(bytes);
1618
- SPACE_IDS_CACHE.set(spaceKey, spaceId);
1619
- return spaceId;
1620
- };
1621
-
1622
- // packages/core/echo/echo-pipeline/src/space/admission-discovery-extension.ts
1623
- import { scheduleTask as scheduleTask3 } from "@dxos/async";
1624
- import { Context as Context4 } from "@dxos/context";
1625
- import { ProtocolError, schema as schema5 } from "@dxos/protocols";
1626
- import { RpcExtension as RpcExtension2 } from "@dxos/teleport";
1627
- var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/admission-discovery-extension.ts";
1628
- var CredentialRetrieverExtension = class extends RpcExtension2 {
1629
- constructor(_request, _onResult) {
1630
- super({
1631
- requested: {
1632
- AdmissionDiscoveryService: schema5.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
1633
- }
1634
- });
1635
- this._request = _request;
1636
- this._onResult = _onResult;
1637
- this._ctx = new Context4(void 0, {
1638
- F: __dxlog_file9,
1639
- L: 25
1640
- });
1641
- }
1642
- async getHandlers() {
1643
- return {};
1644
- }
1645
- async onOpen(context) {
1646
- await super.onOpen(context);
1647
- scheduleTask3(this._ctx, async () => {
1648
- try {
1649
- const result = await this.rpc.AdmissionDiscoveryService.getAdmissionCredential(this._request);
1650
- this._onResult.wake(result.admissionCredential);
1651
- } catch (err) {
1652
- context.close(err);
1653
- }
1654
- });
1655
- }
1656
- async onClose() {
1657
- await this._ctx.dispose();
1658
- }
1659
- async onAbort() {
1660
- await this._ctx.dispose();
1661
- }
1662
- };
1663
- var CredentialServerExtension = class extends RpcExtension2 {
1664
- constructor(_space) {
1665
- super({
1666
- exposed: {
1667
- AdmissionDiscoveryService: schema5.getService("dxos.mesh.teleport.AdmissionDiscoveryService")
1668
- }
1669
- });
1670
- this._space = _space;
1671
- }
1672
- async getHandlers() {
1673
- return {
1674
- AdmissionDiscoveryService: {
1675
- getAdmissionCredential: async (request) => {
1676
- const memberInfo = this._space.spaceState.members.get(request.memberKey);
1677
- if (!memberInfo?.credential) {
1678
- throw new ProtocolError("Space member not found.", request);
1679
- }
1680
- return {
1681
- admissionCredential: memberInfo.credential
1682
- };
1683
- }
1684
- }
1685
- };
1686
- }
1687
- };
1688
-
1689
- // packages/core/echo/echo-pipeline/src/space/space-protocol.ts
1690
- import { discoveryKey, subtleCrypto as subtleCrypto3 } from "@dxos/crypto";
1691
- import { PublicKey as PublicKey5 } from "@dxos/keys";
1692
- import { log as log8, logInfo as logInfo2 } from "@dxos/log";
1693
- import { MMSTTopology } from "@dxos/network-manager";
1694
- import { Teleport } from "@dxos/teleport";
1695
- import { BlobSync } from "@dxos/teleport-extension-object-sync";
1696
- import { ReplicatorExtension } from "@dxos/teleport-extension-replicator";
1697
- import { trace as trace3 } from "@dxos/tracing";
1698
- import { CallbackCollection, ComplexMap as ComplexMap3 } from "@dxos/util";
1699
- function _ts_decorate5(decorators, target, key, desc) {
1700
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1701
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1702
- r = Reflect.decorate(decorators, target, key, desc);
1703
- else
1704
- for (var i = decorators.length - 1; i >= 0; i--)
1705
- if (d = decorators[i])
1706
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1707
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1708
- }
1709
- var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
1710
- var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
1711
- var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
1712
- var SpaceProtocol = class {
1713
- constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore, disableP2pReplication }) {
1714
- this._feeds = /* @__PURE__ */ new Set();
1715
- this._sessions = new ComplexMap3(PublicKey5.hash);
1716
- // TODO(burdon): Move to config (with sensible defaults).
1717
- this._topology = new MMSTTopology({
1718
- originateConnections: 4,
1719
- maxPeers: 10,
1720
- sampleSize: 20
1721
- });
1722
- this.feedAdded = new CallbackCollection();
1723
- this._spaceKey = topic;
1724
- this._networkManager = networkManager;
1725
- this._swarmIdentity = swarmIdentity;
1726
- this._onSessionAuth = onSessionAuth;
1727
- this._onAuthFailure = onAuthFailure;
1728
- this.blobSync = new BlobSync({
1729
- blobStore
1730
- });
1731
- this._topic = subtleCrypto3.digest("SHA-256", topic.asBuffer()).then(discoveryKey).then(PublicKey5.from);
1732
- this._disableP2pReplication = disableP2pReplication ?? false;
1733
- }
1734
- get sessions() {
1735
- return this._sessions;
1736
- }
1737
- get feeds() {
1738
- return this._feeds;
1739
- }
1740
- get _ownPeerKey() {
1741
- return this._swarmIdentity.peerKey;
1742
- }
1743
- // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
1744
- async addFeed(feed) {
1745
- log8("addFeed", {
1746
- key: feed.key
1747
- }, {
1748
- F: __dxlog_file10,
1749
- L: 127,
1750
- S: this,
1751
- C: (f, a) => f(...a)
1752
- });
1753
- this._feeds.add(feed);
1754
- for (const session of this._sessions.values()) {
1755
- session.replicator.addFeed(feed);
1756
- }
1757
- await this.feedAdded.callSerial(feed);
1758
- }
1759
- // TODO(burdon): Rename open? Common open/close interfaces for all services?
1760
- async start() {
1761
- if (this._connection) {
1762
- return;
1763
- }
1764
- const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(""));
1765
- await this.blobSync.open();
1766
- log8("starting...", void 0, {
1767
- F: __dxlog_file10,
1768
- L: 148,
1769
- S: this,
1770
- C: (f, a) => f(...a)
1771
- });
1772
- const topic = await this._topic;
1773
- this._connection = await this._networkManager.joinSwarm({
1774
- protocolProvider: this._createProtocolProvider(credentials),
1775
- peerId: this._swarmIdentity.peerKey,
1776
- topic,
1777
- topology: this._topology,
1778
- label: `swarm ${topic.truncate()} for space ${this._spaceKey.truncate()}`
1779
- });
1780
- log8("started", void 0, {
1781
- F: __dxlog_file10,
1782
- L: 158,
1783
- S: this,
1784
- C: (f, a) => f(...a)
1785
- });
1786
- }
1787
- updateTopology() {
1788
- this._topology.forceUpdate();
1789
- }
1790
- async stop() {
1791
- await this.blobSync.close();
1792
- if (this._connection) {
1793
- log8("stopping...", void 0, {
1794
- F: __dxlog_file10,
1795
- L: 169,
1796
- S: this,
1797
- C: (f, a) => f(...a)
1798
- });
1799
- await this._connection.close();
1800
- log8("stopped", void 0, {
1801
- F: __dxlog_file10,
1802
- L: 171,
1803
- S: this,
1804
- C: (f, a) => f(...a)
1805
- });
1806
- }
1807
- }
1808
- _createProtocolProvider(credentials) {
1809
- return (wireParams) => {
1810
- const session = new SpaceProtocolSession({
1811
- wireParams,
1812
- swarmIdentity: this._swarmIdentity,
1813
- onSessionAuth: this._onSessionAuth,
1814
- onAuthFailure: this._onAuthFailure,
1815
- blobSync: this.blobSync,
1816
- disableP2pReplication: this._disableP2pReplication
1817
- });
1818
- this._sessions.set(wireParams.remotePeerId, session);
1819
- for (const feed of this._feeds) {
1820
- session.replicator.addFeed(feed);
1821
- }
1822
- return session;
1823
- };
1824
- }
1825
- };
1826
- _ts_decorate5([
1827
- logInfo2,
1828
- trace3.info()
1829
- ], SpaceProtocol.prototype, "_topic", void 0);
1830
- _ts_decorate5([
1831
- trace3.info()
1832
- ], SpaceProtocol.prototype, "_spaceKey", void 0);
1833
- _ts_decorate5([
1834
- logInfo2
1835
- ], SpaceProtocol.prototype, "_ownPeerKey", null);
1836
- SpaceProtocol = _ts_decorate5([
1837
- trace3.resource()
1838
- ], SpaceProtocol);
1839
- var AuthStatus;
1840
- (function(AuthStatus2) {
1841
- AuthStatus2["INITIAL"] = "INITIAL";
1842
- AuthStatus2["SUCCESS"] = "SUCCESS";
1843
- AuthStatus2["FAILURE"] = "FAILURE";
1844
- })(AuthStatus || (AuthStatus = {}));
1845
- var SpaceProtocolSession = class {
1846
- // TODO(dmaretskyi): Allow to pass in extra extensions.
1847
- constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync, disableP2pReplication }) {
1848
- // TODO(dmaretskyi): Start with upload=false when switching it on the fly works.
1849
- this.replicator = new ReplicatorExtension().setOptions({
1850
- upload: true
1851
- });
1852
- this._authStatus = "INITIAL";
1853
- this._wireParams = wireParams;
1854
- this._swarmIdentity = swarmIdentity;
1855
- this._onSessionAuth = onSessionAuth;
1856
- this._onAuthFailure = onAuthFailure;
1857
- this._blobSync = blobSync;
1858
- this._teleport = new Teleport(wireParams);
1859
- this._disableP2pReplication = disableP2pReplication ?? false;
1860
- }
1861
- get authStatus() {
1862
- return this._authStatus;
1863
- }
1864
- get stats() {
1865
- return this._teleport.stats;
1866
- }
1867
- get stream() {
1868
- return this._teleport.stream;
1869
- }
1870
- async open(sessionId) {
1871
- await this._teleport.open(sessionId);
1872
- this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
1873
- provider: this._swarmIdentity.credentialProvider,
1874
- verifier: this._swarmIdentity.credentialAuthenticator,
1875
- onAuthSuccess: () => {
1876
- log8("Peer authenticated", void 0, {
1877
- F: __dxlog_file10,
1878
- L: 282,
1879
- S: this,
1880
- C: (f, a) => f(...a)
1881
- });
1882
- this._authStatus = "SUCCESS";
1883
- this._onSessionAuth?.(this._teleport);
1884
- },
1885
- onAuthFailure: () => {
1886
- this._authStatus = "FAILURE";
1887
- this._onAuthFailure?.(this._teleport);
1888
- }
1889
- }));
1890
- if (!this._disableP2pReplication) {
1891
- this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
1892
- }
1893
- this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
1894
- }
1895
- async close() {
1896
- log8("close", void 0, {
1897
- F: __dxlog_file10,
1898
- L: 301,
1899
- S: this,
1900
- C: (f, a) => f(...a)
1901
- });
1902
- await this._teleport.close();
1903
- }
1904
- async abort() {
1905
- await this._teleport.abort();
1906
- }
1907
- };
1908
- _ts_decorate5([
1909
- logInfo2
1910
- ], SpaceProtocolSession.prototype, "_wireParams", void 0);
1911
- _ts_decorate5([
1912
- logInfo2
1913
- ], SpaceProtocolSession.prototype, "authStatus", null);
1914
-
1915
- // packages/core/echo/echo-pipeline/src/space/space-manager.ts
1916
- import { synchronized as synchronized4, trackLeaks as trackLeaks3, Trigger as Trigger3 } from "@dxos/async";
1917
- import { failUndefined as failUndefined2 } from "@dxos/debug";
1918
- import { PublicKey as PublicKey8 } from "@dxos/keys";
1919
- import { log as log14 } from "@dxos/log";
1920
- import { trace as trace6 } from "@dxos/protocols";
1921
- import { ComplexMap as ComplexMap4 } from "@dxos/util";
1922
-
1923
- // packages/core/echo/echo-pipeline/src/db-host/data-service.ts
1924
- import { Stream } from "@dxos/codec-protobuf";
1925
- import { invariant as invariant12 } from "@dxos/invariant";
1926
- import { SpaceId as SpaceId3 } from "@dxos/keys";
1927
- import { log as log13 } from "@dxos/log";
1928
-
1929
- // packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
1930
- import { Event as Event5, asyncTimeout } from "@dxos/async";
1931
- import { getBackend, getHeads, isAutomerge, equals as headsEquals, save } from "@dxos/automerge/automerge";
1932
- import { Repo } from "@dxos/automerge/automerge-repo";
1933
- import { Context as Context5, Resource as Resource5, cancelWithContext as cancelWithContext2 } from "@dxos/context";
1934
- import { invariant as invariant8 } from "@dxos/invariant";
1935
- import { PublicKey as PublicKey6 } from "@dxos/keys";
1936
- import { log as log10 } from "@dxos/log";
1937
- import { objectPointerCodec } from "@dxos/protocols";
1938
- import { trace as trace4 } from "@dxos/tracing";
1939
-
1940
- // packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts
1941
- import { synchronized as synchronized3, Trigger as Trigger2 } from "@dxos/async";
1942
- import { NetworkAdapter } from "@dxos/automerge/automerge-repo";
1943
- import { LifecycleState as LifecycleState2 } from "@dxos/context";
1944
- import { invariant as invariant7 } from "@dxos/invariant";
1945
- import { log as log9 } from "@dxos/log";
1946
- import { nonNullable } from "@dxos/util";
1947
-
1948
- // packages/core/echo/echo-pipeline/src/automerge/network-protocol.ts
1949
- import { MESSAGE_TYPE_COLLECTION_QUERY, MESSAGE_TYPE_COLLECTION_STATE } from "@dxos/protocols";
1950
- var isCollectionQueryMessage = (message) => message.type === MESSAGE_TYPE_COLLECTION_QUERY;
1951
- var isCollectionStateMessage = (message) => message.type === MESSAGE_TYPE_COLLECTION_STATE;
1952
-
1953
- // packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts
1954
- function _ts_decorate6(decorators, target, key, desc) {
1955
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1956
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1957
- r = Reflect.decorate(decorators, target, key, desc);
1958
- else
1959
- for (var i = decorators.length - 1; i >= 0; i--)
1960
- if (d = decorators[i])
1961
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1962
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1963
- }
1964
- var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
1965
- var EchoNetworkAdapter = class extends NetworkAdapter {
1966
- constructor(_params) {
1967
- super();
1968
- this._params = _params;
1969
- this._replicators = /* @__PURE__ */ new Set();
1970
- this._connections = /* @__PURE__ */ new Map();
1971
- this._lifecycleState = LifecycleState2.CLOSED;
1972
- this._connected = new Trigger2();
1973
- }
1974
- connect(peerId, peerMetadata) {
1975
- this.peerId = peerId;
1976
- this.peerMetadata = peerMetadata;
1977
- this._connected.wake();
1978
- }
1979
- send(message) {
1980
- this._send(message);
1981
- }
1982
- disconnect() {
1983
- }
1984
- async open() {
1985
- if (this._lifecycleState === LifecycleState2.OPEN) {
1986
- return;
1987
- }
1988
- this._lifecycleState = LifecycleState2.OPEN;
1989
- log9("emit ready", void 0, {
1990
- F: __dxlog_file11,
1991
- L: 81,
1992
- S: this,
1993
- C: (f, a) => f(...a)
1994
- });
1995
- this.emit("ready", {
1996
- network: this
1997
- });
1998
- }
1999
- async close() {
2000
- if (this._lifecycleState === LifecycleState2.CLOSED) {
2001
- return this;
2002
- }
2003
- for (const replicator of this._replicators) {
2004
- await replicator.disconnect();
2005
- }
2006
- this._replicators.clear();
2007
- this._lifecycleState = LifecycleState2.CLOSED;
2008
- }
2009
- async whenConnected() {
2010
- await this._connected.wait({
2011
- timeout: 1e4
2012
- });
2013
- }
2014
- async addReplicator(replicator) {
2015
- invariant7(this._lifecycleState === LifecycleState2.OPEN, void 0, {
2016
- F: __dxlog_file11,
2017
- L: 107,
2018
- S: this,
2019
- A: [
2020
- "this._lifecycleState === LifecycleState.OPEN",
2021
- ""
2022
- ]
2023
- });
2024
- invariant7(this.peerId, void 0, {
2025
- F: __dxlog_file11,
2026
- L: 108,
2027
- S: this,
2028
- A: [
2029
- "this.peerId",
2030
- ""
2031
- ]
2032
- });
2033
- invariant7(!this._replicators.has(replicator), void 0, {
2034
- F: __dxlog_file11,
2035
- L: 109,
2036
- S: this,
2037
- A: [
2038
- "!this._replicators.has(replicator)",
2039
- ""
2040
- ]
2041
- });
2042
- this._replicators.add(replicator);
2043
- await replicator.connect({
2044
- peerId: this.peerId,
2045
- onConnectionOpen: this._onConnectionOpen.bind(this),
2046
- onConnectionClosed: this._onConnectionClosed.bind(this),
2047
- onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
2048
- isDocumentInRemoteCollection: this._params.isDocumentInRemoteCollection,
2049
- getContainingSpaceForDocument: this._params.getContainingSpaceForDocument,
2050
- getContainingSpaceIdForDocument: async (documentId) => {
2051
- const key = await this._params.getContainingSpaceForDocument(documentId);
2052
- return key ? createIdFromSpaceKey(key) : null;
2053
- }
2054
- });
2055
- }
2056
- async removeReplicator(replicator) {
2057
- invariant7(this._lifecycleState === LifecycleState2.OPEN, void 0, {
2058
- F: __dxlog_file11,
2059
- L: 128,
2060
- S: this,
2061
- A: [
2062
- "this._lifecycleState === LifecycleState.OPEN",
2063
- ""
2064
- ]
2065
- });
2066
- invariant7(this._replicators.has(replicator), void 0, {
2067
- F: __dxlog_file11,
2068
- L: 129,
2069
- S: this,
2070
- A: [
2071
- "this._replicators.has(replicator)",
2072
- ""
2073
- ]
2074
- });
2075
- await replicator.disconnect();
2076
- this._replicators.delete(replicator);
2077
- }
2078
- async shouldAdvertise(peerId, params) {
2079
- const connection = this._connections.get(peerId);
2080
- if (!connection) {
2081
- return false;
2082
- }
2083
- return connection.connection.shouldAdvertise(params);
2084
- }
2085
- shouldSyncCollection(peerId, params) {
2086
- const connection = this._connections.get(peerId);
2087
- if (!connection) {
2088
- return false;
2089
- }
2090
- return connection.connection.shouldSyncCollection(params);
2091
- }
2092
- queryCollectionState(collectionId, targetId) {
2093
- const message = {
2094
- type: "collection-query",
2095
- senderId: this.peerId,
2096
- targetId,
2097
- collectionId
2098
- };
2099
- this._send(message);
2100
- }
2101
- sendCollectionState(collectionId, targetId, state) {
2102
- const message = {
2103
- type: "collection-state",
2104
- senderId: this.peerId,
2105
- targetId,
2106
- collectionId,
2107
- state
2108
- };
2109
- this._send(message);
2110
- }
2111
- _send(message) {
2112
- const connectionEntry = this._connections.get(message.targetId);
2113
- if (!connectionEntry) {
2114
- throw new Error("Connection not found.");
2115
- }
2116
- const writeStart = Date.now();
2117
- connectionEntry.writer.write(message).then(() => {
2118
- const durationMs = Date.now() - writeStart;
2119
- this._params.monitor?.recordMessageSent(message, durationMs);
2120
- }).catch((err) => {
2121
- if (connectionEntry.isOpen) {
2122
- log9.catch(err, void 0, {
2123
- F: __dxlog_file11,
2124
- L: 189,
2125
- S: this,
2126
- C: (f, a) => f(...a)
2127
- });
2128
- }
2129
- this._params.monitor?.recordMessageSendingFailed(message);
2130
- });
2131
- }
2132
- // TODO(dmaretskyi): Remove.
2133
- getPeersInterestedInCollection(collectionId) {
2134
- return Array.from(this._connections.values()).map((connection) => {
2135
- return connection.connection.shouldSyncCollection({
2136
- collectionId
2137
- }) ? connection.connection.peerId : null;
2138
- }).filter(nonNullable);
2139
- }
2140
- _onConnectionOpen(connection) {
2141
- log9("Connection opened", {
2142
- peerId: connection.peerId
2143
- }, {
2144
- F: __dxlog_file11,
2145
- L: 207,
2146
- S: this,
2147
- C: (f, a) => f(...a)
2148
- });
2149
- invariant7(!this._connections.has(connection.peerId), void 0, {
2150
- F: __dxlog_file11,
2151
- L: 208,
2152
- S: this,
2153
- A: [
2154
- "!this._connections.has(connection.peerId as PeerId)",
2155
- ""
2156
- ]
2157
- });
2158
- const reader = connection.readable.getReader();
2159
- const writer = connection.writable.getWriter();
2160
- const connectionEntry = {
2161
- connection,
2162
- reader,
2163
- writer,
2164
- isOpen: true
2165
- };
2166
- this._connections.set(connection.peerId, connectionEntry);
2167
- queueMicrotask(async () => {
2168
- try {
2169
- while (true) {
2170
- const { done, value } = await reader.read();
2171
- if (done) {
2172
- break;
2173
- }
2174
- this._onMessage(value);
2175
- }
2176
- } catch (err) {
2177
- if (connectionEntry.isOpen) {
2178
- log9.catch(err, void 0, {
2179
- F: __dxlog_file11,
2180
- L: 227,
2181
- S: this,
2182
- C: (f, a) => f(...a)
2183
- });
2184
- }
2185
- }
2186
- });
2187
- log9("emit peer-candidate", {
2188
- peerId: connection.peerId
2189
- }, {
2190
- F: __dxlog_file11,
2191
- L: 232,
2192
- S: this,
2193
- C: (f, a) => f(...a)
2194
- });
2195
- this._emitPeerCandidate(connection);
2196
- this._params.monitor?.recordPeerConnected(connection.peerId);
2197
- }
2198
- _onMessage(message) {
2199
- if (isCollectionQueryMessage(message)) {
2200
- this._params.onCollectionStateQueried(message.collectionId, message.senderId);
2201
- } else if (isCollectionStateMessage(message)) {
2202
- this._params.onCollectionStateReceived(message.collectionId, message.senderId, message.state);
2203
- } else {
2204
- this.emit("message", message);
2205
- }
2206
- this._params.monitor?.recordMessageReceived(message);
2207
- }
2208
- /**
2209
- * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
2210
- * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
2211
- */
2212
- _onConnectionAuthScopeChanged(connection) {
2213
- log9("Connection auth scope changed", {
2214
- peerId: connection.peerId
2215
- }, {
2216
- F: __dxlog_file11,
2217
- L: 253,
2218
- S: this,
2219
- C: (f, a) => f(...a)
2220
- });
2221
- const entry = this._connections.get(connection.peerId);
2222
- invariant7(entry, void 0, {
2223
- F: __dxlog_file11,
2224
- L: 255,
2225
- S: this,
2226
- A: [
2227
- "entry",
2228
- ""
2229
- ]
2230
- });
2231
- this.emit("peer-disconnected", {
2232
- peerId: connection.peerId
2233
- });
2234
- this._emitPeerCandidate(connection);
2235
- }
2236
- _onConnectionClosed(connection) {
2237
- log9("Connection closed", {
2238
- peerId: connection.peerId
2239
- }, {
2240
- F: __dxlog_file11,
2241
- L: 261,
2242
- S: this,
2243
- C: (f, a) => f(...a)
2244
- });
2245
- const entry = this._connections.get(connection.peerId);
2246
- invariant7(entry, void 0, {
2247
- F: __dxlog_file11,
2248
- L: 263,
2249
- S: this,
2250
- A: [
2251
- "entry",
2252
- ""
2253
- ]
2254
- });
2255
- entry.isOpen = false;
2256
- this.emit("peer-disconnected", {
2257
- peerId: connection.peerId
2258
- });
2259
- this._params.monitor?.recordPeerDisconnected(connection.peerId);
2260
- void entry.reader.cancel().catch((err) => log9.catch(err, void 0, {
2261
- F: __dxlog_file11,
2262
- L: 269,
2263
- S: this,
2264
- C: (f, a) => f(...a)
2265
- }));
2266
- void entry.writer.abort().catch((err) => log9.catch(err, void 0, {
2267
- F: __dxlog_file11,
2268
- L: 270,
2269
- S: this,
2270
- C: (f, a) => f(...a)
2271
- }));
2272
- this._connections.delete(connection.peerId);
2273
- }
2274
- _emitPeerCandidate(connection) {
2275
- this.emit("peer-candidate", {
2276
- peerId: connection.peerId,
2277
- peerMetadata: createEchoPeerMetadata()
2278
- });
2279
- }
2280
- };
2281
- _ts_decorate6([
2282
- synchronized3
2283
- ], EchoNetworkAdapter.prototype, "open", null);
2284
- _ts_decorate6([
2285
- synchronized3
2286
- ], EchoNetworkAdapter.prototype, "close", null);
2287
- _ts_decorate6([
2288
- synchronized3
2289
- ], EchoNetworkAdapter.prototype, "addReplicator", null);
2290
- _ts_decorate6([
2291
- synchronized3
2292
- ], EchoNetworkAdapter.prototype, "removeReplicator", null);
2293
- var createEchoPeerMetadata = () => ({
2294
- // TODO(dmaretskyi): Refactor this.
2295
- dxos_peerSource: "EchoNetworkAdapter"
2296
- });
2297
- var isEchoPeerMetadata = (metadata) => metadata?.dxos_peerSource === "EchoNetworkAdapter";
2298
-
2299
- // packages/core/echo/echo-pipeline/src/automerge/heads-store.ts
2300
- import { headsEncoding } from "@dxos/indexing";
2301
- var HeadsStore = class {
2302
- constructor({ db }) {
2303
- this._db = db;
2304
- }
2305
- setHeads(documentId, heads, batch) {
2306
- batch.put(documentId, heads, {
2307
- sublevel: this._db,
2308
- keyEncoding: "utf8",
2309
- valueEncoding: headsEncoding
2310
- });
2311
- }
2312
- // TODO(dmaretskyi): Make batched.
2313
- async getHeads(documentIds) {
2314
- return this._db.getMany(documentIds, {
2315
- keyEncoding: "utf8",
2316
- valueEncoding: headsEncoding
2317
- });
2318
- }
2319
- };
2320
-
2321
- // packages/core/echo/echo-pipeline/src/automerge/leveldb-storage-adapter.ts
2322
- import { LifecycleState as LifecycleState3, Resource as Resource4 } from "@dxos/context";
2323
- var LevelDBStorageAdapter = class extends Resource4 {
2324
- constructor(_params) {
2325
- super();
2326
- this._params = _params;
2327
- }
2328
- async load(keyArray) {
2329
- try {
2330
- if (this._lifecycleState !== LifecycleState3.OPEN) {
2331
- return void 0;
2332
- }
2333
- const startMs = Date.now();
2334
- const chunk = await this._params.db.get(keyArray, {
2335
- ...encodingOptions
2336
- });
2337
- this._params.monitor?.recordBytesLoaded(chunk.byteLength);
2338
- this._params.monitor?.recordLoadDuration(Date.now() - startMs);
2339
- return chunk;
2340
- } catch (err) {
2341
- if (isLevelDbNotFoundError(err)) {
2342
- return void 0;
2343
- }
2344
- throw err;
2345
- }
2346
- }
2347
- async save(keyArray, binary) {
2348
- if (this._lifecycleState !== LifecycleState3.OPEN) {
2349
- return void 0;
2350
- }
2351
- const startMs = Date.now();
2352
- const batch = this._params.db.batch();
2353
- await this._params.callbacks?.beforeSave?.({
2354
- path: keyArray,
2355
- batch
2356
- });
2357
- batch.put(keyArray, Buffer.from(binary), {
2358
- ...encodingOptions
2359
- });
2360
- await batch.write();
2361
- this._params.monitor?.recordBytesStored(binary.byteLength);
2362
- await this._params.callbacks?.afterSave?.(keyArray);
2363
- this._params.monitor?.recordStoreDuration(Date.now() - startMs);
2364
- }
2365
- async remove(keyArray) {
2366
- if (this._lifecycleState !== LifecycleState3.OPEN) {
2367
- return void 0;
2368
- }
2369
- await this._params.db.del(keyArray, {
2370
- ...encodingOptions
2371
- });
2372
- }
2373
- async loadRange(keyPrefix) {
2374
- if (this._lifecycleState !== LifecycleState3.OPEN) {
2375
- return [];
2376
- }
2377
- const startMs = Date.now();
2378
- const result = [];
2379
- for await (const [key, value] of this._params.db.iterator({
2380
- gte: keyPrefix,
2381
- lte: [
2382
- ...keyPrefix,
2383
- "\uFFFF"
2384
- ],
2385
- ...encodingOptions
2386
- })) {
2387
- result.push({
2388
- key,
2389
- data: value
2390
- });
2391
- this._params.monitor?.recordBytesLoaded(value.byteLength);
2392
- }
2393
- this._params.monitor?.recordLoadDuration(Date.now() - startMs);
2394
- return result;
2395
- }
2396
- async removeRange(keyPrefix) {
2397
- if (this._lifecycleState !== LifecycleState3.OPEN) {
2398
- return void 0;
2399
- }
2400
- const batch = this._params.db.batch();
2401
- for await (const [key] of this._params.db.iterator({
2402
- gte: keyPrefix,
2403
- lte: [
2404
- ...keyPrefix,
2405
- "\uFFFF"
2406
- ],
2407
- ...encodingOptions
2408
- })) {
2409
- batch.del(key, {
2410
- ...encodingOptions
2411
- });
2412
- }
2413
- await batch.write();
2414
- }
2415
- };
2416
- var keyEncoder = {
2417
- encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
2418
- decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
2419
- format: "buffer"
2420
- };
2421
- var encodingOptions = {
2422
- keyEncoding: keyEncoder,
2423
- valueEncoding: "buffer"
2424
- };
2425
- var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
2426
-
2427
- // packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
2428
- function _ts_decorate7(decorators, target, key, desc) {
2429
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2430
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2431
- r = Reflect.decorate(decorators, target, key, desc);
2432
- else
2433
- for (var i = decorators.length - 1; i >= 0; i--)
2434
- if (d = decorators[i])
2435
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2436
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2437
- }
2438
- var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
2439
- var AutomergeHost = class extends Resource5 {
2440
- constructor({ db, indexMetadataStore, dataMonitor }) {
2441
- super();
2442
- this._collectionSynchronizer = new CollectionSynchronizer({
2443
- queryCollectionState: this._queryCollectionState.bind(this),
2444
- sendCollectionState: this._sendCollectionState.bind(this),
2445
- shouldSyncCollection: this._shouldSyncCollection.bind(this)
2446
- });
2447
- this._db = db;
2448
- this._storage = new LevelDBStorageAdapter({
2449
- db: db.sublevel("automerge"),
2450
- callbacks: {
2451
- beforeSave: async (params) => this._beforeSave(params),
2452
- afterSave: async (key) => this._afterSave(key)
2453
- },
2454
- monitor: dataMonitor
2455
- });
2456
- this._echoNetworkAdapter = new EchoNetworkAdapter({
2457
- getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
2458
- isDocumentInRemoteCollection: this._isDocumentInRemoteCollection.bind(this),
2459
- onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
2460
- onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
2461
- monitor: dataMonitor
2462
- });
2463
- this._headsStore = new HeadsStore({
2464
- db: db.sublevel("heads")
2465
- });
2466
- this._indexMetadataStore = indexMetadataStore;
2467
- }
2468
- async _open() {
2469
- this._peerId = `host-${PublicKey6.random().toHex()}`;
2470
- await this._storage.open?.();
2471
- this._repo = new Repo({
2472
- peerId: this._peerId,
2473
- sharePolicy: this._sharePolicy.bind(this),
2474
- storage: this._storage,
2475
- network: [
2476
- // Upstream swarm.
2477
- this._echoNetworkAdapter
2478
- ]
2479
- });
2480
- Event5.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => this._onPeerConnected(e.peerId));
2481
- Event5.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => this._onPeerDisconnected(e.peerId));
2482
- this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
2483
- this._onRemoteCollectionStateUpdated(collectionId, peerId);
2484
- });
2485
- await this._echoNetworkAdapter.open();
2486
- await this._collectionSynchronizer.open();
2487
- await this._echoNetworkAdapter.open();
2488
- await this._echoNetworkAdapter.whenConnected();
2489
- }
2490
- async _close() {
2491
- await this._collectionSynchronizer.close();
2492
- await this._storage.close?.();
2493
- await this._echoNetworkAdapter.close();
2494
- await this._ctx.dispose();
2495
- }
2496
- /**
2497
- * @deprecated To be abstracted away.
2498
- */
2499
- get repo() {
2500
- return this._repo;
2501
- }
2502
- get peerId() {
2503
- return this._peerId;
2504
- }
2505
- get loadedDocsCount() {
2506
- return Object.keys(this._repo.handles).length;
2507
- }
2508
- async addReplicator(replicator) {
2509
- await this._echoNetworkAdapter.addReplicator(replicator);
2510
- }
2511
- async removeReplicator(replicator) {
2512
- await this._echoNetworkAdapter.removeReplicator(replicator);
2513
- }
2514
- /**
2515
- * Loads the document handle from the repo and waits for it to be ready.
2516
- */
2517
- async loadDoc(ctx, documentId, opts) {
2518
- let handle;
2519
- if (typeof documentId === "string") {
2520
- handle = this._repo.handles[documentId];
2521
- }
2522
- if (!handle) {
2523
- handle = this._repo.find(documentId);
2524
- }
2525
- if (!handle.isReady()) {
2526
- if (!opts?.timeout) {
2527
- await cancelWithContext2(ctx, handle.whenReady());
2528
- } else {
2529
- await cancelWithContext2(ctx, asyncTimeout(handle.whenReady(), opts.timeout));
2530
- }
2531
- }
2532
- return handle;
2533
- }
2534
- /**
2535
- * Create new persisted document.
2536
- */
2537
- createDoc(initialValue, opts) {
2538
- if (opts?.preserveHistory) {
2539
- if (!isAutomerge(initialValue)) {
2540
- throw new TypeError("Initial value must be an Automerge document");
2541
- }
2542
- return this._repo.import(save(initialValue));
2543
- } else {
2544
- return this._repo.create(initialValue);
2545
- }
2546
- }
2547
- async waitUntilHeadsReplicated(heads) {
2548
- const entries = heads.entries;
2549
- if (!entries?.length) {
2550
- return;
2551
- }
2552
- const documentIds = entries.map((entry) => entry.documentId);
2553
- const documentHeads = await this.getHeads(documentIds);
2554
- const headsToWait = entries.filter((entry, index) => {
2555
- const targetHeads = entry.heads;
2556
- if (!targetHeads || targetHeads.length === 0) {
2557
- return false;
2558
- }
2559
- const currentHeads = documentHeads[index];
2560
- return !(currentHeads !== null && headsEquals(currentHeads, targetHeads));
2561
- });
2562
- if (headsToWait.length > 0) {
2563
- await Promise.all(headsToWait.map(async (entry, index) => {
2564
- const handle = await this.loadDoc(Context5.default(void 0, {
2565
- F: __dxlog_file12,
2566
- L: 227
2567
- }), entry.documentId);
2568
- await waitForHeads(handle, entry.heads);
2569
- }));
2570
- }
2571
- await this._repo.flush(documentIds.filter((documentId) => !!this._repo.handles[documentId]));
2572
- }
2573
- async reIndexHeads(documentIds) {
2574
- for (const documentId of documentIds) {
2575
- log10.info("re-indexing heads for document", {
2576
- documentId
2577
- }, {
2578
- F: __dxlog_file12,
2579
- L: 239,
2580
- S: this,
2581
- C: (f, a) => f(...a)
2582
- });
2583
- const handle = this._repo.find(documentId);
2584
- await handle.whenReady([
2585
- "ready",
2586
- "requesting"
2587
- ]);
2588
- if (handle.inState([
2589
- "requesting"
2590
- ])) {
2591
- log10.warn("document is not available locally, skipping", {
2592
- documentId
2593
- }, {
2594
- F: __dxlog_file12,
2595
- L: 243,
2596
- S: this,
2597
- C: (f, a) => f(...a)
2598
- });
2599
- continue;
2600
- }
2601
- const doc = handle.docSync();
2602
- invariant8(doc, void 0, {
2603
- F: __dxlog_file12,
2604
- L: 248,
2605
- S: this,
2606
- A: [
2607
- "doc",
2608
- ""
2609
- ]
2610
- });
2611
- const heads = getHeads(doc);
2612
- const batch = this._db.batch();
2613
- this._headsStore.setHeads(documentId, heads, batch);
2614
- await batch.write();
2615
- }
2616
- log10.info("done re-indexing heads", void 0, {
2617
- F: __dxlog_file12,
2618
- L: 255,
2619
- S: this,
2620
- C: (f, a) => f(...a)
2621
- });
2622
- }
2623
- // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
2624
- // Hosts, running in the worker, don't share documents unless requested by other peers.
2625
- // NOTE: If both peers return sharePolicy=false the replication will not happen
2626
- // https://github.com/automerge/automerge-repo/pull/292
2627
- async _sharePolicy(peerId, documentId) {
2628
- if (peerId.startsWith("client-")) {
2629
- return false;
2630
- }
2631
- if (!documentId) {
2632
- return false;
2633
- }
2634
- const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
2635
- if (isEchoPeerMetadata(peerMetadata)) {
2636
- return this._echoNetworkAdapter.shouldAdvertise(peerId, {
2637
- documentId
2638
- });
2639
- }
2640
- return false;
2641
- }
2642
- async _beforeSave({ path, batch }) {
2643
- const handle = this._repo.handles[path[0]];
2644
- if (!handle) {
2645
- return;
2646
- }
2647
- const doc = handle.docSync();
2648
- if (!doc) {
2649
- return;
2650
- }
2651
- const heads = getHeads(doc);
2652
- this._headsStore.setHeads(handle.documentId, heads, batch);
2653
- const spaceKey = getSpaceKeyFromDoc(doc) ?? void 0;
2654
- const objectIds = Object.keys(doc.objects ?? {});
2655
- const encodedIds = objectIds.map((objectId) => objectPointerCodec.encode({
2656
- documentId: handle.documentId,
2657
- objectId,
2658
- spaceKey
2659
- }));
2660
- const idToLastHash = new Map(encodedIds.map((id) => [
2661
- id,
2662
- heads
2663
- ]));
2664
- this._indexMetadataStore.markDirty(idToLastHash, batch);
2665
- }
2666
- _shouldSyncCollection(collectionId, peerId) {
2667
- const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
2668
- if (isEchoPeerMetadata(peerMetadata)) {
2669
- return this._echoNetworkAdapter.shouldSyncCollection(peerId, {
2670
- collectionId
2671
- });
2672
- }
2673
- return false;
2674
- }
2675
- /**
2676
- * Called by AutomergeStorageAdapter after levelDB batch commit.
2677
- */
2678
- async _afterSave(path) {
2679
- this._indexMetadataStore.notifyMarkedDirty();
2680
- const documentId = path[0];
2681
- const document = this._repo.handles[documentId]?.docSync();
2682
- if (document) {
2683
- const heads = getHeads(document);
2684
- this._onHeadsChanged(documentId, heads);
2685
- }
2686
- }
2687
- _automergePeers() {
2688
- return this._repo.peers;
2689
- }
2690
- async _isDocumentInRemoteCollection(params) {
2691
- for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
2692
- const remoteCollections = this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
2693
- const remotePeerDocs = remoteCollections.get(params.peerId)?.documents;
2694
- if (remotePeerDocs && params.documentId in remotePeerDocs) {
2695
- return true;
2696
- }
2697
- }
2698
- return false;
2699
- }
2700
- async _getContainingSpaceForDocument(documentId) {
2701
- const doc = this._repo.handles[documentId]?.docSync();
2702
- if (!doc) {
2703
- return null;
2704
- }
2705
- const spaceKeyHex = getSpaceKeyFromDoc(doc);
2706
- if (!spaceKeyHex) {
2707
- return null;
2708
- }
2709
- return PublicKey6.from(spaceKeyHex);
2710
- }
2711
- /**
2712
- * Flush documents to disk.
2713
- */
2714
- async flush({ documentIds } = {}) {
2715
- await this._repo.flush(documentIds);
2716
- }
2717
- async getHeads(documentIds) {
2718
- const result = [];
2719
- const storeRequestIds = [];
2720
- const storeResultIndices = [];
2721
- for (const documentId of documentIds) {
2722
- const doc = this._repo.handles[documentId]?.docSync();
2723
- if (doc) {
2724
- result.push(getHeads(doc));
2725
- } else {
2726
- storeRequestIds.push(documentId);
2727
- storeResultIndices.push(result.length);
2728
- result.push(void 0);
2729
- }
2730
- }
2731
- if (storeRequestIds.length > 0) {
2732
- const storedHeads = await this._headsStore.getHeads(storeRequestIds);
2733
- for (let i = 0; i < storedHeads.length; i++) {
2734
- result[storeResultIndices[i]] = storedHeads[i];
2735
- }
2736
- }
2737
- return result;
2738
- }
2739
- //
2740
- // Collection sync.
2741
- //
2742
- getLocalCollectionState(collectionId) {
2743
- return this._collectionSynchronizer.getLocalCollectionState(collectionId);
2744
- }
2745
- getRemoteCollectionStates(collectionId) {
2746
- return this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
2747
- }
2748
- refreshCollection(collectionId) {
2749
- this._collectionSynchronizer.refreshCollection(collectionId);
2750
- }
2751
- async getCollectionSyncState(collectionId) {
2752
- const result = {
2753
- peers: []
2754
- };
2755
- const localState = this.getLocalCollectionState(collectionId);
2756
- const remoteState = this.getRemoteCollectionStates(collectionId);
2757
- if (!localState) {
2758
- return result;
2759
- }
2760
- for (const [peerId, state] of remoteState) {
2761
- const diff = diffCollectionState(localState, state);
2762
- result.peers.push({
2763
- peerId,
2764
- differentDocuments: diff.different.length
2765
- });
2766
- }
2767
- return result;
2768
- }
2769
- /**
2770
- * Update the local collection state based on the locally stored document heads.
2771
- */
2772
- async updateLocalCollectionState(collectionId, documentIds) {
2773
- const heads = await this.getHeads(documentIds);
2774
- const documents = Object.fromEntries(heads.map((heads2, index) => [
2775
- documentIds[index],
2776
- heads2 ?? []
2777
- ]));
2778
- this._collectionSynchronizer.setLocalCollectionState(collectionId, {
2779
- documents
2780
- });
2781
- }
2782
- _onCollectionStateQueried(collectionId, peerId) {
2783
- this._collectionSynchronizer.onCollectionStateQueried(collectionId, peerId);
2784
- }
2785
- _onCollectionStateReceived(collectionId, peerId, state) {
2786
- this._collectionSynchronizer.onRemoteStateReceived(collectionId, peerId, decodeCollectionState(state));
2787
- }
2788
- _queryCollectionState(collectionId, peerId) {
2789
- this._echoNetworkAdapter.queryCollectionState(collectionId, peerId);
2790
- }
2791
- _sendCollectionState(collectionId, peerId, state) {
2792
- this._echoNetworkAdapter.sendCollectionState(collectionId, peerId, encodeCollectionState(state));
2793
- }
2794
- _onPeerConnected(peerId) {
2795
- this._collectionSynchronizer.onConnectionOpen(peerId);
2796
- }
2797
- _onPeerDisconnected(peerId) {
2798
- this._collectionSynchronizer.onConnectionClosed(peerId);
2799
- }
2800
- _onRemoteCollectionStateUpdated(collectionId, peerId) {
2801
- const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
2802
- const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
2803
- if (!localState || !remoteState) {
2804
- return;
2805
- }
2806
- const { different } = diffCollectionState(localState, remoteState);
2807
- if (different.length === 0) {
2808
- return;
2809
- }
2810
- log10.info("replication documents after collection sync", {
2811
- count: different.length
2812
- }, {
2813
- F: __dxlog_file12,
2814
- L: 475,
2815
- S: this,
2816
- C: (f, a) => f(...a)
2817
- });
2818
- for (const documentId of different) {
2819
- this._repo.find(documentId);
2820
- }
2821
- }
2822
- _onHeadsChanged(documentId, heads) {
2823
- for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
2824
- const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
2825
- if (state?.documents[documentId]) {
2826
- const newState = structuredClone(state);
2827
- newState.documents[documentId] = heads;
2828
- this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
2829
- }
2830
- }
2831
- }
2832
- };
2833
- _ts_decorate7([
2834
- trace4.info()
2835
- ], AutomergeHost.prototype, "_peerId", void 0);
2836
- _ts_decorate7([
2837
- trace4.info({
2838
- depth: null
2839
- })
2840
- ], AutomergeHost.prototype, "_automergePeers", null);
2841
- _ts_decorate7([
2842
- trace4.span({
2843
- showInBrowserTimeline: true
2844
- })
2845
- ], AutomergeHost.prototype, "flush", null);
2846
- AutomergeHost = _ts_decorate7([
2847
- trace4.resource()
2848
- ], AutomergeHost);
2849
- var getSpaceKeyFromDoc = (doc) => {
2850
- const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
2851
- if (rawSpaceKey == null) {
2852
- return null;
2853
- }
2854
- return String(rawSpaceKey);
2855
- };
2856
- var waitForHeads = async (handle, heads) => {
2857
- const unavailableHeads = new Set(heads);
2858
- await handle.whenReady();
2859
- await Event5.wrap(handle, "change").waitForCondition(() => {
2860
- for (const changeHash of unavailableHeads.values()) {
2861
- if (changeIsPresentInDoc(handle.docSync(), changeHash)) {
2862
- unavailableHeads.delete(changeHash);
2863
- }
2864
- }
2865
- return unavailableHeads.size === 0;
2866
- });
2867
- };
2868
- var changeIsPresentInDoc = (doc, changeHash) => {
2869
- return !!getBackend(doc).getChangeByHash(changeHash);
2870
- };
2871
- var decodeCollectionState = (state) => {
2872
- invariant8(typeof state === "object" && state !== null, "Invalid state", {
2873
- F: __dxlog_file12,
2874
- L: 528,
2875
- S: void 0,
2876
- A: [
2877
- "typeof state === 'object' && state !== null",
2878
- "'Invalid state'"
2879
- ]
2880
- });
2881
- return state;
2882
- };
2883
- var encodeCollectionState = (state) => {
2884
- return state;
2885
- };
2886
-
2887
- // packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts
2888
- import { invariant as invariant11 } from "@dxos/invariant";
2889
- import { PublicKey as PublicKey7 } from "@dxos/keys";
2890
- import { log as log12 } from "@dxos/log";
2891
- import { ComplexSet, defaultMap as defaultMap2 } from "@dxos/util";
2892
-
2893
- // packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts
2894
- import * as A2 from "@dxos/automerge/automerge";
2895
- import { cbor } from "@dxos/automerge/automerge-repo";
2896
- import { Resource as Resource6 } from "@dxos/context";
2897
- import { invariant as invariant9 } from "@dxos/invariant";
2898
- import { log as log11 } from "@dxos/log";
2899
- import { AutomergeReplicator } from "@dxos/teleport-extension-automerge-replicator";
2900
- var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
2901
- var DEFAULT_FACTORY = (params) => new AutomergeReplicator(...params);
2902
- var MeshReplicatorConnection = class extends Resource6 {
2903
- constructor(_params) {
2904
- super();
2905
- this._params = _params;
2906
- this.remoteDeviceKey = null;
2907
- this._remotePeerId = null;
2908
- this._isEnabled = false;
2909
- let readableStreamController;
2910
- this.readable = new ReadableStream({
2911
- start: (controller) => {
2912
- readableStreamController = controller;
2913
- this._ctx.onDispose(() => controller.close());
2914
- }
2915
- });
2916
- this.writable = new WritableStream({
2917
- write: async (message, controller) => {
2918
- invariant9(this._isEnabled, "Writing to a disabled connection", {
2919
- F: __dxlog_file13,
2920
- L: 49,
2921
- S: this,
2922
- A: [
2923
- "this._isEnabled",
2924
- "'Writing to a disabled connection'"
2925
- ]
2926
- });
2927
- try {
2928
- logSendSync(message);
2929
- await this.replicatorExtension.sendSyncMessage({
2930
- payload: cbor.encode(message)
2931
- });
2932
- } catch (err) {
2933
- controller.error(err);
2934
- this._disconnectIfEnabled();
2935
- }
2936
- }
2937
- });
2938
- const createAutomergeReplicator = this._params.replicatorFactory ?? DEFAULT_FACTORY;
2939
- this.replicatorExtension = createAutomergeReplicator([
2940
- {
2941
- peerId: this._params.ownPeerId
2942
- },
2943
- {
2944
- onStartReplication: async (info, remotePeerId) => {
2945
- this.remoteDeviceKey = remotePeerId;
2946
- this._remotePeerId = info.id;
2947
- log11("onStartReplication", {
2948
- id: info.id,
2949
- thisPeerId: this.peerId,
2950
- remotePeerId: remotePeerId.toHex()
2951
- }, {
2952
- F: __dxlog_file13,
2953
- L: 84,
2954
- S: this,
2955
- C: (f, a) => f(...a)
2956
- });
2957
- this._params.onRemoteConnected();
2958
- },
2959
- onSyncMessage: async ({ payload }) => {
2960
- if (!this._isEnabled) {
2961
- return;
2962
- }
2963
- const message = cbor.decode(payload);
2964
- readableStreamController.enqueue(message);
2965
- },
2966
- onClose: async () => {
2967
- this._disconnectIfEnabled();
2968
- }
2969
- }
2970
- ]);
2971
- }
2972
- _disconnectIfEnabled() {
2973
- if (this._isEnabled) {
2974
- this._params.onRemoteDisconnected();
2975
- }
2976
- }
2977
- get peerId() {
2978
- invariant9(this._remotePeerId != null, "Remote peer has not connected yet.", {
2979
- F: __dxlog_file13,
2980
- L: 110,
2981
- S: this,
2982
- A: [
2983
- "this._remotePeerId != null",
2984
- "'Remote peer has not connected yet.'"
2985
- ]
2986
- });
2987
- return this._remotePeerId;
2988
- }
2989
- async shouldAdvertise(params) {
2990
- return this._params.shouldAdvertise(params);
2991
- }
2992
- shouldSyncCollection(params) {
2993
- return this._params.shouldSyncCollection(params);
2994
- }
2995
- /**
2996
- * Start exchanging messages with the remote peer.
2997
- * Call after the remote peer has connected.
2998
- */
2999
- enable() {
3000
- invariant9(this._remotePeerId != null, "Remote peer has not connected yet.", {
3001
- F: __dxlog_file13,
3002
- L: 127,
3003
- S: this,
3004
- A: [
3005
- "this._remotePeerId != null",
3006
- "'Remote peer has not connected yet.'"
3007
- ]
3008
- });
3009
- this._isEnabled = true;
3010
- }
3011
- /**
3012
- * Stop exchanging messages with the remote peer.
3013
- */
3014
- disable() {
3015
- this._isEnabled = false;
3016
- }
3017
- };
3018
- var logSendSync = (message) => {
3019
- log11("sendSyncMessage", () => {
3020
- const decodedSyncMessage = message.type === "sync" && message.data ? A2.decodeSyncMessage(message.data) : void 0;
3021
- return {
3022
- sync: decodedSyncMessage && {
3023
- headsLength: decodedSyncMessage.heads.length,
3024
- requesting: decodedSyncMessage.need.length > 0,
3025
- sendingChanges: decodedSyncMessage.changes.length > 0
3026
- },
3027
- type: message.type,
3028
- from: message.senderId,
3029
- to: message.targetId
3030
- };
3031
- }, {
3032
- F: __dxlog_file13,
3033
- L: 140,
3034
- S: void 0,
3035
- C: (f, a) => f(...a)
3036
- });
3037
- };
3038
-
3039
- // packages/core/echo/echo-pipeline/src/automerge/space-collection.ts
3040
- import { invariant as invariant10 } from "@dxos/invariant";
3041
- import { SpaceId as SpaceId2 } from "@dxos/keys";
3042
- var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
3043
- var deriveCollectionIdFromSpaceId = (spaceId) => `space:${spaceId}`;
3044
- var getSpaceIdFromCollectionId = (collectionId) => {
3045
- const spaceId = collectionId.replace(/^space:/, "");
3046
- invariant10(SpaceId2.isValid(spaceId), void 0, {
3047
- F: __dxlog_file14,
3048
- L: 12,
3049
- S: void 0,
3050
- A: [
3051
- "SpaceId.isValid(spaceId)",
3052
- ""
3053
- ]
3054
- });
3055
- return spaceId;
3056
- };
3057
-
3058
- // packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts
3059
- var __dxlog_file15 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts";
3060
- var MeshEchoReplicator = class {
3061
- constructor() {
3062
- this._connections = /* @__PURE__ */ new Set();
3063
- /**
3064
- * Using automerge peerId as a key.
3065
- */
3066
- this._connectionsPerPeer = /* @__PURE__ */ new Map();
3067
- /**
3068
- * spaceId -> deviceKey[]
3069
- */
3070
- this._authorizedDevices = /* @__PURE__ */ new Map();
3071
- this._context = null;
3072
- }
3073
- async connect(context) {
3074
- this._context = context;
3075
- }
3076
- async disconnect() {
3077
- for (const connection of this._connectionsPerPeer.values()) {
3078
- this._context?.onConnectionClosed(connection);
3079
- }
3080
- for (const connection of this._connections) {
3081
- await connection.close();
3082
- }
3083
- this._connections.clear();
3084
- this._connectionsPerPeer.clear();
3085
- this._context = null;
3086
- }
3087
- createExtension(extensionFactory) {
3088
- invariant11(this._context, void 0, {
3089
- F: __dxlog_file15,
3090
- L: 56,
3091
- S: this,
3092
- A: [
3093
- "this._context",
3094
- ""
3095
- ]
3096
- });
3097
- const connection = new MeshReplicatorConnection({
3098
- ownPeerId: this._context.peerId,
3099
- replicatorFactory: extensionFactory,
3100
- onRemoteConnected: async () => {
3101
- log12("onRemoteConnected", {
3102
- peerId: connection.peerId
3103
- }, {
3104
- F: __dxlog_file15,
3105
- L: 62,
3106
- S: this,
3107
- C: (f, a) => f(...a)
3108
- });
3109
- invariant11(this._context, void 0, {
3110
- F: __dxlog_file15,
3111
- L: 63,
3112
- S: this,
3113
- A: [
3114
- "this._context",
3115
- ""
3116
- ]
3117
- });
3118
- if (this._connectionsPerPeer.has(connection.peerId)) {
3119
- this._context.onConnectionAuthScopeChanged(connection);
3120
- } else {
3121
- this._connectionsPerPeer.set(connection.peerId, connection);
3122
- this._context.onConnectionOpen(connection);
3123
- connection.enable();
3124
- }
3125
- },
3126
- onRemoteDisconnected: async () => {
3127
- log12("onRemoteDisconnected", {
3128
- peerId: connection.peerId
3129
- }, {
3130
- F: __dxlog_file15,
3131
- L: 74,
3132
- S: this,
3133
- C: (f, a) => f(...a)
3134
- });
3135
- this._context?.onConnectionClosed(connection);
3136
- this._connectionsPerPeer.delete(connection.peerId);
3137
- connection.disable();
3138
- this._connections.delete(connection);
3139
- },
3140
- shouldAdvertise: async (params) => {
3141
- log12("shouldAdvertise", {
3142
- peerId: connection.peerId,
3143
- documentId: params.documentId
3144
- }, {
3145
- F: __dxlog_file15,
3146
- L: 81,
3147
- S: this,
3148
- C: (f, a) => f(...a)
3149
- });
3150
- invariant11(this._context, void 0, {
3151
- F: __dxlog_file15,
3152
- L: 82,
3153
- S: this,
3154
- A: [
3155
- "this._context",
3156
- ""
3157
- ]
3158
- });
3159
- try {
3160
- const spaceKey = await this._context.getContainingSpaceForDocument(params.documentId);
3161
- if (!spaceKey) {
3162
- const remoteDocumentExists = await this._context.isDocumentInRemoteCollection({
3163
- documentId: params.documentId,
3164
- peerId: connection.peerId
3165
- });
3166
- log12("document not found locally for share policy check, accepting the remote document", {
3167
- peerId: connection.peerId,
3168
- documentId: params.documentId,
3169
- remoteDocumentExists
3170
- }, {
3171
- F: __dxlog_file15,
3172
- L: 90,
3173
- S: this,
3174
- C: (f, a) => f(...a)
3175
- });
3176
- return remoteDocumentExists;
3177
- }
3178
- const spaceId = await createIdFromSpaceKey(spaceKey);
3179
- const authorizedDevices = this._authorizedDevices.get(spaceId);
3180
- if (!connection.remoteDeviceKey) {
3181
- log12("device key not found for share policy check", {
3182
- peerId: connection.peerId,
3183
- documentId: params.documentId
3184
- }, {
3185
- F: __dxlog_file15,
3186
- L: 106,
3187
- S: this,
3188
- C: (f, a) => f(...a)
3189
- });
3190
- return false;
3191
- }
3192
- const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
3193
- log12("share policy check", {
3194
- localPeer: this._context.peerId,
3195
- remotePeer: connection.peerId,
3196
- documentId: params.documentId,
3197
- deviceKey: connection.remoteDeviceKey,
3198
- spaceKey,
3199
- isAuthorized
3200
- }, {
3201
- F: __dxlog_file15,
3202
- L: 114,
3203
- S: this,
3204
- C: (f, a) => f(...a)
3205
- });
3206
- return isAuthorized;
3207
- } catch (err) {
3208
- log12.catch(err, void 0, {
3209
- F: __dxlog_file15,
3210
- L: 124,
3211
- S: this,
3212
- C: (f, a) => f(...a)
3213
- });
3214
- return false;
3215
- }
3216
- },
3217
- shouldSyncCollection: ({ collectionId }) => {
3218
- const spaceId = getSpaceIdFromCollectionId(collectionId);
3219
- const authorizedDevices = this._authorizedDevices.get(spaceId);
3220
- if (!connection.remoteDeviceKey) {
3221
- log12("device key not found for collection sync check", {
3222
- peerId: connection.peerId,
3223
- collectionId
3224
- }, {
3225
- F: __dxlog_file15,
3226
- L: 134,
3227
- S: this,
3228
- C: (f, a) => f(...a)
3229
- });
3230
- return false;
3231
- }
3232
- const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
3233
- return isAuthorized;
3234
- }
3235
- });
3236
- this._connections.add(connection);
3237
- return connection.replicatorExtension;
3238
- }
3239
- async authorizeDevice(spaceKey, deviceKey) {
3240
- log12("authorizeDevice", {
3241
- spaceKey,
3242
- deviceKey
3243
- }, {
3244
- F: __dxlog_file15,
3245
- L: 151,
3246
- S: this,
3247
- C: (f, a) => f(...a)
3248
- });
3249
- const spaceId = await createIdFromSpaceKey(spaceKey);
3250
- defaultMap2(this._authorizedDevices, spaceId, () => new ComplexSet(PublicKey7.hash)).add(deviceKey);
3251
- for (const connection of this._connections) {
3252
- if (connection.remoteDeviceKey && connection.remoteDeviceKey.equals(deviceKey)) {
3253
- if (this._connectionsPerPeer.has(connection.peerId)) {
3254
- this._context?.onConnectionAuthScopeChanged(connection);
3255
- }
3256
- }
3257
- }
3258
- }
3259
- };
3260
-
3261
- // packages/core/echo/echo-pipeline/src/automerge/echo-data-monitor.ts
3262
- import { trace as trace5 } from "@dxos/tracing";
3263
- import { CircularBuffer, mapValues, SlidingWindowSummary } from "@dxos/util";
3264
- function _ts_decorate8(decorators, target, key, desc) {
3265
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3266
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
3267
- r = Reflect.decorate(decorators, target, key, desc);
3268
- else
3269
- for (var i = decorators.length - 1; i >= 0; i--)
3270
- if (d = decorators[i])
3271
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
3272
- return c > 3 && r && Object.defineProperty(target, key, r), r;
3273
- }
3274
- var PER_SECOND_RATE_AVG_WINDOW_SIZE = 5;
3275
- var DEFAULT_AVG_WINDOW_SIZE = 25;
3276
- var EchoDataMonitor = class {
3277
- constructor(_params = {
3278
- timeSeriesLength: 30
3279
- }) {
3280
- this._params = _params;
3281
- this._lastTick = 0;
3282
- this._activeCounters = createLocalCounters();
3283
- this._localTimeSeries = createLocalTimeSeries();
3284
- this._storageAverages = createStorageAverages();
3285
- this._replicationAverages = createNetworkAverages();
3286
- this._sizeByMessageType = {};
3287
- this._lastReceivedMessages = new CircularBuffer(100);
3288
- this._lastSentMessages = new CircularBuffer(100);
3289
- this._connectionsCount = 0;
3290
- }
3291
- tick(timeMs) {
3292
- this._advanceTimeWindow(timeMs - this._lastTick);
3293
- this._lastTick = timeMs;
3294
- }
3295
- computeStats() {
3296
- return {
3297
- meta: {
3298
- rateAverageOverSeconds: PER_SECOND_RATE_AVG_WINDOW_SIZE
3299
- },
3300
- storage: {
3301
- reads: {
3302
- payloadSize: this._storageAverages.loadedChunkSize.average(),
3303
- opDuration: this._storageAverages.loadDuration.average(),
3304
- countPerSecond: this._storageAverages.loadsPerSecond.average()
3305
- },
3306
- writes: {
3307
- payloadSize: this._storageAverages.storedChunkSize.average(),
3308
- opDuration: this._storageAverages.storeDuration.average(),
3309
- countPerSecond: this._storageAverages.storesPerSecond.average()
3310
- }
3311
- },
3312
- replicator: {
3313
- connections: this._connectionsCount,
3314
- receivedMessages: {
3315
- payloadSize: this._replicationAverages.receivedMessageSize.average(),
3316
- countPerSecond: this._replicationAverages.receivedPerSecond.average()
3317
- },
3318
- sentMessages: {
3319
- payloadSize: this._replicationAverages.sentMessageSize.average(),
3320
- opDuration: this._replicationAverages.sendDuration.average(),
3321
- countPerSecond: this._replicationAverages.sentPerSecond.average(),
3322
- failedPerSecond: this._replicationAverages.sendsFailedPerSecond.average()
3323
- },
3324
- countByMessageType: this._computeMessageHistogram("type"),
3325
- avgSizeByMessageType: mapValues(this._sizeByMessageType, (summary) => summary.average())
3326
- }
3327
- };
3328
- }
3329
- get connectionsCount() {
3330
- return this._connectionsCount;
3331
- }
3332
- /**
3333
- * @internal
3334
- */
3335
- get lastPerSecondStats() {
3336
- return this._lastCompleteCounters;
3337
- }
3338
- /**
3339
- * @internal
3340
- */
3341
- get timeSeries() {
3342
- return {
3343
- ...this._localTimeSeries.storage,
3344
- ...this._localTimeSeries.replication
3345
- };
3346
- }
3347
- /**
3348
- * @internal
3349
- */
3350
- get messagesByPeerId() {
3351
- return this._computeMessageHistogram("peerId");
3352
- }
3353
- _advanceTimeWindow(millisPassed) {
3354
- const oldMetrics = Object.freeze(this._activeCounters);
3355
- this._activeCounters = createLocalCounters();
3356
- this._lastCompleteCounters = oldMetrics;
3357
- for (const peerId of Object.keys(oldMetrics.byPeerId)) {
3358
- this._activeCounters.byPeerId[peerId] = createMessageCounter();
3359
- }
3360
- this._addToTimeSeries(oldMetrics.replication, this._localTimeSeries.replication);
3361
- this._addToTimeSeries(oldMetrics.storage, this._localTimeSeries.storage);
3362
- if (Math.abs(millisPassed - 1e3) < 100) {
3363
- this._reportPerSecondRate(oldMetrics);
3364
- }
3365
- }
3366
- _addToTimeSeries(values, timeSeries) {
3367
- for (const [key, value] of Object.entries(values)) {
3368
- const values2 = timeSeries[key];
3369
- values2.push(value);
3370
- if (values2.length > this._params.timeSeriesLength) {
3371
- values2.shift();
3372
- }
3373
- }
3374
- }
3375
- _reportPerSecondRate(metrics) {
3376
- const toReport = [
3377
- [
3378
- "storage.load",
3379
- metrics.storage.loadedChunks,
3380
- this._storageAverages.loadsPerSecond
3381
- ],
3382
- [
3383
- "storage.store",
3384
- metrics.storage.storedChunks,
3385
- this._storageAverages.storesPerSecond
3386
- ],
3387
- [
3388
- "network.receive",
3389
- metrics.replication.received,
3390
- this._replicationAverages.receivedPerSecond
3391
- ],
3392
- [
3393
- "network.send",
3394
- metrics.replication.sent,
3395
- this._replicationAverages.sentPerSecond
3396
- ]
3397
- ];
3398
- for (const [metricName, metric, summary] of toReport) {
3399
- summary.record(metric);
3400
- if (metric > 0) {
3401
- trace5.metrics.distribution(`dxos.echo.${metricName}-rate`, metric);
3402
- trace5.metrics.increment(`dxos.echo.${metricName}`, 1, {
3403
- tags: {
3404
- status: "busy"
3405
- }
3406
- });
3407
- } else {
3408
- trace5.metrics.increment(`dxos.echo.${metricName}`, 1, {
3409
- tags: {
3410
- status: "idle"
3411
- }
3412
- });
3413
- }
3414
- }
3415
- this._replicationAverages.sendsFailedPerSecond.record(metrics.replication.failed);
3416
- }
3417
- recordPeerConnected(peerId) {
3418
- this._activeCounters.byPeerId[peerId] = createMessageCounter();
3419
- this._connectionsCount++;
3420
- }
3421
- recordPeerDisconnected(peerId) {
3422
- this._connectionsCount--;
3423
- delete this._activeCounters.byPeerId[peerId];
3424
- }
3425
- recordBytesStored(count) {
3426
- this._activeCounters.storage.storedChunks++;
3427
- this._activeCounters.storage.storedBytes += count;
3428
- this._storageAverages.storedChunkSize.record(count);
3429
- trace5.metrics.distribution("dxos.echo.storage.bytes-stored", count, {
3430
- unit: "bytes"
3431
- });
3432
- }
3433
- recordLoadDuration(durationMs) {
3434
- this._storageAverages.loadDuration.record(durationMs);
3435
- }
3436
- recordStoreDuration(durationMs) {
3437
- this._storageAverages.storeDuration.record(durationMs);
3438
- }
3439
- recordBytesLoaded(count) {
3440
- this._activeCounters.storage.loadedChunks++;
3441
- this._activeCounters.storage.loadedBytes += count;
3442
- this._storageAverages.loadedChunkSize.record(count);
3443
- trace5.metrics.distribution("dxos.echo.storage.bytes-loaded", count, {
3444
- unit: "bytes"
3445
- });
3446
- }
3447
- recordMessageSent(message, duration) {
3448
- let metricsGroupName;
3449
- const bytes = getByteCount(message);
3450
- const tags = {
3451
- type: message.type
3452
- };
3453
- if (isAutomergeProtocolMessage(message)) {
3454
- this._activeCounters.replication.sent++;
3455
- this._replicationAverages.sendDuration.record(duration);
3456
- this._replicationAverages.sentMessageSize.record(bytes);
3457
- metricsGroupName = "replication";
3458
- } else {
3459
- metricsGroupName = "collection-sync";
3460
- }
3461
- trace5.metrics.distribution(`dxos.echo.${metricsGroupName}.bytes-sent`, bytes, {
3462
- unit: "bytes",
3463
- tags
3464
- });
3465
- trace5.metrics.distribution(`dxos.echo.${metricsGroupName}.send-duration`, duration, {
3466
- unit: "millisecond",
3467
- tags
3468
- });
3469
- trace5.metrics.increment(`dxos.echo.${metricsGroupName}.send-status`, 1, {
3470
- tags: {
3471
- ...tags,
3472
- success: true
3473
- }
3474
- });
3475
- const { messageSize, messageCounts } = this._getStatsForType(message);
3476
- messageSize.record(bytes);
3477
- messageCounts.sent++;
3478
- this._lastSentMessages.push({
3479
- type: message.type,
3480
- peerId: message.targetId
3481
- });
3482
- }
3483
- recordMessageReceived(message) {
3484
- const bytes = getByteCount(message);
3485
- const tags = {
3486
- type: message.type
3487
- };
3488
- if (isAutomergeProtocolMessage(message)) {
3489
- this._activeCounters.replication.received++;
3490
- this._replicationAverages.receivedMessageSize.record(bytes);
3491
- trace5.metrics.distribution("dxos.echo.replication.bytes-received", bytes, {
3492
- unit: "bytes",
3493
- tags
3494
- });
3495
- } else {
3496
- trace5.metrics.distribution("dxos.echo.collection-sync.bytes-received", bytes, {
3497
- unit: "bytes",
3498
- tags
3499
- });
3500
- }
3501
- const { messageSize, messageCounts } = this._getStatsForType(message);
3502
- messageSize.record(bytes);
3503
- messageCounts.received++;
3504
- this._lastReceivedMessages.push({
3505
- type: message.type,
3506
- peerId: message.senderId
3507
- });
3508
- }
3509
- recordMessageSendingFailed(message) {
3510
- const tags = {
3511
- type: message.type,
3512
- success: false
3513
- };
3514
- if (isAutomergeProtocolMessage(message)) {
3515
- this._activeCounters.replication.failed++;
3516
- trace5.metrics.increment("dxos.echo.replication.send-status", 1, {
3517
- unit: "bytes",
3518
- tags
3519
- });
3520
- } else {
3521
- trace5.metrics.increment("dxos.echo.collection-sync.send-status", 1, {
3522
- unit: "bytes",
3523
- tags
3524
- });
3525
- }
3526
- const { messageCounts } = this._getStatsForType(message);
3527
- messageCounts.failed++;
3528
- }
3529
- _getStatsForType(message) {
3530
- const messageSize = this._sizeByMessageType[message.type] ??= createSlidingWindow();
3531
- const messageCounts = this._activeCounters.byType[message.type] ??= createMessageCounter();
3532
- return {
3533
- messageCounts,
3534
- messageSize
3535
- };
3536
- }
3537
- _computeMessageHistogram(groupKey) {
3538
- const result = {};
3539
- for (const receivedMessage of this._lastReceivedMessages) {
3540
- const counters = result[receivedMessage[groupKey]] ??= {
3541
- received: 0,
3542
- sent: 0
3543
- };
3544
- counters.received++;
3545
- }
3546
- for (const receivedMessage of this._lastSentMessages) {
3547
- const counters = result[receivedMessage[groupKey]] ??= {
3548
- received: 0,
3549
- sent: 0
3550
- };
3551
- counters.sent++;
3552
- }
3553
- return result;
3554
- }
3555
- };
3556
- EchoDataMonitor = _ts_decorate8([
3557
- trace5.resource()
3558
- ], EchoDataMonitor);
3559
- var isAutomergeProtocolMessage = (message) => {
3560
- return !(isCollectionQueryMessage(message) || isCollectionStateMessage(message));
3561
- };
3562
- var createSlidingWindow = (overrides) => new SlidingWindowSummary({
3563
- dataPoints: DEFAULT_AVG_WINDOW_SIZE,
3564
- precision: 2,
3565
- ...overrides
3566
- });
3567
- var createLocalCounters = () => ({
3568
- storage: {
3569
- loadedBytes: 0,
3570
- storedBytes: 0,
3571
- storedChunks: 0,
3572
- loadedChunks: 0
3573
- },
3574
- replication: createMessageCounter(),
3575
- byPeerId: {},
3576
- byType: {}
3577
- });
3578
- var createLocalTimeSeries = () => ({
3579
- storage: {
3580
- loadedBytes: [],
3581
- storedBytes: [],
3582
- storedChunks: [],
3583
- loadedChunks: []
3584
- },
3585
- replication: {
3586
- sent: [],
3587
- failed: [],
3588
- received: []
3589
- }
3590
- });
3591
- var createMessageCounter = () => ({
3592
- sent: 0,
3593
- received: 0,
3594
- failed: 0
3595
- });
3596
- var createNetworkAverages = () => ({
3597
- receivedMessageSize: createSlidingWindow(),
3598
- sentMessageSize: createSlidingWindow(),
3599
- sendDuration: createSlidingWindow(),
3600
- receivedPerSecond: createSlidingWindow({
3601
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
3602
- }),
3603
- sentPerSecond: createSlidingWindow({
3604
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
3605
- }),
3606
- sendsFailedPerSecond: createSlidingWindow({
3607
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
3608
- })
3609
- });
3610
- var createStorageAverages = () => ({
3611
- storedChunkSize: createSlidingWindow(),
3612
- loadedChunkSize: createSlidingWindow(),
3613
- loadDuration: createSlidingWindow(),
3614
- storeDuration: createSlidingWindow(),
3615
- loadsPerSecond: createSlidingWindow({
3616
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
3617
- }),
3618
- storesPerSecond: createSlidingWindow({
3619
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
3620
- })
3621
- });
3622
- var getByteCount = (message) => {
3623
- return message.type.length + message.senderId.length + message.targetId.length + (message.data?.byteLength ?? 0) + (message.documentId?.length ?? 0);
3624
- };
3625
-
3626
- // packages/core/echo/echo-pipeline/src/db-host/data-service.ts
3627
- var __dxlog_file16 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
3628
- var DataServiceImpl = class {
3629
- constructor(params) {
3630
- /**
3631
- * Map of subscriptions.
3632
- * subscriptionId -> DocumentsSynchronizer
3633
- */
3634
- this._subscriptions = /* @__PURE__ */ new Map();
3635
- this._automergeHost = params.automergeHost;
3636
- this._updateIndexes = params.updateIndexes;
3637
- }
3638
- subscribe(request) {
3639
- return new Stream(({ next, ready }) => {
3640
- const synchronizer = new DocumentsSynchronizer({
3641
- repo: this._automergeHost.repo,
3642
- sendUpdates: (updates) => next(updates)
3643
- });
3644
- synchronizer.open().then(() => {
3645
- this._subscriptions.set(request.subscriptionId, synchronizer);
3646
- ready();
3647
- }).catch((err) => log13.catch(err, void 0, {
3648
- F: __dxlog_file16,
3649
- L: 64,
3650
- S: this,
3651
- C: (f, a) => f(...a)
3652
- }));
3653
- return () => synchronizer.close();
3654
- });
3655
- }
3656
- async updateSubscription(request) {
3657
- const synchronizer = this._subscriptions.get(request.subscriptionId);
3658
- invariant12(synchronizer, "Subscription not found", {
3659
- F: __dxlog_file16,
3660
- L: 71,
3661
- S: this,
3662
- A: [
3663
- "synchronizer",
3664
- "'Subscription not found'"
3665
- ]
3666
- });
3667
- if (request.addIds?.length) {
3668
- await synchronizer.addDocuments(request.addIds);
3669
- }
3670
- if (request.removeIds?.length) {
3671
- await synchronizer.removeDocuments(request.removeIds);
3672
- }
3673
- }
3674
- async update(request) {
3675
- if (!request.updates) {
3676
- return;
3677
- }
3678
- const synchronizer = this._subscriptions.get(request.subscriptionId);
3679
- invariant12(synchronizer, "Subscription not found", {
3680
- F: __dxlog_file16,
3681
- L: 86,
3682
- S: this,
3683
- A: [
3684
- "synchronizer",
3685
- "'Subscription not found'"
3686
- ]
3687
- });
3688
- synchronizer.update(request.updates);
3689
- }
3690
- async flush(request) {
3691
- await this._automergeHost.flush(request);
3692
- }
3693
- async getDocumentHeads(request) {
3694
- const documentIds = request.documentIds;
3695
- if (!documentIds) {
3696
- return {
3697
- heads: {
3698
- entries: []
3699
- }
3700
- };
3701
- }
3702
- const heads = await this._automergeHost.getHeads(documentIds);
3703
- return {
3704
- heads: {
3705
- entries: heads.map((heads2, idx) => ({
3706
- documentId: documentIds[idx],
3707
- heads: heads2
3708
- }))
3709
- }
3710
- };
3711
- }
3712
- async waitUntilHeadsReplicated(request, options) {
3713
- await this._automergeHost.waitUntilHeadsReplicated(request.heads);
3714
- }
3715
- async reIndexHeads(request, options) {
3716
- await this._automergeHost.reIndexHeads(request.documentIds ?? []);
3717
- }
3718
- async updateIndexes() {
3719
- await this._updateIndexes();
3720
- }
3721
- async getSpaceSyncState(request, options) {
3722
- invariant12(SpaceId3.isValid(request.spaceId), void 0, {
3723
- F: __dxlog_file16,
3724
- L: 127,
3725
- S: this,
3726
- A: [
3727
- "SpaceId.isValid(request.spaceId)",
3728
- ""
3729
- ]
3730
- });
3731
- const collectionId = deriveCollectionIdFromSpaceId(request.spaceId);
3732
- const state = await this._automergeHost.getCollectionSyncState(collectionId);
3733
- return {
3734
- peers: state.peers.map((peer) => ({
3735
- peerId: peer.peerId,
3736
- documentsToReconcile: peer.differentDocuments
3737
- }))
3738
- };
3739
- }
3740
- };
3741
-
3742
- // packages/core/echo/echo-pipeline/src/space/space-manager.ts
3743
- function _ts_decorate9(decorators, target, key, desc) {
3744
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3745
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
3746
- r = Reflect.decorate(decorators, target, key, desc);
3747
- else
3748
- for (var i = decorators.length - 1; i >= 0; i--)
3749
- if (d = decorators[i])
3750
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
3751
- return c > 3 && r && Object.defineProperty(target, key, r), r;
3752
- }
3753
- var __dxlog_file17 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
3754
- var SpaceManager = class {
3755
- constructor({ feedStore, networkManager, metadataStore, snapshotStore, blobStore, disableP2pReplication }) {
3756
- this._spaces = new ComplexMap4(PublicKey8.hash);
3757
- this._instanceId = PublicKey8.random().toHex();
3758
- this._feedStore = feedStore;
3759
- this._networkManager = networkManager;
3760
- this._metadataStore = metadataStore;
3761
- this._snapshotStore = snapshotStore;
3762
- this._blobStore = blobStore;
3763
- this._disableP2pReplication = disableP2pReplication ?? false;
3764
- }
3765
- // TODO(burdon): Remove.
3766
- get spaces() {
3767
- return this._spaces;
3768
- }
3769
- async open() {
3770
- }
3771
- async close() {
3772
- await Promise.all([
3773
- ...this._spaces.values()
3774
- ].map((space) => space.close()));
3775
- }
3776
- async constructSpace({ metadata, swarmIdentity, onAuthorizedConnection, onAuthFailure, onDelegatedInvitationStatusChange, onMemberRolesChanged, memberKey }) {
3777
- log14.trace("dxos.echo.space-manager.construct-space", trace6.begin({
3778
- id: this._instanceId
3779
- }), {
3780
- F: __dxlog_file17,
3781
- L: 114,
3782
- S: this,
3783
- C: (f, a) => f(...a)
3784
- });
3785
- log14("constructing space...", {
3786
- spaceKey: metadata.genesisFeedKey
3787
- }, {
3788
- F: __dxlog_file17,
3789
- L: 115,
3790
- S: this,
3791
- C: (f, a) => f(...a)
3792
- });
3793
- const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? failUndefined2());
3794
- const spaceKey = metadata.key;
3795
- const spaceId = await createIdFromSpaceKey(spaceKey);
3796
- const protocol = new SpaceProtocol({
3797
- topic: spaceKey,
3798
- swarmIdentity,
3799
- networkManager: this._networkManager,
3800
- onSessionAuth: onAuthorizedConnection,
3801
- onAuthFailure,
3802
- blobStore: this._blobStore,
3803
- disableP2pReplication: this._disableP2pReplication
3804
- });
3805
- const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
3806
- const space = new Space({
3807
- id: spaceId,
3808
- spaceKey,
3809
- protocol,
3810
- genesisFeed,
3811
- feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
3812
- metadataStore: this._metadataStore,
3813
- snapshotManager,
3814
- memberKey,
3815
- onDelegatedInvitationStatusChange,
3816
- onMemberRolesChanged
3817
- });
3818
- this._spaces.set(space.key, space);
3819
- log14.trace("dxos.echo.space-manager.construct-space", trace6.end({
3820
- id: this._instanceId
3821
- }), {
3822
- F: __dxlog_file17,
3823
- L: 147,
3824
- S: this,
3825
- C: (f, a) => f(...a)
3826
- });
3827
- return space;
3828
- }
3829
- async requestSpaceAdmissionCredential(params) {
3830
- const traceKey = "dxos.echo.space-manager.request-space-admission";
3831
- log14.trace(traceKey, trace6.begin({
3832
- id: this._instanceId
3833
- }), {
3834
- F: __dxlog_file17,
3835
- L: 153,
3836
- S: this,
3837
- C: (f, a) => f(...a)
3838
- });
3839
- log14("requesting space admission credential...", {
3840
- spaceKey: params.spaceKey
3841
- }, {
3842
- F: __dxlog_file17,
3843
- L: 154,
3844
- S: this,
3845
- C: (f, a) => f(...a)
3846
- });
3847
- const onCredentialResolved = new Trigger3();
3848
- const protocol = new SpaceProtocol({
3849
- topic: params.spaceKey,
3850
- swarmIdentity: params.swarmIdentity,
3851
- networkManager: this._networkManager,
3852
- onSessionAuth: (session) => {
3853
- session.addExtension("dxos.mesh.teleport.admission-discovery", new CredentialRetrieverExtension({
3854
- spaceKey: params.spaceKey,
3855
- memberKey: params.identityKey
3856
- }, onCredentialResolved));
3857
- },
3858
- onAuthFailure: (session) => session.close(),
3859
- blobStore: this._blobStore,
3860
- disableP2pReplication: this._disableP2pReplication
3861
- });
3862
- try {
3863
- await protocol.start();
3864
- const credential = await onCredentialResolved.wait({
3865
- timeout: params.timeout
3866
- });
3867
- log14.trace(traceKey, trace6.end({
3868
- id: this._instanceId
3869
- }), {
3870
- F: __dxlog_file17,
3871
- L: 178,
3872
- S: this,
3873
- C: (f, a) => f(...a)
3874
- });
3875
- return credential;
3876
- } catch (err) {
3877
- log14.trace(traceKey, trace6.error({
3878
- id: this._instanceId,
3879
- error: err
3880
- }), {
3881
- F: __dxlog_file17,
3882
- L: 181,
3883
- S: this,
3884
- C: (f, a) => f(...a)
3885
- });
3886
- throw err;
3887
- } finally {
3888
- await protocol.stop();
3889
- }
3890
- }
3891
- };
3892
- _ts_decorate9([
3893
- synchronized4
3894
- ], SpaceManager.prototype, "open", null);
3895
- _ts_decorate9([
3896
- synchronized4
3897
- ], SpaceManager.prototype, "close", null);
3898
- SpaceManager = _ts_decorate9([
3899
- trackLeaks3("open", "close")
3900
- ], SpaceManager);
3901
-
3902
- // packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
3903
- import CRC32 from "crc-32";
3904
- import { Event as Event6, scheduleTaskInterval as scheduleTaskInterval2, synchronized as synchronized5 } from "@dxos/async";
3905
- import { Context as Context6 } from "@dxos/context";
3906
- import { invariant as invariant13 } from "@dxos/invariant";
3907
- import { PublicKey as PublicKey9 } from "@dxos/keys";
3908
- import { log as log15 } from "@dxos/log";
3909
- import { DataCorruptionError, STORAGE_VERSION, schema as schema6 } from "@dxos/protocols";
3910
- import { Invitation, SpaceState } from "@dxos/protocols/proto/dxos/client/services";
3911
- import { ComplexMap as ComplexMap5, arrayToBuffer, forEachAsync, isNotNullOrUndefined } from "@dxos/util";
3912
- function _ts_decorate10(decorators, target, key, desc) {
3913
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3914
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
3915
- r = Reflect.decorate(decorators, target, key, desc);
3916
- else
3917
- for (var i = decorators.length - 1; i >= 0; i--)
3918
- if (d = decorators[i])
3919
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
3920
- return c > 3 && r && Object.defineProperty(target, key, r), r;
3921
- }
3922
- var __dxlog_file18 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
3923
- var EXPIRED_INVITATION_CLEANUP_INTERVAL = 60 * 60 * 1e3;
3924
- var emptyEchoMetadata = () => ({
3925
- version: STORAGE_VERSION,
3926
- spaces: [],
3927
- created: /* @__PURE__ */ new Date(),
3928
- updated: /* @__PURE__ */ new Date()
3929
- });
3930
- var emptyLargeSpaceMetadata = () => ({});
3931
- var EchoMetadata = schema6.getCodecForType("dxos.echo.metadata.EchoMetadata");
3932
- var LargeSpaceMetadata = schema6.getCodecForType("dxos.echo.metadata.LargeSpaceMetadata");
3933
- var MetadataStore = class {
3934
- constructor(directory) {
3935
- this._metadata = emptyEchoMetadata();
3936
- this._spaceLargeMetadata = new ComplexMap5(PublicKey9.hash);
3937
- this._metadataFile = void 0;
3938
- this.update = new Event6();
3939
- this._invitationCleanupCtx = new Context6(void 0, {
3940
- F: __dxlog_file18,
3941
- L: 53
3942
- });
3943
- this._directory = directory;
3944
- }
3945
- get metadata() {
3946
- return this._metadata;
3947
- }
3948
- get version() {
3949
- return this._metadata.version ?? 0;
3950
- }
3951
- /**
3952
- * Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
3953
- * addSpaceFeed functions.
3954
- */
3955
- get spaces() {
3956
- return this._metadata.spaces ?? [];
3957
- }
3958
- async _readFile(file, codec2) {
3959
- try {
3960
- const { size: fileLength } = await file.stat();
3961
- if (fileLength < 8) {
3962
- return;
3963
- }
3964
- const dataSize = fromBytesInt32(await file.read(0, 4));
3965
- const checksum = fromBytesInt32(await file.read(4, 4));
3966
- log15("loaded", {
3967
- size: dataSize,
3968
- checksum,
3969
- name: file.filename
3970
- }, {
3971
- F: __dxlog_file18,
3972
- L: 89,
3973
- S: this,
3974
- C: (f, a) => f(...a)
3975
- });
3976
- if (fileLength < dataSize + 8) {
3977
- throw new DataCorruptionError("Metadata size is smaller than expected.", {
3978
- fileLength,
3979
- dataSize
3980
- });
3981
- }
3982
- const data = await file.read(8, dataSize);
3983
- const calculatedChecksum = CRC32.buf(data);
3984
- if (calculatedChecksum !== checksum) {
3985
- throw new DataCorruptionError("Metadata checksum is invalid.");
3986
- }
3987
- return codec2.decode(data);
3988
- } finally {
3989
- await file.close();
3990
- }
3991
- }
3992
- /**
3993
- * @internal
3994
- */
3995
- async _writeFile(file, codec2, data) {
3996
- const encoded = arrayToBuffer(codec2.encode(data));
3997
- const checksum = CRC32.buf(encoded);
3998
- const result = Buffer.alloc(8 + encoded.length);
3999
- result.writeInt32LE(encoded.length, 0);
4000
- result.writeInt32LE(checksum, 4);
4001
- encoded.copy(result, 8);
4002
- await file.write(0, result);
4003
- log15("saved", {
4004
- size: encoded.length,
4005
- checksum
4006
- }, {
4007
- F: __dxlog_file18,
4008
- L: 124,
4009
- S: this,
4010
- C: (f, a) => f(...a)
4011
- });
4012
- }
4013
- async close() {
4014
- await this._invitationCleanupCtx.dispose();
4015
- await this.flush();
4016
- await this._metadataFile?.close();
4017
- this._metadataFile = void 0;
4018
- this._metadata = emptyEchoMetadata();
4019
- this._spaceLargeMetadata.clear();
4020
- }
4021
- /**
4022
- * Loads metadata from persistent storage.
4023
- */
4024
- async load() {
4025
- if (!this._metadataFile || this._metadataFile.closed) {
4026
- this._metadataFile = this._directory.getOrCreateFile("EchoMetadata");
4027
- }
4028
- try {
4029
- const metadata = await this._readFile(this._metadataFile, EchoMetadata);
4030
- if (metadata) {
4031
- this._metadata = metadata;
4032
- }
4033
- this._metadata.spaces?.forEach((space) => {
4034
- space.state ??= SpaceState.SPACE_ACTIVE;
4035
- });
4036
- } catch (err) {
4037
- log15.error("failed to load metadata", {
4038
- err
4039
- }, {
4040
- F: __dxlog_file18,
4041
- L: 156,
4042
- S: this,
4043
- C: (f, a) => f(...a)
4044
- });
4045
- this._metadata = emptyEchoMetadata();
4046
- }
4047
- await forEachAsync([
4048
- this._metadata.identity?.haloSpace.key,
4049
- ...this._metadata.spaces?.map((space) => space.key) ?? []
4050
- ].filter(isNotNullOrUndefined), async (key) => {
4051
- try {
4052
- await this._loadSpaceLargeMetadata(key);
4053
- } catch (err) {
4054
- log15.error("failed to load space large metadata", {
4055
- err
4056
- }, {
4057
- F: __dxlog_file18,
4058
- L: 168,
4059
- S: this,
4060
- C: (f, a) => f(...a)
4061
- });
4062
- }
4063
- });
4064
- scheduleTaskInterval2(this._invitationCleanupCtx, async () => {
4065
- for (const invitation of this._metadata.invitations ?? []) {
4066
- if (hasInvitationExpired(invitation) || isLegacyInvitationFormat(invitation)) {
4067
- await this.removeInvitation(invitation.invitationId);
4068
- }
4069
- }
4070
- }, EXPIRED_INVITATION_CLEANUP_INTERVAL);
4071
- }
4072
- async _save() {
4073
- const data = {
4074
- ...this._metadata,
4075
- version: STORAGE_VERSION,
4076
- created: this._metadata.created ?? /* @__PURE__ */ new Date(),
4077
- updated: /* @__PURE__ */ new Date()
4078
- };
4079
- this.update.emit(data);
4080
- const file = this._directory.getOrCreateFile("EchoMetadata");
4081
- await this._writeFile(file, EchoMetadata, data);
4082
- }
4083
- async _loadSpaceLargeMetadata(key) {
4084
- const file = this._directory.getOrCreateFile(`space_${key.toHex()}_large`);
4085
- try {
4086
- const metadata = await this._readFile(file, LargeSpaceMetadata);
4087
- if (metadata) {
4088
- this._spaceLargeMetadata.set(key, metadata);
4089
- }
4090
- } catch (err) {
4091
- log15.error("failed to load space large metadata", {
4092
- err
4093
- }, {
4094
- F: __dxlog_file18,
4095
- L: 210,
4096
- S: this,
4097
- C: (f, a) => f(...a)
4098
- });
4099
- }
4100
- }
4101
- async _saveSpaceLargeMetadata(key) {
4102
- const data = this._getLargeSpaceMetadata(key);
4103
- const file = this._directory.getOrCreateFile(`space_${key.toHex()}_large`);
4104
- await this._writeFile(file, LargeSpaceMetadata, data);
4105
- }
4106
- async flush() {
4107
- await this._directory.flush();
4108
- }
4109
- _getSpace(spaceKey) {
4110
- if (this._metadata.identity?.haloSpace.key.equals(spaceKey)) {
4111
- return this._metadata.identity.haloSpace;
4112
- }
4113
- const space = this.spaces.find((space2) => space2.key === spaceKey);
4114
- invariant13(space, "Space not found", {
4115
- F: __dxlog_file18,
4116
- L: 232,
4117
- S: this,
4118
- A: [
4119
- "space",
4120
- "'Space not found'"
4121
- ]
4122
- });
4123
- return space;
4124
- }
4125
- _getLargeSpaceMetadata(key) {
4126
- let entry = this._spaceLargeMetadata.get(key);
4127
- if (entry) {
4128
- return entry;
4129
- }
4130
- entry = emptyLargeSpaceMetadata();
4131
- this._spaceLargeMetadata.set(key, entry);
4132
- return entry;
4133
- }
4134
- /**
4135
- * Clears storage - doesn't work for now.
4136
- */
4137
- async clear() {
4138
- log15("clearing all metadata", void 0, {
4139
- F: __dxlog_file18,
4140
- L: 251,
4141
- S: this,
4142
- C: (f, a) => f(...a)
4143
- });
4144
- await this._directory.delete();
4145
- this._metadata = emptyEchoMetadata();
4146
- }
4147
- getIdentityRecord() {
4148
- return this._metadata.identity;
4149
- }
4150
- async setIdentityRecord(record) {
4151
- invariant13(!this._metadata.identity, "Cannot overwrite existing identity in metadata", {
4152
- F: __dxlog_file18,
4153
- L: 261,
4154
- S: this,
4155
- A: [
4156
- "!this._metadata.identity",
4157
- "'Cannot overwrite existing identity in metadata'"
4158
- ]
4159
- });
4160
- this._metadata.identity = record;
4161
- await this._save();
4162
- await this.flush();
4163
- }
4164
- getInvitations() {
4165
- return this._metadata.invitations ?? [];
4166
- }
4167
- async addInvitation(invitation) {
4168
- if (this._metadata.invitations?.find((i) => i.invitationId === invitation.invitationId)) {
4169
- return;
4170
- }
4171
- (this._metadata.invitations ??= []).push(invitation);
4172
- await this._save();
4173
- await this.flush();
4174
- }
4175
- async removeInvitation(invitationId) {
4176
- this._metadata.invitations = (this._metadata.invitations ?? []).filter((i) => i.invitationId !== invitationId);
4177
- await this._save();
4178
- await this.flush();
4179
- }
4180
- async addSpace(record) {
4181
- invariant13(!(this._metadata.spaces ?? []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata", {
4182
- F: __dxlog_file18,
4183
- L: 289,
4184
- S: this,
4185
- A: [
4186
- "!(this._metadata.spaces ?? []).find((space) => space.key === record.key)",
4187
- "'Cannot overwrite existing space in metadata'"
4188
- ]
4189
- });
4190
- (this._metadata.spaces ??= []).push(record);
4191
- await this._save();
4192
- await this.flush();
4193
- }
4194
- async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
4195
- this._getSpace(spaceKey).dataTimeframe = timeframe;
4196
- await this._save();
4197
- }
4198
- async setSpaceControlLatestTimeframe(spaceKey, timeframe) {
4199
- this._getSpace(spaceKey).controlTimeframe = timeframe;
4200
- await this._save();
4201
- await this.flush();
4202
- }
4203
- async setCache(spaceKey, cache) {
4204
- this._getSpace(spaceKey).cache = cache;
4205
- await this._save();
4206
- }
4207
- async setWritableFeedKeys(spaceKey, controlFeedKey, dataFeedKey) {
4208
- const space = this._getSpace(spaceKey);
4209
- space.controlFeedKey = controlFeedKey;
4210
- space.dataFeedKey = dataFeedKey;
4211
- await this._save();
4212
- await this.flush();
4213
- }
4214
- async setSpaceState(spaceKey, state) {
4215
- this._getSpace(spaceKey).state = state;
4216
- await this._save();
4217
- await this.flush();
4218
- }
4219
- getSpaceControlPipelineSnapshot(spaceKey) {
4220
- return this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot;
4221
- }
4222
- async setSpaceControlPipelineSnapshot(spaceKey, snapshot) {
4223
- this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot = snapshot;
4224
- await this._saveSpaceLargeMetadata(spaceKey);
4225
- await this.flush();
4226
- }
4227
- };
4228
- _ts_decorate10([
4229
- synchronized5
4230
- ], MetadataStore.prototype, "load", null);
4231
- _ts_decorate10([
4232
- synchronized5
4233
- ], MetadataStore.prototype, "_save", null);
4234
- _ts_decorate10([
4235
- synchronized5
4236
- ], MetadataStore.prototype, "_saveSpaceLargeMetadata", null);
4237
- var fromBytesInt32 = (buf) => buf.readInt32LE(0);
4238
- var hasInvitationExpired = (invitation) => {
4239
- return Boolean(invitation.created && invitation.lifetime && invitation.lifetime !== 0 && invitation.created.getTime() + invitation.lifetime * 1e3 < Date.now());
4240
- };
4241
- var isLegacyInvitationFormat = (invitation) => {
4242
- return invitation.type === Invitation.Type.MULTIUSE;
4243
- };
4244
-
4245
- export {
4246
- codec,
4247
- valueEncoding,
4248
- createMappedFeedWriter,
4249
- SnapshotManager,
4250
- SnapshotStore,
4251
- DocumentsSynchronizer,
4252
- diffCollectionState,
4253
- AuthExtension,
4254
- mapTimeframeToFeedIndexes,
4255
- mapFeedIndexesToTimeframe,
4256
- startAfter,
4257
- TimeframeClock,
4258
- Pipeline,
4259
- Space,
4260
- createIdFromSpaceKey,
4261
- CredentialRetrieverExtension,
4262
- CredentialServerExtension,
4263
- MOCK_AUTH_PROVIDER,
4264
- MOCK_AUTH_VERIFIER,
4265
- SpaceProtocol,
4266
- AuthStatus,
4267
- SpaceProtocolSession,
4268
- SpaceManager,
4269
- LevelDBStorageAdapter,
4270
- encodingOptions,
4271
- AutomergeHost,
4272
- getSpaceKeyFromDoc,
4273
- deriveCollectionIdFromSpaceId,
4274
- getSpaceIdFromCollectionId,
4275
- MeshEchoReplicator,
4276
- EchoDataMonitor,
4277
- DataServiceImpl,
4278
- MetadataStore,
4279
- hasInvitationExpired
4280
- };
4281
- //# sourceMappingURL=chunk-P6XSIJKM.mjs.map