@dxos/echo-pipeline 0.8.4-main.84f28bd → 0.8.4-main.ae835ea

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/dist/lib/browser/{chunk-GBFX5J5B.mjs → chunk-FR6GJ4S4.mjs} +239 -162
  2. package/dist/lib/browser/chunk-FR6GJ4S4.mjs.map +7 -0
  3. package/dist/lib/browser/chunk-WJJXJTNS.mjs +240 -0
  4. package/dist/lib/browser/chunk-WJJXJTNS.mjs.map +7 -0
  5. package/dist/lib/browser/filter/index.mjs +3 -1
  6. package/dist/lib/browser/index.mjs +1094 -345
  7. package/dist/lib/browser/index.mjs.map +4 -4
  8. package/dist/lib/browser/meta.json +1 -1
  9. package/dist/lib/browser/testing/index.mjs +36 -14
  10. package/dist/lib/browser/testing/index.mjs.map +3 -3
  11. package/dist/lib/node-esm/{chunk-FQFKWA3X.mjs → chunk-JQCCVDLT.mjs} +239 -162
  12. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs.map +7 -0
  13. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs +240 -0
  14. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs.map +7 -0
  15. package/dist/lib/node-esm/filter/index.mjs +3 -1
  16. package/dist/lib/node-esm/index.mjs +1094 -345
  17. package/dist/lib/node-esm/index.mjs.map +4 -4
  18. package/dist/lib/node-esm/meta.json +1 -1
  19. package/dist/lib/node-esm/testing/index.mjs +36 -14
  20. package/dist/lib/node-esm/testing/index.mjs.map +3 -3
  21. package/dist/types/src/automerge/automerge-host.d.ts +15 -28
  22. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  23. package/dist/types/src/automerge/collection-synchronizer.d.ts +3 -2
  24. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  25. package/dist/types/src/automerge/echo-network-adapter.d.ts +8 -1
  26. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  27. package/dist/types/src/automerge/echo-replicator.d.ts +21 -2
  28. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  29. package/dist/types/src/automerge/index.d.ts +1 -1
  30. package/dist/types/src/automerge/index.d.ts.map +1 -1
  31. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
  32. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  33. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +1 -0
  34. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  35. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  36. package/dist/types/src/common/codec.d.ts +1 -1
  37. package/dist/types/src/common/codec.d.ts.map +1 -1
  38. package/dist/types/src/db-host/data-service.d.ts +2 -2
  39. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  40. package/dist/types/src/db-host/database-root.d.ts.map +1 -1
  41. package/dist/types/src/db-host/documents-synchronizer.d.ts +2 -2
  42. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  43. package/dist/types/src/db-host/echo-host.d.ts +2 -2
  44. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  45. package/dist/types/src/db-host/query-service.d.ts +1 -1
  46. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  47. package/dist/types/src/db-host/space-state-manager.d.ts +1 -1
  48. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  49. package/dist/types/src/edge/echo-edge-replicator.d.ts +4 -2
  50. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  51. package/dist/types/src/filter/filter-match.d.ts +4 -1
  52. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  53. package/dist/types/src/metadata/metadata-store.d.ts +1 -1
  54. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  55. package/dist/types/src/pipeline/pipeline.d.ts +1 -1
  56. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  57. package/dist/types/src/query/errors.d.ts +24 -8
  58. package/dist/types/src/query/errors.d.ts.map +1 -1
  59. package/dist/types/src/query/plan.d.ts +8 -1
  60. package/dist/types/src/query/plan.d.ts.map +1 -1
  61. package/dist/types/src/query/query-executor.d.ts +13 -1
  62. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  63. package/dist/types/src/query/query-planner.d.ts +2 -0
  64. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  65. package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
  66. package/dist/types/src/space/control-pipeline.d.ts +1 -1
  67. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  68. package/dist/types/src/space/space-manager.d.ts +1 -1
  69. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  70. package/dist/types/src/space/space-protocol.d.ts +1 -1
  71. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  72. package/dist/types/src/space/space.d.ts +1 -1
  73. package/dist/types/src/space/space.d.ts.map +1 -1
  74. package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
  75. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  76. package/dist/types/src/testing/test-data.d.ts.map +1 -1
  77. package/dist/types/src/testing/test-replicator.d.ts +1 -0
  78. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  79. package/dist/types/src/testing/test-schema.d.ts +1 -1
  80. package/dist/types/src/testing/test-schema.d.ts.map +1 -1
  81. package/dist/types/src/util.d.ts +1 -1
  82. package/dist/types/src/util.d.ts.map +1 -1
  83. package/dist/types/tsconfig.tsbuildinfo +1 -1
  84. package/package.json +40 -38
  85. package/src/automerge/automerge-host.test.ts +18 -8
  86. package/src/automerge/automerge-host.ts +273 -74
  87. package/src/automerge/automerge-repo.test.ts +67 -16
  88. package/src/automerge/collection-synchronizer.test.ts +6 -5
  89. package/src/automerge/collection-synchronizer.ts +16 -11
  90. package/src/automerge/echo-data-monitor.ts +1 -1
  91. package/src/automerge/echo-network-adapter.test.ts +3 -3
  92. package/src/automerge/echo-network-adapter.ts +38 -6
  93. package/src/automerge/echo-replicator.ts +23 -2
  94. package/src/automerge/index.ts +1 -1
  95. package/src/automerge/leveldb-storage-adapter.ts +7 -7
  96. package/src/automerge/mesh-echo-replicator-connection.ts +4 -0
  97. package/src/automerge/mesh-echo-replicator.ts +2 -1
  98. package/src/automerge/storage-adapter.test.ts +1 -1
  99. package/src/common/space-id.ts +1 -1
  100. package/src/db-host/data-service.ts +9 -17
  101. package/src/db-host/database-root.ts +2 -2
  102. package/src/db-host/documents-synchronizer.test.ts +1 -1
  103. package/src/db-host/documents-synchronizer.ts +39 -26
  104. package/src/db-host/echo-host.ts +14 -15
  105. package/src/db-host/query-service.ts +10 -2
  106. package/src/db-host/space-state-manager.ts +2 -2
  107. package/src/edge/echo-edge-replicator.test.ts +5 -3
  108. package/src/edge/echo-edge-replicator.ts +105 -27
  109. package/src/filter/filter-match.test.ts +33 -13
  110. package/src/filter/filter-match.ts +164 -10
  111. package/src/metadata/metadata-store.ts +3 -3
  112. package/src/pipeline/pipeline-stress.test.ts +4 -2
  113. package/src/pipeline/pipeline.test.ts +3 -2
  114. package/src/pipeline/pipeline.ts +8 -5
  115. package/src/query/errors.ts +2 -0
  116. package/src/query/plan.ts +12 -1
  117. package/src/query/query-executor.ts +154 -14
  118. package/src/query/query-planner.test.ts +146 -2
  119. package/src/query/query-planner.ts +81 -11
  120. package/src/space/admission-discovery-extension.ts +2 -2
  121. package/src/space/control-pipeline.test.ts +4 -3
  122. package/src/space/control-pipeline.ts +9 -6
  123. package/src/space/space-manager.browser.test.ts +1 -1
  124. package/src/space/space-manager.ts +5 -4
  125. package/src/space/space-protocol.browser.test.ts +2 -2
  126. package/src/space/space-protocol.test.ts +3 -2
  127. package/src/space/space-protocol.ts +6 -3
  128. package/src/space/space.test.ts +1 -1
  129. package/src/space/space.ts +3 -2
  130. package/src/testing/test-agent-builder.ts +4 -3
  131. package/src/testing/test-data.ts +1 -1
  132. package/src/testing/test-replicator.ts +4 -0
  133. package/src/testing/test-schema.ts +1 -1
  134. package/src/util.ts +1 -1
  135. package/dist/lib/browser/chunk-ANZAS5CC.mjs +0 -126
  136. package/dist/lib/browser/chunk-ANZAS5CC.mjs.map +0 -7
  137. package/dist/lib/browser/chunk-GBFX5J5B.mjs.map +0 -7
  138. package/dist/lib/node-esm/chunk-2SAZ7CCF.mjs +0 -126
  139. package/dist/lib/node-esm/chunk-2SAZ7CCF.mjs.map +0 -7
  140. package/dist/lib/node-esm/chunk-FQFKWA3X.mjs.map +0 -7
@@ -1,8 +1,9 @@
1
1
  import "@dxos/node-std/globals";
2
2
  import {
3
3
  filterMatchObject,
4
+ filterMatchObjectJSON,
4
5
  filterMatchValue
5
- } from "./chunk-ANZAS5CC.mjs";
6
+ } from "./chunk-WJJXJTNS.mjs";
6
7
  import {
7
8
  AuthExtension,
8
9
  AuthStatus,
@@ -25,7 +26,7 @@ import {
25
26
  mapTimeframeToFeedIndexes,
26
27
  startAfter,
27
28
  valueEncoding
28
- } from "./chunk-GBFX5J5B.mjs";
29
+ } from "./chunk-FR6GJ4S4.mjs";
29
30
  import "./chunk-CGS2ULMK.mjs";
30
31
 
31
32
  // src/db-host/data-service.ts
@@ -35,17 +36,10 @@ import { invariant as invariant7 } from "@dxos/invariant";
35
36
  import { SpaceId as SpaceId2 } from "@dxos/keys";
36
37
  import { log as log7 } from "@dxos/log";
37
38
 
38
- // src/db-host/documents-synchronizer.ts
39
- import { next as A2 } from "@automerge/automerge";
40
- import { UpdateScheduler } from "@dxos/async";
41
- import { Resource as Resource5 } from "@dxos/context";
42
- import { invariant as invariant6 } from "@dxos/invariant";
43
- import { log as log6 } from "@dxos/log";
44
-
45
39
  // src/automerge/automerge-host.ts
46
- import { getBackend, getHeads, isAutomerge, equals as headsEquals, save } from "@automerge/automerge";
40
+ import { getBackend, getHeads, equals as headsEquals, isAutomerge, save } from "@automerge/automerge";
47
41
  import { Repo, interpretAsDocumentId } from "@automerge/automerge-repo";
48
- import { Event as Event2, asyncTimeout } from "@dxos/async";
42
+ import { DeferredTask, Event as Event2, asyncTimeout } from "@dxos/async";
49
43
  import { Context, Resource as Resource3, cancelWithContext } from "@dxos/context";
50
44
  import { DatabaseDirectory } from "@dxos/echo-protocol";
51
45
  import { invariant as invariant2 } from "@dxos/invariant";
@@ -53,11 +47,13 @@ import { PublicKey } from "@dxos/keys";
53
47
  import { log as log3 } from "@dxos/log";
54
48
  import { objectPointerCodec } from "@dxos/protocols";
55
49
  import { trace as trace2 } from "@dxos/tracing";
56
- import { bufferToArray } from "@dxos/util";
50
+ import { ComplexSet, bufferToArray, isNonNullable as isNonNullable2, range } from "@dxos/util";
57
51
 
58
52
  // src/automerge/collection-synchronizer.ts
59
- import { next as am } from "@automerge/automerge";
60
- import { asyncReturn, Event, scheduleTask, scheduleTaskInterval } from "@dxos/async";
53
+ import { next as A } from "@automerge/automerge";
54
+ import * as Array2 from "effect/Array";
55
+ import * as Record from "effect/Record";
56
+ import { Event, asyncReturn, scheduleTask, scheduleTaskInterval } from "@dxos/async";
61
57
  import { Resource } from "@dxos/context";
62
58
  import { log } from "@dxos/log";
63
59
  import { trace } from "@dxos/tracing";
@@ -72,15 +68,18 @@ var __dxlog_file = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/automerg
72
68
  var MIN_QUERY_INTERVAL = 5e3;
73
69
  var POLL_INTERVAL = 3e4;
74
70
  var CollectionSynchronizer = class extends Resource {
71
+ _sendCollectionState;
72
+ _queryCollectionState;
73
+ _shouldSyncCollection;
74
+ /**
75
+ * CollectionId -> State.
76
+ */
77
+ _perCollectionStates = /* @__PURE__ */ new Map();
78
+ _activeCollections = /* @__PURE__ */ new Set();
79
+ _connectedPeers = /* @__PURE__ */ new Set();
80
+ remoteStateUpdated = new Event();
75
81
  constructor(params) {
76
82
  super();
77
- /**
78
- * CollectionId -> State.
79
- */
80
- this._perCollectionStates = /* @__PURE__ */ new Map();
81
- this._activeCollections = /* @__PURE__ */ new Set();
82
- this._connectedPeers = /* @__PURE__ */ new Set();
83
- this.remoteStateUpdated = new Event();
84
83
  this._sendCollectionState = params.sendCollectionState;
85
84
  this._queryCollectionState = params.queryCollectionState;
86
85
  this._shouldSyncCollection = params.shouldSyncCollection;
@@ -110,7 +109,7 @@ var CollectionSynchronizer = class extends Resource {
110
109
  state
111
110
  }, {
112
111
  F: __dxlog_file,
113
- L: 76,
112
+ L: 78,
114
113
  S: this,
115
114
  C: (f, a) => f(...a)
116
115
  });
@@ -129,7 +128,7 @@ var CollectionSynchronizer = class extends Resource {
129
128
  collectionId
130
129
  }, {
131
130
  F: __dxlog_file,
132
- L: 90,
131
+ L: 92,
133
132
  S: this,
134
133
  C: (f, a) => f(...a)
135
134
  });
@@ -212,7 +211,7 @@ var CollectionSynchronizer = class extends Resource {
212
211
  state
213
212
  }, {
214
213
  F: __dxlog_file,
215
- L: 171,
214
+ L: 173,
216
215
  S: this,
217
216
  C: (f, a) => f(...a)
218
217
  });
@@ -268,19 +267,23 @@ CollectionSynchronizer = _ts_decorate([
268
267
  trace.resource()
269
268
  ], CollectionSynchronizer);
270
269
  var diffCollectionState = (local, remote) => {
271
- const allDocuments = /* @__PURE__ */ new Set([
272
- ...Object.keys(local.documents),
273
- ...Object.keys(remote.documents)
274
- ]);
270
+ const localDocuments = Record.filter(local.documents, (heads) => heads.length > 0);
271
+ const remoteDocuments = Record.filter(remote.documents, (heads) => heads.length > 0);
272
+ const allDocuments = [
273
+ .../* @__PURE__ */ new Set([
274
+ ...Record.keys(localDocuments),
275
+ ...Record.keys(remoteDocuments)
276
+ ])
277
+ ];
275
278
  const missingOnRemote = [];
276
279
  const missingOnLocal = [];
277
280
  const different = [];
278
281
  for (const documentId of allDocuments) {
279
- if (!local.documents[documentId]) {
282
+ if (!localDocuments[documentId]) {
280
283
  missingOnLocal.push(documentId);
281
- } else if (!remote.documents[documentId]) {
284
+ } else if (!remoteDocuments[documentId]) {
282
285
  missingOnRemote.push(documentId);
283
- } else if (!am.equals(local.documents[documentId], remote.documents[documentId])) {
286
+ } else if (!A.equals(local.documents[documentId], remote.documents[documentId])) {
284
287
  different.push(documentId);
285
288
  }
286
289
  }
@@ -295,7 +298,7 @@ var validateCollectionState = (state) => {
295
298
  if (!isValidDocumentId(documentId)) {
296
299
  throw new Error(`Invalid documentId: ${documentId}`);
297
300
  }
298
- if (Array.isArray(heads) && heads.some((head) => typeof head !== "string")) {
301
+ if (Array2.isArray(heads) && heads.some((head) => typeof head !== "string")) {
299
302
  throw new Error(`Invalid heads: ${heads}`);
300
303
  }
301
304
  });
@@ -309,7 +312,7 @@ var getSpanName = (peerId) => {
309
312
 
310
313
  // src/automerge/echo-network-adapter.ts
311
314
  import { NetworkAdapter } from "@automerge/automerge-repo";
312
- import { synchronized, Trigger } from "@dxos/async";
315
+ import { Trigger, synchronized } from "@dxos/async";
313
316
  import { LifecycleState } from "@dxos/context";
314
317
  import { invariant } from "@dxos/invariant";
315
318
  import { log as log2 } from "@dxos/log";
@@ -329,8 +332,17 @@ function _ts_decorate2(decorators, target, key, desc) {
329
332
  }
330
333
  var __dxlog_file2 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
331
334
  var EchoNetworkAdapter = class extends NetworkAdapter {
335
+ _params;
336
+ _replicators = /* @__PURE__ */ new Set();
337
+ /**
338
+ * Remote peer id -> connection.
339
+ */
340
+ _connections = /* @__PURE__ */ new Map();
341
+ _lifecycleState = LifecycleState.CLOSED;
342
+ _connected = new Trigger();
343
+ _ready = new Trigger();
332
344
  constructor(_params) {
333
- super(), this._params = _params, this._replicators = /* @__PURE__ */ new Set(), this._connections = /* @__PURE__ */ new Map(), this._lifecycleState = LifecycleState.CLOSED, this._connected = new Trigger(), this._ready = new Trigger();
345
+ super(), this._params = _params;
334
346
  }
335
347
  isReady() {
336
348
  return this._lifecycleState === LifecycleState.OPEN;
@@ -380,7 +392,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
380
392
  async addReplicator(replicator) {
381
393
  invariant(this._lifecycleState === LifecycleState.OPEN, void 0, {
382
394
  F: __dxlog_file2,
383
- L: 129,
395
+ L: 137,
384
396
  S: this,
385
397
  A: [
386
398
  "this._lifecycleState === LifecycleState.OPEN",
@@ -389,7 +401,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
389
401
  });
390
402
  invariant(this.peerId, void 0, {
391
403
  F: __dxlog_file2,
392
- L: 130,
404
+ L: 138,
393
405
  S: this,
394
406
  A: [
395
407
  "this.peerId",
@@ -398,7 +410,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
398
410
  });
399
411
  invariant(!this._replicators.has(replicator), void 0, {
400
412
  F: __dxlog_file2,
401
- L: 131,
413
+ L: 139,
402
414
  S: this,
403
415
  A: [
404
416
  "!this._replicators.has(replicator)",
@@ -422,7 +434,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
422
434
  async removeReplicator(replicator) {
423
435
  invariant(this._lifecycleState === LifecycleState.OPEN, void 0, {
424
436
  F: __dxlog_file2,
425
- L: 150,
437
+ L: 158,
426
438
  S: this,
427
439
  A: [
428
440
  "this._lifecycleState === LifecycleState.OPEN",
@@ -431,7 +443,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
431
443
  });
432
444
  invariant(this._replicators.has(replicator), void 0, {
433
445
  F: __dxlog_file2,
434
- L: 151,
446
+ L: 159,
435
447
  S: this,
436
448
  A: [
437
449
  "this._replicators.has(replicator)",
@@ -482,6 +494,27 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
482
494
  }) ? connection.connection.peerId : null;
483
495
  }).filter(isNonNullable);
484
496
  }
497
+ bundleSyncEnabledForPeer(peerId) {
498
+ const connection = this._connections.get(peerId);
499
+ if (!connection) {
500
+ return false;
501
+ }
502
+ return connection.connection.bundleSyncEnabled;
503
+ }
504
+ async pushBundle(peerId, bundle) {
505
+ const connection = this._connections.get(peerId);
506
+ if (!connection) {
507
+ throw new Error("Connection not found.");
508
+ }
509
+ return connection.connection.pushBundle(bundle);
510
+ }
511
+ async pullBundle(peerId, docHeads) {
512
+ const connection = this._connections.get(peerId);
513
+ if (!connection) {
514
+ throw new Error("Connection not found.");
515
+ }
516
+ return connection.connection.pullBundle(docHeads);
517
+ }
485
518
  _send(message) {
486
519
  const connectionEntry = this._connections.get(message.targetId);
487
520
  if (!connectionEntry) {
@@ -494,7 +527,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
494
527
  if (connectionEntry.isOpen) {
495
528
  log2.catch(err, void 0, {
496
529
  F: __dxlog_file2,
497
- L: 221,
530
+ L: 253,
498
531
  S: this,
499
532
  C: (f, a) => f(...a)
500
533
  });
@@ -507,13 +540,13 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
507
540
  peerId: connection.peerId
508
541
  }, {
509
542
  F: __dxlog_file2,
510
- L: 229,
543
+ L: 261,
511
544
  S: this,
512
545
  C: (f, a) => f(...a)
513
546
  });
514
547
  invariant(!this._connections.has(connection.peerId), void 0, {
515
548
  F: __dxlog_file2,
516
- L: 230,
549
+ L: 262,
517
550
  S: this,
518
551
  A: [
519
552
  "!this._connections.has(connection.peerId as PeerId)",
@@ -540,7 +573,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
540
573
  if (connectionEntry.isOpen) {
541
574
  log2.catch(err, void 0, {
542
575
  F: __dxlog_file2,
543
- L: 254,
576
+ L: 286,
544
577
  S: this,
545
578
  C: (f, a) => f(...a)
546
579
  });
@@ -551,7 +584,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
551
584
  peerId: connection.peerId
552
585
  }, {
553
586
  F: __dxlog_file2,
554
- L: 259,
587
+ L: 291,
555
588
  S: this,
556
589
  C: (f, a) => f(...a)
557
590
  });
@@ -573,14 +606,14 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
573
606
  peerId: connection.peerId
574
607
  }, {
575
608
  F: __dxlog_file2,
576
- L: 276,
609
+ L: 308,
577
610
  S: this,
578
611
  C: (f, a) => f(...a)
579
612
  });
580
613
  const entry = this._connections.get(connection.peerId);
581
614
  invariant(entry, void 0, {
582
615
  F: __dxlog_file2,
583
- L: 278,
616
+ L: 310,
584
617
  S: this,
585
618
  A: [
586
619
  "entry",
@@ -594,13 +627,13 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
594
627
  this._params.monitor?.recordPeerDisconnected(connection.peerId);
595
628
  void entry.writer.abort().catch((err) => log2.catch(err, void 0, {
596
629
  F: __dxlog_file2,
597
- L: 284,
630
+ L: 316,
598
631
  S: this,
599
632
  C: (f, a) => f(...a)
600
633
  }));
601
634
  void entry.reader.cancel().catch((err) => log2.catch(err, void 0, {
602
635
  F: __dxlog_file2,
603
- L: 285,
636
+ L: 317,
604
637
  S: this,
605
638
  C: (f, a) => f(...a)
606
639
  }));
@@ -615,14 +648,14 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
615
648
  peerId: connection.peerId
616
649
  }, {
617
650
  F: __dxlog_file2,
618
- L: 295,
651
+ L: 327,
619
652
  S: this,
620
653
  C: (f, a) => f(...a)
621
654
  });
622
655
  const entry = this._connections.get(connection.peerId);
623
656
  invariant(entry, void 0, {
624
657
  F: __dxlog_file2,
625
- L: 297,
658
+ L: 329,
626
659
  S: this,
627
660
  A: [
628
661
  "entry",
@@ -662,6 +695,7 @@ var isEchoPeerMetadata = (metadata) => metadata?.dxos_peerSource === "EchoNetwor
662
695
  // src/automerge/heads-store.ts
663
696
  import { headsEncoding } from "@dxos/indexing";
664
697
  var HeadsStore = class {
698
+ _db;
665
699
  constructor({ db }) {
666
700
  this._db = db;
667
701
  }
@@ -682,14 +716,15 @@ var HeadsStore = class {
682
716
  };
683
717
 
684
718
  // src/automerge/leveldb-storage-adapter.ts
685
- import { LifecycleState as LifecycleState2, Resource as Resource2 } from "@dxos/context";
719
+ import { Resource as Resource2 } from "@dxos/context";
686
720
  var LevelDBStorageAdapter = class extends Resource2 {
721
+ _params;
687
722
  constructor(_params) {
688
723
  super(), this._params = _params;
689
724
  }
690
725
  async load(keyArray) {
691
726
  try {
692
- if (this._lifecycleState !== LifecycleState2.OPEN) {
727
+ if (!this.isOpen) {
693
728
  return void 0;
694
729
  }
695
730
  const startMs = Date.now();
@@ -707,7 +742,7 @@ var LevelDBStorageAdapter = class extends Resource2 {
707
742
  }
708
743
  }
709
744
  async save(keyArray, binary) {
710
- if (this._lifecycleState !== LifecycleState2.OPEN) {
745
+ if (!this.isOpen) {
711
746
  return void 0;
712
747
  }
713
748
  const startMs = Date.now();
@@ -725,7 +760,7 @@ var LevelDBStorageAdapter = class extends Resource2 {
725
760
  this._params.monitor?.recordStoreDuration(Date.now() - startMs);
726
761
  }
727
762
  async remove(keyArray) {
728
- if (this._lifecycleState !== LifecycleState2.OPEN) {
763
+ if (!this.isOpen) {
729
764
  return void 0;
730
765
  }
731
766
  await this._params.db.del(keyArray, {
@@ -733,7 +768,7 @@ var LevelDBStorageAdapter = class extends Resource2 {
733
768
  });
734
769
  }
735
770
  async loadRange(keyPrefix) {
736
- if (this._lifecycleState !== LifecycleState2.OPEN) {
771
+ if (!this.isOpen) {
737
772
  return [];
738
773
  }
739
774
  const startMs = Date.now();
@@ -756,7 +791,7 @@ var LevelDBStorageAdapter = class extends Resource2 {
756
791
  return result;
757
792
  }
758
793
  async removeRange(keyPrefix) {
759
- if (this._lifecycleState !== LifecycleState2.OPEN) {
794
+ if (!this.isOpen) {
760
795
  return void 0;
761
796
  }
762
797
  const batch = this._params.db.batch();
@@ -800,19 +835,38 @@ var FIND_PARAMS = {
800
835
  "requesting"
801
836
  ]
802
837
  };
838
+ var BUNDLE_SIZE = 100;
839
+ var BUNDLE_SYNC_CONCURRENCY = 2;
840
+ var BUNDLE_SYNC_THRESHOLD = 50;
803
841
  var AutomergeHost = class extends Resource3 {
842
+ _db;
843
+ _indexMetadataStore;
844
+ _echoNetworkAdapter;
845
+ _collectionSynchronizer = new CollectionSynchronizer({
846
+ queryCollectionState: this._queryCollectionState.bind(this),
847
+ sendCollectionState: this._sendCollectionState.bind(this),
848
+ shouldSyncCollection: this._shouldSyncCollection.bind(this)
849
+ });
850
+ _repo;
851
+ _storage;
852
+ _headsStore;
853
+ _syncTask = void 0;
854
+ /**
855
+ * Cache of collections that would be synced on next sync task run.
856
+ */
857
+ _collectionsToSync = new ComplexSet(({ collectionId, peerId }) => `${collectionId}|${peerId}`);
858
+ _peerId;
859
+ _peerIdProvider;
860
+ _getSpaceKeyByRootDocumentId;
861
+ collectionStateUpdated = new Event2();
862
+ /**
863
+ * Fired after a batch of documents was saved to disk.
864
+ */
865
+ documentsSaved = new Event2();
866
+ _headsUpdates = /* @__PURE__ */ new Map();
867
+ _onHeadsChangedTask;
804
868
  constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider, getSpaceKeyByRootDocumentId }) {
805
869
  super();
806
- this._collectionSynchronizer = new CollectionSynchronizer({
807
- queryCollectionState: this._queryCollectionState.bind(this),
808
- sendCollectionState: this._sendCollectionState.bind(this),
809
- shouldSyncCollection: this._shouldSyncCollection.bind(this)
810
- });
811
- this.collectionStateUpdated = new Event2();
812
- /**
813
- * Fired after a batch of documents was saved to disk.
814
- */
815
- this.documentsSaved = new Event2();
816
870
  this._db = db;
817
871
  this._storage = new LevelDBStorageAdapter({
818
872
  db: db.sublevel("automerge"),
@@ -838,6 +892,11 @@ var AutomergeHost = class extends Resource3 {
838
892
  }
839
893
  async _open() {
840
894
  this._peerId = `host-${this._peerIdProvider?.() ?? PublicKey.random().toHex()}`;
895
+ this._onHeadsChangedTask = new DeferredTask(this._ctx, async () => {
896
+ const docHeads = Array.from(this._headsUpdates.entries());
897
+ this._headsUpdates.clear();
898
+ this._onHeadsChanged(docHeads);
899
+ });
841
900
  await this._storage.open?.();
842
901
  this._repo = new Repo({
843
902
  peerId: this._peerId,
@@ -865,6 +924,28 @@ var AutomergeHost = class extends Resource3 {
865
924
  }
866
925
  }
867
926
  });
927
+ this._syncTask = new DeferredTask(this._ctx, async () => {
928
+ const collectionToSync = Array.from(this._collectionsToSync.values());
929
+ if (collectionToSync.length === 0) {
930
+ return;
931
+ }
932
+ await Promise.all(collectionToSync.map(async ({ collectionId, peerId }) => {
933
+ try {
934
+ await this._handleCollectionSync(collectionId, peerId);
935
+ } catch (err) {
936
+ log3.error("failed to sync collection", {
937
+ collectionId,
938
+ peerId,
939
+ err
940
+ }, {
941
+ F: __dxlog_file3,
942
+ L: 223,
943
+ S: this,
944
+ C: (f, a) => f(...a)
945
+ });
946
+ }
947
+ }));
948
+ });
868
949
  await this._echoNetworkAdapter.open();
869
950
  await this._collectionSynchronizer.open();
870
951
  await this._echoNetworkAdapter.open();
@@ -874,7 +955,8 @@ var AutomergeHost = class extends Resource3 {
874
955
  await this._collectionSynchronizer.close();
875
956
  await this._storage.close?.();
876
957
  await this._echoNetworkAdapter.close();
877
- await this._ctx.dispose();
958
+ this._syncTask = void 0;
959
+ this._onHeadsChangedTask = void 0;
878
960
  }
879
961
  /**
880
962
  * @deprecated To be abstracted away.
@@ -889,15 +971,42 @@ var AutomergeHost = class extends Resource3 {
889
971
  return Object.keys(this._repo.handles).length;
890
972
  }
891
973
  async addReplicator(replicator) {
974
+ invariant2(this.isOpen, "AutomergeHost is not open", {
975
+ F: __dxlog_file3,
976
+ L: 259,
977
+ S: this,
978
+ A: [
979
+ "this.isOpen",
980
+ "'AutomergeHost is not open'"
981
+ ]
982
+ });
892
983
  await this._echoNetworkAdapter.addReplicator(replicator);
893
984
  }
894
985
  async removeReplicator(replicator) {
986
+ invariant2(this.isOpen, "AutomergeHost is not open", {
987
+ F: __dxlog_file3,
988
+ L: 264,
989
+ S: this,
990
+ A: [
991
+ "this.isOpen",
992
+ "'AutomergeHost is not open'"
993
+ ]
994
+ });
895
995
  await this._echoNetworkAdapter.removeReplicator(replicator);
896
996
  }
897
997
  /**
898
998
  * Loads the document handle from the repo and waits for it to be ready.
899
999
  */
900
1000
  async loadDoc(ctx, documentId, opts) {
1001
+ invariant2(this.isOpen, "AutomergeHost is not open", {
1002
+ F: __dxlog_file3,
1003
+ L: 272,
1004
+ S: this,
1005
+ A: [
1006
+ "this.isOpen",
1007
+ "'AutomergeHost is not open'"
1008
+ ]
1009
+ });
901
1010
  let handle;
902
1011
  if (typeof documentId === "string") {
903
1012
  handle = this._repo.handles[documentId];
@@ -915,6 +1024,15 @@ var AutomergeHost = class extends Resource3 {
915
1024
  return handle;
916
1025
  }
917
1026
  async exportDoc(ctx, id) {
1027
+ invariant2(this.isOpen, "AutomergeHost is not open", {
1028
+ F: __dxlog_file3,
1029
+ L: 295,
1030
+ S: this,
1031
+ A: [
1032
+ "this.isOpen",
1033
+ "'AutomergeHost is not open'"
1034
+ ]
1035
+ });
918
1036
  const documentId = interpretAsDocumentId(id);
919
1037
  const chunks = await this._storage.loadRange([
920
1038
  documentId
@@ -925,6 +1043,15 @@ var AutomergeHost = class extends Resource3 {
925
1043
  * Create new persisted document.
926
1044
  */
927
1045
  createDoc(initialValue, opts) {
1046
+ invariant2(this.isOpen, "AutomergeHost is not open", {
1047
+ F: __dxlog_file3,
1048
+ L: 306,
1049
+ S: this,
1050
+ A: [
1051
+ "this.isOpen",
1052
+ "'AutomergeHost is not open'"
1053
+ ]
1054
+ });
928
1055
  if (opts?.preserveHistory) {
929
1056
  if (initialValue instanceof Uint8Array) {
930
1057
  return this._repo.import(initialValue);
@@ -941,6 +1068,15 @@ var AutomergeHost = class extends Resource3 {
941
1068
  }
942
1069
  }
943
1070
  async waitUntilHeadsReplicated(heads) {
1071
+ invariant2(this.isOpen, "AutomergeHost is not open", {
1072
+ F: __dxlog_file3,
1073
+ L: 328,
1074
+ S: this,
1075
+ A: [
1076
+ "this.isOpen",
1077
+ "'AutomergeHost is not open'"
1078
+ ]
1079
+ });
944
1080
  const entries = heads.entries;
945
1081
  if (!entries?.length) {
946
1082
  return;
@@ -956,10 +1092,10 @@ var AutomergeHost = class extends Resource3 {
956
1092
  return !(currentHeads !== null && headsEquals(currentHeads, targetHeads));
957
1093
  });
958
1094
  if (headsToWait.length > 0) {
959
- await Promise.all(headsToWait.map(async (entry, index) => {
1095
+ await Promise.all(headsToWait.map(async (entry) => {
960
1096
  const handle = await this.loadDoc(Context.default(void 0, {
961
1097
  F: __dxlog_file3,
962
- L: 293
1098
+ L: 346
963
1099
  }), entry.documentId);
964
1100
  await waitForHeads(handle, entry.heads);
965
1101
  }));
@@ -967,12 +1103,21 @@ var AutomergeHost = class extends Resource3 {
967
1103
  await this._repo.flush(documentIds.filter((documentId) => this._repo.handles[documentId] && this._repo.handles[documentId].isReady()));
968
1104
  }
969
1105
  async reIndexHeads(documentIds) {
1106
+ invariant2(this.isOpen, "AutomergeHost is not open", {
1107
+ F: __dxlog_file3,
1108
+ L: 359,
1109
+ S: this,
1110
+ A: [
1111
+ "this.isOpen",
1112
+ "'AutomergeHost is not open'"
1113
+ ]
1114
+ });
970
1115
  for (const documentId of documentIds) {
971
1116
  log3("re-indexing heads for document", {
972
1117
  documentId
973
1118
  }, {
974
1119
  F: __dxlog_file3,
975
- L: 307,
1120
+ L: 361,
976
1121
  S: this,
977
1122
  C: (f, a) => f(...a)
978
1123
  });
@@ -982,7 +1127,7 @@ var AutomergeHost = class extends Resource3 {
982
1127
  documentId
983
1128
  }, {
984
1129
  F: __dxlog_file3,
985
- L: 310,
1130
+ L: 364,
986
1131
  S: this,
987
1132
  C: (f, a) => f(...a)
988
1133
  });
@@ -995,7 +1140,7 @@ var AutomergeHost = class extends Resource3 {
995
1140
  }
996
1141
  log3("done re-indexing heads", void 0, {
997
1142
  F: __dxlog_file3,
998
- L: 319,
1143
+ L: 373,
999
1144
  S: this,
1000
1145
  C: (f, a) => f(...a)
1001
1146
  });
@@ -1005,9 +1150,6 @@ var AutomergeHost = class extends Resource3 {
1005
1150
  // NOTE: If both peers return sharePolicy=false the replication will not happen
1006
1151
  // https://github.com/automerge/automerge-repo/pull/292
1007
1152
  async _sharePolicy(peerId, documentId) {
1008
- if (peerId.startsWith("client-")) {
1009
- return false;
1010
- }
1011
1153
  if (!documentId) {
1012
1154
  return false;
1013
1155
  }
@@ -1056,13 +1198,31 @@ var AutomergeHost = class extends Resource3 {
1056
1198
  * Called by AutomergeStorageAdapter after levelDB batch commit.
1057
1199
  */
1058
1200
  async _afterSave(path) {
1201
+ if (!this.isOpen) {
1202
+ return void 0;
1203
+ }
1059
1204
  this._indexMetadataStore.notifyMarkedDirty();
1060
1205
  const documentId = path[0];
1061
- const document = this._repo.handles[documentId]?.doc();
1062
- if (document) {
1063
- const heads = getHeads(document);
1064
- this._onHeadsChanged(documentId, heads);
1206
+ const handle = this._repo.handles[documentId];
1207
+ if (!handle || !handle.isReady()) {
1208
+ return;
1065
1209
  }
1210
+ const document = handle.doc();
1211
+ if (!document) {
1212
+ return;
1213
+ }
1214
+ const heads = getHeads(document);
1215
+ this._headsUpdates.set(documentId, heads);
1216
+ invariant2(this._onHeadsChangedTask, "onHeadsChangedTask is not initialized", {
1217
+ F: __dxlog_file3,
1218
+ L: 445,
1219
+ S: this,
1220
+ A: [
1221
+ "this._onHeadsChangedTask",
1222
+ "'onHeadsChangedTask is not initialized'"
1223
+ ]
1224
+ });
1225
+ this._onHeadsChangedTask.schedule();
1066
1226
  this.documentsSaved.emit();
1067
1227
  }
1068
1228
  _automergePeers() {
@@ -1099,11 +1259,12 @@ var AutomergeHost = class extends Resource3 {
1099
1259
  * Flush documents to disk.
1100
1260
  */
1101
1261
  async flush({ documentIds } = {}) {
1102
- const loadedDocuments = documentIds?.filter((documentId) => {
1262
+ const loadedDocuments = (documentIds ?? Object.keys(this._repo.handles)).filter((documentId) => {
1103
1263
  const handle = this._repo.handles[documentId];
1104
1264
  return handle && handle.isReady();
1105
1265
  });
1106
1266
  await this._repo.flush(loadedDocuments);
1267
+ await this._onHeadsChangedTask?.runBlocking();
1107
1268
  }
1108
1269
  async getHeads(documentIds) {
1109
1270
  const result = [];
@@ -1155,8 +1316,17 @@ var AutomergeHost = class extends Resource3 {
1155
1316
  missingOnRemote: diff.missingOnRemote.length,
1156
1317
  missingOnLocal: diff.missingOnLocal.length,
1157
1318
  differentDocuments: diff.different.length,
1158
- localDocumentCount: Object.keys(localState.documents).length,
1159
- remoteDocumentCount: Object.keys(state.documents).length
1319
+ localDocumentCount: Object.entries(localState.documents).filter(([_, heads]) => heads.length > 0).length,
1320
+ remoteDocumentCount: Object.entries(state.documents).filter(([_, heads]) => heads.length > 0).length,
1321
+ totalDocumentCount: (/* @__PURE__ */ new Set([
1322
+ ...Object.keys(localState.documents),
1323
+ ...Object.keys(state.documents)
1324
+ ])).size,
1325
+ unsyncedDocumentCount: (/* @__PURE__ */ new Set([
1326
+ ...diff.missingOnLocal,
1327
+ ...diff.missingOnRemote,
1328
+ ...diff.different
1329
+ ])).size
1160
1330
  });
1161
1331
  }
1162
1332
  return result;
@@ -1173,6 +1343,14 @@ var AutomergeHost = class extends Resource3 {
1173
1343
  this._collectionSynchronizer.setLocalCollectionState(collectionId, {
1174
1344
  documents
1175
1345
  });
1346
+ const interestedPeers = this._echoNetworkAdapter.getPeersInterestedInCollection(collectionId);
1347
+ if (interestedPeers.length > 0) {
1348
+ for (const peerId of interestedPeers) {
1349
+ this._sendCollectionState(collectionId, peerId, {
1350
+ documents
1351
+ });
1352
+ }
1353
+ }
1176
1354
  }
1177
1355
  async clearLocalCollectionState(collectionId) {
1178
1356
  this._collectionSynchronizer.clearLocalCollectionState(collectionId);
@@ -1196,42 +1374,231 @@ var AutomergeHost = class extends Resource3 {
1196
1374
  this._collectionSynchronizer.onConnectionClosed(peerId);
1197
1375
  }
1198
1376
  _onRemoteCollectionStateUpdated(collectionId, peerId) {
1377
+ this._collectionsToSync.add({
1378
+ collectionId,
1379
+ peerId
1380
+ });
1381
+ this._syncTask?.schedule();
1382
+ }
1383
+ async _handleCollectionSync(collectionId, peerId) {
1199
1384
  const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
1200
1385
  const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
1201
1386
  if (!localState || !remoteState) {
1202
1387
  return;
1203
1388
  }
1204
1389
  const { different, missingOnLocal, missingOnRemote } = diffCollectionState(localState, remoteState);
1205
- const toReplicate = [
1206
- ...missingOnLocal,
1207
- ...missingOnRemote,
1390
+ if (different.length === 0 && missingOnLocal.length === 0 && missingOnRemote.length === 0) {
1391
+ return;
1392
+ }
1393
+ const toReplicateWithoutBatching = [
1208
1394
  ...different
1209
1395
  ];
1210
- if (toReplicate.length === 0) {
1396
+ const bundleSyncEnabled = this._echoNetworkAdapter.bundleSyncEnabledForPeer(peerId);
1397
+ if (bundleSyncEnabled && missingOnRemote.length >= BUNDLE_SYNC_THRESHOLD) {
1398
+ log3("pushing bundle", {
1399
+ amount: missingOnRemote.length
1400
+ }, {
1401
+ F: __dxlog_file3,
1402
+ L: 650,
1403
+ S: this,
1404
+ C: (f, a) => f(...a)
1405
+ });
1406
+ const { syncInteractively } = await this._pushInBundles(peerId, missingOnRemote);
1407
+ toReplicateWithoutBatching.push(...syncInteractively);
1408
+ } else {
1409
+ log3.verbose("failed to push bundle, replicating interactively", {
1410
+ collectionId,
1411
+ peerId,
1412
+ amount: missingOnRemote.length
1413
+ }, {
1414
+ F: __dxlog_file3,
1415
+ L: 654,
1416
+ S: this,
1417
+ C: (f, a) => f(...a)
1418
+ });
1419
+ toReplicateWithoutBatching.push(...missingOnRemote);
1420
+ }
1421
+ if (bundleSyncEnabled && missingOnLocal.length >= BUNDLE_SYNC_THRESHOLD) {
1422
+ log3("pulling bundle", {
1423
+ amount: missingOnLocal.length
1424
+ }, {
1425
+ F: __dxlog_file3,
1426
+ L: 662,
1427
+ S: this,
1428
+ C: (f, a) => f(...a)
1429
+ });
1430
+ const { syncInteractively } = await this._pullInBundles(peerId, missingOnLocal);
1431
+ toReplicateWithoutBatching.push(...syncInteractively);
1432
+ } else {
1433
+ log3.verbose("failed to pull bundle, replicating interactively", {
1434
+ collectionId,
1435
+ peerId,
1436
+ amount: missingOnLocal.length
1437
+ }, {
1438
+ F: __dxlog_file3,
1439
+ L: 666,
1440
+ S: this,
1441
+ C: (f, a) => f(...a)
1442
+ });
1443
+ toReplicateWithoutBatching.push(...missingOnLocal);
1444
+ }
1445
+ if (toReplicateWithoutBatching.length === 0) {
1211
1446
  return;
1212
1447
  }
1213
1448
  log3("replicating documents after collection sync", {
1214
1449
  collectionId,
1215
1450
  peerId,
1216
- toReplicate,
1217
- count: toReplicate.length
1451
+ toReplicateWithoutBatching,
1452
+ count: toReplicateWithoutBatching.length
1218
1453
  }, {
1219
1454
  F: __dxlog_file3,
1220
- L: 563,
1455
+ L: 678,
1221
1456
  S: this,
1222
1457
  C: (f, a) => f(...a)
1223
1458
  });
1224
- for (const documentId of toReplicate) {
1459
+ for (const documentId of toReplicateWithoutBatching) {
1225
1460
  this._repo.findWithProgress(documentId);
1226
1461
  }
1227
1462
  }
1228
- _onHeadsChanged(documentId, heads) {
1463
+ // TODO(mykola): Add retries of batches https://gist.github.com/mykola-vrmchk/fde270259e9209fcbf1331e5abbf12cf
1464
+ // TODO(mykola): Use effect to retry batches.
1465
+ async _pushInBundles(peerId, documentIds) {
1466
+ const documentsToPush = [
1467
+ ...documentIds
1468
+ ];
1469
+ const syncInteractively = [];
1470
+ while (documentsToPush.length > 0) {
1471
+ await Promise.all(range(BUNDLE_SYNC_CONCURRENCY).map(async () => {
1472
+ const bundle = documentsToPush.splice(0, BUNDLE_SIZE);
1473
+ if (bundle.length === 0) {
1474
+ return;
1475
+ }
1476
+ await this._pushBundle(peerId, bundle).catch((err) => {
1477
+ log3.warn("failed to push bundle, replicating interactively", {
1478
+ peerId,
1479
+ bundle,
1480
+ err
1481
+ }, {
1482
+ F: __dxlog_file3,
1483
+ L: 709,
1484
+ S: this,
1485
+ C: (f, a) => f(...a)
1486
+ });
1487
+ syncInteractively.push(...bundle);
1488
+ });
1489
+ }));
1490
+ }
1491
+ return {
1492
+ syncInteractively
1493
+ };
1494
+ }
1495
+ async _pushBundle(peerId, documentIds) {
1496
+ if (this._ctx.disposed) {
1497
+ return;
1498
+ }
1499
+ const docs = documentIds.map((documentId) => {
1500
+ const handle = this._repo.handles[documentId];
1501
+ if (!handle || !handle.isReady()) {
1502
+ log3.warn("document not ready, skipping", {
1503
+ documentId
1504
+ }, {
1505
+ F: __dxlog_file3,
1506
+ L: 727,
1507
+ S: this,
1508
+ C: (f, a) => f(...a)
1509
+ });
1510
+ return;
1511
+ }
1512
+ const doc = handle.doc();
1513
+ if (!doc) {
1514
+ log3.warn("document not available, skipping", {
1515
+ documentId
1516
+ }, {
1517
+ F: __dxlog_file3,
1518
+ L: 732,
1519
+ S: this,
1520
+ C: (f, a) => f(...a)
1521
+ });
1522
+ return;
1523
+ }
1524
+ return {
1525
+ documentId,
1526
+ data: save(doc),
1527
+ heads: getHeads(doc)
1528
+ };
1529
+ });
1530
+ await this._echoNetworkAdapter.pushBundle(peerId, docs.filter(isNonNullable2));
1531
+ }
1532
+ async _pullInBundles(peerId, documentIds) {
1533
+ const documentsToPull = [
1534
+ ...documentIds
1535
+ ];
1536
+ const syncInteractively = [];
1537
+ const docsToImport = {};
1538
+ while (documentsToPull.length > 0) {
1539
+ await Promise.all(range(BUNDLE_SYNC_CONCURRENCY).map(async () => {
1540
+ const bundle = documentsToPull.splice(0, BUNDLE_SIZE);
1541
+ if (bundle.length === 0) {
1542
+ return;
1543
+ }
1544
+ const result = await this._pullBundle(peerId, bundle).catch((err) => {
1545
+ log3.warn("failed to pull bundle, replicating interactively", {
1546
+ peerId,
1547
+ bundle,
1548
+ err
1549
+ }, {
1550
+ F: __dxlog_file3,
1551
+ L: 762,
1552
+ S: this,
1553
+ C: (f, a) => f(...a)
1554
+ });
1555
+ syncInteractively.push(...bundle);
1556
+ });
1557
+ if (result) {
1558
+ Object.assign(docsToImport, result.docsToImport);
1559
+ }
1560
+ }));
1561
+ }
1562
+ for (const [documentId, data] of Object.entries(docsToImport)) {
1563
+ this._repo.import(data, {
1564
+ docId: documentId
1565
+ });
1566
+ }
1567
+ await this._repo.flush(Object.keys(docsToImport));
1568
+ return {
1569
+ syncInteractively
1570
+ };
1571
+ }
1572
+ async _pullBundle(peerId, documentIds) {
1573
+ if (this._ctx.disposed) {
1574
+ return;
1575
+ }
1576
+ const docHeads = Object.fromEntries(documentIds.map((documentId) => [
1577
+ documentId,
1578
+ []
1579
+ ]));
1580
+ const bundle = await this._echoNetworkAdapter.pullBundle(peerId, docHeads);
1581
+ return {
1582
+ docsToImport: bundle
1583
+ };
1584
+ }
1585
+ _onHeadsChanged(docHeads) {
1229
1586
  const collectionsChanged = /* @__PURE__ */ new Set();
1230
1587
  for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
1231
1588
  const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
1232
- if (state?.documents[documentId]) {
1233
- const newState = structuredClone(state);
1234
- newState.documents[documentId] = heads;
1589
+ if (!state) {
1590
+ continue;
1591
+ }
1592
+ let newState;
1593
+ for (const [documentId, heads] of docHeads) {
1594
+ if (documentId in state.documents) {
1595
+ if (!newState) {
1596
+ newState = structuredClone(state);
1597
+ }
1598
+ newState.documents[documentId] = heads;
1599
+ }
1600
+ }
1601
+ if (newState) {
1235
1602
  this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
1236
1603
  collectionsChanged.add(collectionId);
1237
1604
  }
@@ -1277,7 +1644,7 @@ var changeIsPresentInDoc = (doc, changeHash) => {
1277
1644
  var decodeCollectionState = (state) => {
1278
1645
  invariant2(typeof state === "object" && state !== null, "Invalid state", {
1279
1646
  F: __dxlog_file3,
1280
- L: 614,
1647
+ L: 845,
1281
1648
  S: void 0,
1282
1649
  A: [
1283
1650
  "typeof state === 'object' && state !== null",
@@ -1294,10 +1661,10 @@ var encodeCollectionState = (state) => {
1294
1661
  import { invariant as invariant5 } from "@dxos/invariant";
1295
1662
  import { PublicKey as PublicKey2 } from "@dxos/keys";
1296
1663
  import { log as log5 } from "@dxos/log";
1297
- import { ComplexSet, defaultMap as defaultMap2 } from "@dxos/util";
1664
+ import { ComplexSet as ComplexSet2, defaultMap as defaultMap2 } from "@dxos/util";
1298
1665
 
1299
1666
  // src/automerge/mesh-echo-replicator-connection.ts
1300
- import * as A from "@automerge/automerge";
1667
+ import * as A2 from "@automerge/automerge";
1301
1668
  import { cbor } from "@automerge/automerge-repo";
1302
1669
  import { Resource as Resource4 } from "@dxos/context";
1303
1670
  import { invariant as invariant3 } from "@dxos/invariant";
@@ -1306,8 +1673,15 @@ import { AutomergeReplicator } from "@dxos/teleport-extension-automerge-replicat
1306
1673
  var __dxlog_file4 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
1307
1674
  var DEFAULT_FACTORY = (params) => new AutomergeReplicator(...params);
1308
1675
  var MeshReplicatorConnection = class extends Resource4 {
1676
+ _params;
1677
+ readable;
1678
+ writable;
1679
+ remoteDeviceKey = null;
1680
+ replicatorExtension;
1681
+ _remotePeerId = null;
1682
+ _isEnabled = false;
1309
1683
  constructor(_params) {
1310
- super(), this._params = _params, this.remoteDeviceKey = null, this._remotePeerId = null, this._isEnabled = false;
1684
+ super(), this._params = _params;
1311
1685
  let readableStreamController;
1312
1686
  this.readable = new ReadableStream({
1313
1687
  start: (controller) => {
@@ -1391,6 +1765,9 @@ var MeshReplicatorConnection = class extends Resource4 {
1391
1765
  get isEnabled() {
1392
1766
  return this._isEnabled;
1393
1767
  }
1768
+ get bundleSyncEnabled() {
1769
+ return false;
1770
+ }
1394
1771
  async shouldAdvertise(params) {
1395
1772
  return this._params.shouldAdvertise(params);
1396
1773
  }
@@ -1404,7 +1781,7 @@ var MeshReplicatorConnection = class extends Resource4 {
1404
1781
  enable() {
1405
1782
  invariant3(this._remotePeerId != null, "Remote peer has not connected yet.", {
1406
1783
  F: __dxlog_file4,
1407
- L: 127,
1784
+ L: 131,
1408
1785
  S: this,
1409
1786
  A: [
1410
1787
  "this._remotePeerId != null",
@@ -1422,7 +1799,7 @@ var MeshReplicatorConnection = class extends Resource4 {
1422
1799
  };
1423
1800
  var logSendSync = (message) => {
1424
1801
  log4("sendSyncMessage", () => {
1425
- const decodedSyncMessage = message.type === "sync" && message.data ? A.decodeSyncMessage(message.data) : void 0;
1802
+ const decodedSyncMessage = message.type === "sync" && message.data ? A2.decodeSyncMessage(message.data) : void 0;
1426
1803
  return {
1427
1804
  sync: decodedSyncMessage && {
1428
1805
  headsLength: decodedSyncMessage.heads.length,
@@ -1435,7 +1812,7 @@ var logSendSync = (message) => {
1435
1812
  };
1436
1813
  }, {
1437
1814
  F: __dxlog_file4,
1438
- L: 140,
1815
+ L: 144,
1439
1816
  S: void 0,
1440
1817
  C: (f, a) => f(...a)
1441
1818
  });
@@ -1463,24 +1840,22 @@ var getSpaceIdFromCollectionId = (collectionId) => {
1463
1840
  // src/automerge/mesh-echo-replicator.ts
1464
1841
  var __dxlog_file6 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts";
1465
1842
  var MeshEchoReplicator = class {
1466
- constructor() {
1467
- /**
1468
- * We might have multiple connections open with a peer (one per space), but there'll be only one enabled
1469
- * connection at any given moment, because there's a single repo for all the spaces.
1470
- * When a connection closes (space was closed) it gets removed from the list and the next connection
1471
- * in the line gets enabled.
1472
- */
1473
- this._connectionsPerPeer = /* @__PURE__ */ new Map();
1474
- /**
1475
- * A set of all connections (enabled and disabled).
1476
- */
1477
- this._connections = /* @__PURE__ */ new Set();
1478
- /**
1479
- * spaceId -> deviceKey[]
1480
- */
1481
- this._authorizedDevices = /* @__PURE__ */ new Map();
1482
- this._context = null;
1483
- }
1843
+ /**
1844
+ * We might have multiple connections open with a peer (one per space), but there'll be only one enabled
1845
+ * connection at any given moment, because there's a single repo for all the spaces.
1846
+ * When a connection closes (space was closed) it gets removed from the list and the next connection
1847
+ * in the line gets enabled.
1848
+ */
1849
+ _connectionsPerPeer = /* @__PURE__ */ new Map();
1850
+ /**
1851
+ * A set of all connections (enabled and disabled).
1852
+ */
1853
+ _connections = /* @__PURE__ */ new Set();
1854
+ /**
1855
+ * spaceId -> deviceKey[]
1856
+ */
1857
+ _authorizedDevices = /* @__PURE__ */ new Map();
1858
+ _context = null;
1484
1859
  async connect(context) {
1485
1860
  this._context = context;
1486
1861
  }
@@ -1500,7 +1875,7 @@ var MeshEchoReplicator = class {
1500
1875
  createExtension(extensionFactory) {
1501
1876
  invariant5(this._context, void 0, {
1502
1877
  F: __dxlog_file6,
1503
- L: 67,
1878
+ L: 68,
1504
1879
  S: this,
1505
1880
  A: [
1506
1881
  "this._context",
@@ -1515,13 +1890,13 @@ var MeshEchoReplicator = class {
1515
1890
  peerId: connection.peerId
1516
1891
  }, {
1517
1892
  F: __dxlog_file6,
1518
- L: 73,
1893
+ L: 74,
1519
1894
  S: this,
1520
1895
  C: (f, a) => f(...a)
1521
1896
  });
1522
1897
  invariant5(this._context, void 0, {
1523
1898
  F: __dxlog_file6,
1524
- L: 74,
1899
+ L: 75,
1525
1900
  S: this,
1526
1901
  A: [
1527
1902
  "this._context",
@@ -1546,7 +1921,7 @@ var MeshEchoReplicator = class {
1546
1921
  peerId: connection.peerId
1547
1922
  }, {
1548
1923
  F: __dxlog_file6,
1549
- L: 88,
1924
+ L: 89,
1550
1925
  S: this,
1551
1926
  C: (f, a) => f(...a)
1552
1927
  });
@@ -1558,7 +1933,7 @@ var MeshEchoReplicator = class {
1558
1933
  peerId: connection.peerId
1559
1934
  }, {
1560
1935
  F: __dxlog_file6,
1561
- L: 96,
1936
+ L: 97,
1562
1937
  S: this,
1563
1938
  C: (f, a) => f(...a)
1564
1939
  });
@@ -1580,13 +1955,13 @@ var MeshEchoReplicator = class {
1580
1955
  documentId: params.documentId
1581
1956
  }, {
1582
1957
  F: __dxlog_file6,
1583
- L: 114,
1958
+ L: 115,
1584
1959
  S: this,
1585
1960
  C: (f, a) => f(...a)
1586
1961
  });
1587
1962
  invariant5(this._context, void 0, {
1588
1963
  F: __dxlog_file6,
1589
- L: 115,
1964
+ L: 116,
1590
1965
  S: this,
1591
1966
  A: [
1592
1967
  "this._context",
@@ -1606,7 +1981,7 @@ var MeshEchoReplicator = class {
1606
1981
  acceptDocument: remoteDocumentExists
1607
1982
  }, {
1608
1983
  F: __dxlog_file6,
1609
- L: 123,
1984
+ L: 124,
1610
1985
  S: this,
1611
1986
  C: (f, a) => f(...a)
1612
1987
  });
@@ -1620,7 +1995,7 @@ var MeshEchoReplicator = class {
1620
1995
  documentId: params.documentId
1621
1996
  }, {
1622
1997
  F: __dxlog_file6,
1623
- L: 140,
1998
+ L: 141,
1624
1999
  S: this,
1625
2000
  C: (f, a) => f(...a)
1626
2001
  });
@@ -1636,7 +2011,7 @@ var MeshEchoReplicator = class {
1636
2011
  isAuthorized
1637
2012
  }, {
1638
2013
  F: __dxlog_file6,
1639
- L: 148,
2014
+ L: 149,
1640
2015
  S: this,
1641
2016
  C: (f, a) => f(...a)
1642
2017
  });
@@ -1644,7 +2019,7 @@ var MeshEchoReplicator = class {
1644
2019
  } catch (err) {
1645
2020
  log5.catch(err, void 0, {
1646
2021
  F: __dxlog_file6,
1647
- L: 158,
2022
+ L: 159,
1648
2023
  S: this,
1649
2024
  C: (f, a) => f(...a)
1650
2025
  });
@@ -1660,7 +2035,7 @@ var MeshEchoReplicator = class {
1660
2035
  collectionId
1661
2036
  }, {
1662
2037
  F: __dxlog_file6,
1663
- L: 168,
2038
+ L: 169,
1664
2039
  S: this,
1665
2040
  C: (f, a) => f(...a)
1666
2041
  });
@@ -1679,12 +2054,12 @@ var MeshEchoReplicator = class {
1679
2054
  deviceKey
1680
2055
  }, {
1681
2056
  F: __dxlog_file6,
1682
- L: 185,
2057
+ L: 186,
1683
2058
  S: this,
1684
2059
  C: (f, a) => f(...a)
1685
2060
  });
1686
2061
  const spaceId = await createIdFromSpaceKey(spaceKey);
1687
- defaultMap2(this._authorizedDevices, spaceId, () => new ComplexSet(PublicKey2.hash)).add(deviceKey);
2062
+ defaultMap2(this._authorizedDevices, spaceId, () => new ComplexSet2(PublicKey2.hash)).add(deviceKey);
1688
2063
  for (const connection of this._connections) {
1689
2064
  if (connection.isEnabled && connection.remoteDeviceKey && connection.remoteDeviceKey.equals(deviceKey)) {
1690
2065
  if (this._connectionsPerPeer.has(connection.peerId)) {
@@ -1697,7 +2072,7 @@ var MeshEchoReplicator = class {
1697
2072
 
1698
2073
  // src/automerge/echo-data-monitor.ts
1699
2074
  import { trace as trace3 } from "@dxos/tracing";
1700
- import { CircularBuffer, mapValues, SlidingWindowSummary } from "@dxos/util";
2075
+ import { CircularBuffer, SlidingWindowSummary, mapValues } from "@dxos/util";
1701
2076
  function _ts_decorate4(decorators, target, key, desc) {
1702
2077
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1703
2078
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
@@ -1707,19 +2082,21 @@ function _ts_decorate4(decorators, target, key, desc) {
1707
2082
  var PER_SECOND_RATE_AVG_WINDOW_SIZE = 5;
1708
2083
  var DEFAULT_AVG_WINDOW_SIZE = 25;
1709
2084
  var EchoDataMonitor = class {
2085
+ _params;
2086
+ _lastTick = 0;
2087
+ _activeCounters = createLocalCounters();
2088
+ _lastCompleteCounters;
2089
+ _localTimeSeries = createLocalTimeSeries();
2090
+ _storageAverages = createStorageAverages();
2091
+ _replicationAverages = createNetworkAverages();
2092
+ _sizeByMessage = {};
2093
+ _lastReceivedMessages = new CircularBuffer(100);
2094
+ _lastSentMessages = new CircularBuffer(100);
2095
+ _connectionsCount = 0;
1710
2096
  constructor(_params = {
1711
2097
  timeSeriesLength: 30
1712
2098
  }) {
1713
2099
  this._params = _params;
1714
- this._lastTick = 0;
1715
- this._activeCounters = createLocalCounters();
1716
- this._localTimeSeries = createLocalTimeSeries();
1717
- this._storageAverages = createStorageAverages();
1718
- this._replicationAverages = createNetworkAverages();
1719
- this._sizeByMessage = {};
1720
- this._lastReceivedMessages = new CircularBuffer(100);
1721
- this._lastSentMessages = new CircularBuffer(100);
1722
- this._connectionsCount = 0;
1723
2100
  }
1724
2101
  tick(timeMs) {
1725
2102
  this._advanceTimeWindow(timeMs - this._lastTick);
@@ -2057,19 +2434,37 @@ var getByteCount = (message) => {
2057
2434
  };
2058
2435
 
2059
2436
  // src/db-host/documents-synchronizer.ts
2437
+ import { next as A3 } from "@automerge/automerge";
2438
+ import { UpdateScheduler, sleep } from "@dxos/async";
2439
+ import { LifecycleState as LifecycleState2, Resource as Resource5, cancelWithContext as cancelWithContext2 } from "@dxos/context";
2440
+ import { invariant as invariant6 } from "@dxos/invariant";
2441
+ import { log as log6 } from "@dxos/log";
2060
2442
  var __dxlog_file7 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/documents-synchronizer.ts";
2061
2443
  var MAX_UPDATE_FREQ = 10;
2444
+ var WRAP_AROUND_RETRY_LIMIT = 3;
2445
+ var WRAP_AROUND_RETRY_INITIAL_DELAY = 100;
2062
2446
  var DocumentsSynchronizer = class extends Resource5 {
2447
+ _params;
2448
+ _syncStates = /* @__PURE__ */ new Map();
2449
+ /**
2450
+ * Documents that have pending updates.
2451
+ * Used to batch updates.
2452
+ */
2453
+ _pendingUpdates = /* @__PURE__ */ new Set();
2454
+ /**
2455
+ * Job that schedules if there are pending updates.
2456
+ */
2457
+ _sendUpdatesJob = void 0;
2063
2458
  constructor(_params) {
2064
- super(), this._params = _params, this._syncStates = /* @__PURE__ */ new Map(), this._pendingUpdates = /* @__PURE__ */ new Set(), this._sendUpdatesJob = void 0;
2459
+ super(), this._params = _params;
2065
2460
  }
2066
- addDocuments(documentIds, retryCounter = 0) {
2067
- if (retryCounter > 3) {
2461
+ addDocuments(documentIds, retryCounter = 0, wrapAroundRetryDelay = WRAP_AROUND_RETRY_INITIAL_DELAY) {
2462
+ if (retryCounter > WRAP_AROUND_RETRY_LIMIT) {
2068
2463
  log6.warn("Failed to load document, retry limit reached", {
2069
2464
  documentIds
2070
2465
  }, {
2071
2466
  F: __dxlog_file7,
2072
- L: 52,
2467
+ L: 57,
2073
2468
  S: this,
2074
2469
  C: (f, a) => f(...a)
2075
2470
  });
@@ -2087,13 +2482,13 @@ var DocumentsSynchronizer = class extends Resource5 {
2087
2482
  error
2088
2483
  }, {
2089
2484
  F: __dxlog_file7,
2090
- L: 66,
2485
+ L: 71,
2091
2486
  S: this,
2092
2487
  C: (f, a) => f(...a)
2093
2488
  });
2094
- this.addDocuments([
2489
+ void cancelWithContext2(this._ctx, sleep(wrapAroundRetryDelay)).then(() => this.addDocuments([
2095
2490
  documentId
2096
- ], retryCounter + 1);
2491
+ ], retryCounter + 1, wrapAroundRetryDelay * 2));
2097
2492
  });
2098
2493
  }
2099
2494
  }
@@ -2115,14 +2510,9 @@ var DocumentsSynchronizer = class extends Resource5 {
2115
2510
  }
2116
2511
  async update(updates) {
2117
2512
  for (const { documentId, mutation, isNew } of updates) {
2118
- if (isNew) {
2119
- const doc = await this._params.repo.find(documentId, FIND_PARAMS);
2120
- doc.update((doc2) => A2.loadIncremental(doc2, mutation));
2121
- this._startSync(doc);
2122
- } else {
2123
- this._writeMutation(documentId, mutation);
2124
- }
2513
+ this._writeMutation(documentId, mutation, isNew);
2125
2514
  }
2515
+ await this._params.repo.flush(updates.map(({ documentId }) => documentId));
2126
2516
  }
2127
2517
  _startSync(doc) {
2128
2518
  if (this._syncStates.has(doc.documentId)) {
@@ -2130,7 +2520,7 @@ var DocumentsSynchronizer = class extends Resource5 {
2130
2520
  documentId: doc.documentId
2131
2521
  }, {
2132
2522
  F: __dxlog_file7,
2133
- L: 105,
2523
+ L: 108,
2134
2524
  S: this,
2135
2525
  C: (f, a) => f(...a)
2136
2526
  });
@@ -2141,6 +2531,7 @@ var DocumentsSynchronizer = class extends Resource5 {
2141
2531
  };
2142
2532
  this._subscribeForChanges(syncState);
2143
2533
  this._syncStates.set(doc.documentId, syncState);
2534
+ return syncState;
2144
2535
  }
2145
2536
  _subscribeForChanges(syncState) {
2146
2537
  const handler = () => {
@@ -2173,7 +2564,7 @@ var DocumentsSynchronizer = class extends Resource5 {
2173
2564
  const syncState = this._syncStates.get(documentId);
2174
2565
  invariant6(syncState, "Sync state for document not found", {
2175
2566
  F: __dxlog_file7,
2176
- L: 146,
2567
+ L: 150,
2177
2568
  S: this,
2178
2569
  A: [
2179
2570
  "syncState",
@@ -2185,44 +2576,57 @@ var DocumentsSynchronizer = class extends Resource5 {
2185
2576
  return;
2186
2577
  }
2187
2578
  const doc = handle.doc();
2188
- const mutation = syncState.lastSentHead ? A2.saveSince(doc, syncState.lastSentHead) : A2.save(doc);
2579
+ const mutation = syncState.lastSentHead ? A3.saveSince(doc, syncState.lastSentHead) : A3.save(doc);
2189
2580
  if (mutation.length === 0) {
2190
2581
  return;
2191
2582
  }
2192
- syncState.lastSentHead = A2.getHeads(doc);
2583
+ syncState.lastSentHead = A3.getHeads(doc);
2193
2584
  return mutation;
2194
2585
  }
2195
- _writeMutation(documentId, mutation) {
2196
- const syncState = this._syncStates.get(documentId);
2197
- invariant6(syncState, "Sync state for document not found", {
2198
- F: __dxlog_file7,
2199
- L: 162,
2200
- S: this,
2201
- A: [
2202
- "syncState",
2203
- "'Sync state for document not found'"
2204
- ]
2205
- });
2206
- syncState.handle.update((doc) => {
2207
- const headsBefore = A2.getHeads(doc);
2208
- const newDoc = A2.loadIncremental(doc, mutation);
2209
- if (A2.equals(headsBefore, syncState.lastSentHead)) {
2210
- syncState.lastSentHead = A2.getHeads(newDoc);
2586
+ _writeMutation(documentId, mutation, isNew) {
2587
+ if (this._lifecycleState === LifecycleState2.CLOSED) {
2588
+ return;
2589
+ }
2590
+ if (isNew) {
2591
+ const newHandle = this._params.repo.import(mutation, {
2592
+ docId: documentId
2593
+ });
2594
+ const syncState = this._startSync(newHandle);
2595
+ syncState.lastSentHead = A3.getHeads(newHandle.doc());
2596
+ } else {
2597
+ const syncState = this._syncStates.get(documentId);
2598
+ invariant6(syncState, "Sync state for document not found", {
2599
+ F: __dxlog_file7,
2600
+ L: 174,
2601
+ S: this,
2602
+ A: [
2603
+ "syncState",
2604
+ "'Sync state for document not found'"
2605
+ ]
2606
+ });
2607
+ const headsBefore = A3.getHeads(syncState.handle.doc());
2608
+ this._params.repo.import(mutation, {
2609
+ docId: documentId
2610
+ });
2611
+ if (A3.equals(headsBefore, syncState.lastSentHead)) {
2612
+ syncState.lastSentHead = A3.getHeads(syncState.handle.doc());
2211
2613
  }
2212
- return newDoc;
2213
- });
2614
+ }
2214
2615
  }
2215
2616
  };
2216
2617
 
2217
2618
  // src/db-host/data-service.ts
2218
2619
  var __dxlog_file8 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
2219
2620
  var DataServiceImpl = class {
2621
+ /**
2622
+ * Map of subscriptions.
2623
+ * subscriptionId -> DocumentsSynchronizer
2624
+ */
2625
+ _subscriptions = /* @__PURE__ */ new Map();
2626
+ _automergeHost;
2627
+ _spaceStateManager;
2628
+ _updateIndexes;
2220
2629
  constructor(params) {
2221
- /**
2222
- * Map of subscriptions.
2223
- * subscriptionId -> DocumentsSynchronizer
2224
- */
2225
- this._subscriptions = /* @__PURE__ */ new Map();
2226
2630
  this._automergeHost = params.automergeHost;
2227
2631
  this._spaceStateManager = params.spaceStateManager;
2228
2632
  this._updateIndexes = params.updateIndexes;
@@ -2238,7 +2642,7 @@ var DataServiceImpl = class {
2238
2642
  ready();
2239
2643
  }).catch((err) => log7.catch(err, void 0, {
2240
2644
  F: __dxlog_file8,
2241
- L: 71,
2645
+ L: 72,
2242
2646
  S: this,
2243
2647
  C: (f, a) => f(...a)
2244
2648
  }));
@@ -2249,7 +2653,7 @@ var DataServiceImpl = class {
2249
2653
  const synchronizer = this._subscriptions.get(request.subscriptionId);
2250
2654
  invariant7(synchronizer, "Subscription not found", {
2251
2655
  F: __dxlog_file8,
2252
- L: 78,
2656
+ L: 79,
2253
2657
  S: this,
2254
2658
  A: [
2255
2659
  "synchronizer",
@@ -2270,7 +2674,7 @@ var DataServiceImpl = class {
2270
2674
  const synchronizer = this._subscriptions.get(request.subscriptionId);
2271
2675
  invariant7(synchronizer, "Subscription not found", {
2272
2676
  F: __dxlog_file8,
2273
- L: 93,
2677
+ L: 94,
2274
2678
  S: this,
2275
2679
  A: [
2276
2680
  "synchronizer",
@@ -2315,7 +2719,7 @@ var DataServiceImpl = class {
2315
2719
  const spaceId = request.spaceId;
2316
2720
  invariant7(SpaceId2.isValid(spaceId), void 0, {
2317
2721
  F: __dxlog_file8,
2318
- L: 133,
2722
+ L: 134,
2319
2723
  S: this,
2320
2724
  A: [
2321
2725
  "SpaceId.isValid(spaceId)",
@@ -2336,14 +2740,7 @@ var DataServiceImpl = class {
2336
2740
  peers: []
2337
2741
  };
2338
2742
  next({
2339
- peers: state.peers.map((peer) => ({
2340
- peerId: peer.peerId,
2341
- missingOnRemote: peer.missingOnRemote,
2342
- missingOnLocal: peer.missingOnLocal,
2343
- differentDocuments: peer.differentDocuments,
2344
- localDocumentCount: peer.localDocumentCount,
2345
- remoteDocumentCount: peer.remoteDocumentCount
2346
- }))
2743
+ peers: state.peers
2347
2744
  });
2348
2745
  });
2349
2746
  this._automergeHost.collectionStateUpdated.on(ctx, (e) => {
@@ -2359,14 +2756,14 @@ var DataServiceImpl = class {
2359
2756
  // src/db-host/echo-host.ts
2360
2757
  import { LifecycleState as LifecycleState5, Resource as Resource9 } from "@dxos/context";
2361
2758
  import { todo } from "@dxos/debug";
2362
- import { createIdFromSpaceKey as createIdFromSpaceKey2, SpaceDocVersion as SpaceDocVersion3 } from "@dxos/echo-protocol";
2759
+ import { SpaceDocVersion as SpaceDocVersion3, createIdFromSpaceKey as createIdFromSpaceKey2 } from "@dxos/echo-protocol";
2363
2760
  import { IndexMetadataStore, IndexStore, Indexer } from "@dxos/indexing";
2364
2761
  import { invariant as invariant13 } from "@dxos/invariant";
2365
2762
  import { IndexKind } from "@dxos/protocols/proto/dxos/echo/indexing";
2366
2763
  import { trace as trace5 } from "@dxos/tracing";
2367
2764
 
2368
2765
  // src/db-host/documents-iterator.ts
2369
- import * as A3 from "@automerge/automerge";
2766
+ import * as A4 from "@automerge/automerge";
2370
2767
  import { Context as Context2 } from "@dxos/context";
2371
2768
  import { DatabaseDirectory as DatabaseDirectory2, SpaceDocVersion } from "@dxos/echo-protocol";
2372
2769
  import { invariant as invariant8 } from "@dxos/invariant";
@@ -2397,10 +2794,10 @@ var createSelectedDocumentsIterator = (automergeHost) => (
2397
2794
  ""
2398
2795
  ]
2399
2796
  });
2400
- const currentHeads = A3.getHeads(doc);
2401
- if (!A3.equals(currentHeads, heads)) {
2797
+ const currentHeads = A4.getHeads(doc);
2798
+ if (!A4.equals(currentHeads, heads)) {
2402
2799
  const begin = Date.now();
2403
- doc = A3.view(doc, heads);
2800
+ doc = A4.view(doc, heads);
2404
2801
  const end = Date.now();
2405
2802
  if (end - begin > LOG_VIEW_OPERATION_THRESHOLD) {
2406
2803
  log8("Checking out document version is taking too long", {
@@ -2455,8 +2852,8 @@ var createSelectedDocumentsIterator = (automergeHost) => (
2455
2852
 
2456
2853
  // src/db-host/query-service.ts
2457
2854
  import { getHeads as getHeads3 } from "@automerge/automerge";
2458
- import { Schema } from "effect";
2459
- import { DeferredTask, scheduleMicroTask, synchronized as synchronized2 } from "@dxos/async";
2855
+ import * as Schema from "effect/Schema";
2856
+ import { DeferredTask as DeferredTask2, scheduleMicroTask, synchronized as synchronized2 } from "@dxos/async";
2460
2857
  import { Stream as Stream2 } from "@dxos/codec-protobuf/stream";
2461
2858
  import { Context as Context4, Resource as Resource7 } from "@dxos/context";
2462
2859
  import { raise } from "@dxos/debug";
@@ -2466,23 +2863,27 @@ import { objectPointerCodec as objectPointerCodec4 } from "@dxos/protocols";
2466
2863
  import { trace as trace4 } from "@dxos/tracing";
2467
2864
 
2468
2865
  // src/query/query-executor.ts
2469
- import { Match } from "effect";
2866
+ import * as Match from "effect/Match";
2867
+ import * as Predicate from "effect/Predicate";
2470
2868
  import { Context as Context3, ContextDisposedError, LifecycleState as LifecycleState3, Resource as Resource6 } from "@dxos/context";
2471
- import { DatabaseDirectory as DatabaseDirectory3, isEncodedReference, ObjectStructure } from "@dxos/echo-protocol";
2869
+ import { DatabaseDirectory as DatabaseDirectory3, ObjectStructure, isEncodedReference } from "@dxos/echo-protocol";
2472
2870
  import { EscapedPropPath } from "@dxos/indexing";
2473
2871
  import { invariant as invariant10 } from "@dxos/invariant";
2474
2872
  import { DXN, PublicKey as PublicKey3 } from "@dxos/keys";
2475
2873
  import { log as log9 } from "@dxos/log";
2476
2874
  import { objectPointerCodec as objectPointerCodec3 } from "@dxos/protocols";
2477
- import { getDeep, isNonNullable as isNonNullable2 } from "@dxos/util";
2875
+ import { getDeep, isNonNullable as isNonNullable3 } from "@dxos/util";
2478
2876
 
2479
2877
  // src/query/query-planner.ts
2878
+ import { Order } from "@dxos/echo";
2480
2879
  import { invariant as invariant9 } from "@dxos/invariant";
2481
2880
 
2482
2881
  // src/query/errors.ts
2483
2882
  import { BaseError } from "@dxos/errors";
2484
2883
  var QueryError = class extends BaseError.extend("QUERY_ERROR") {
2485
2884
  };
2885
+ var InvalidQueryError = class extends QueryError.extend("INVALID_QUERY") {
2886
+ };
2486
2887
 
2487
2888
  // src/query/plan.ts
2488
2889
  (function(QueryPlan2) {
@@ -2511,6 +2912,7 @@ var DEFAULT_OPTIONS = {
2511
2912
  defaultTextSearchKind: "full-text"
2512
2913
  };
2513
2914
  var QueryPlanner = class {
2915
+ _options;
2514
2916
  constructor(options) {
2515
2917
  this._options = {
2516
2918
  ...DEFAULT_OPTIONS,
@@ -2524,6 +2926,7 @@ var QueryPlanner = class {
2524
2926
  });
2525
2927
  plan = this._optimizeEmptyFilters(plan);
2526
2928
  plan = this._optimizeSoloUnions(plan);
2929
+ plan = this._ensureOrderStep(plan);
2527
2930
  return plan;
2528
2931
  }
2529
2932
  _generate(query, context) {
@@ -2546,8 +2949,11 @@ var QueryPlanner = class {
2546
2949
  return this._generateUnionClause(query, context);
2547
2950
  case "set-difference":
2548
2951
  return this._generateSetDifferenceClause(query, context);
2952
+ case "order":
2953
+ return this._generateOrderClause(query, context);
2549
2954
  default:
2550
- throw new QueryError(`Unsupported query type: ${query.type}`, {
2955
+ throw new QueryError({
2956
+ message: `Unsupported query type: ${query.type}`,
2551
2957
  context: {
2552
2958
  query: context.originalQuery
2553
2959
  }
@@ -2571,10 +2977,11 @@ var QueryPlanner = class {
2571
2977
  }
2572
2978
  // TODO(dmaretskyi): This can be rewritten as a function of (filter[]) -> (selection ? undefined, rest: filter[]) that recurses onto itself.
2573
2979
  // TODO(dmaretskyi): If the tip of the query ast is a [select, ...filter] shape we can reorder the filters so the query is most efficient.
2574
- _generateSelectionFromFilter(filter, context) {
2575
- switch (filter.type) {
2980
+ _generateSelectionFromFilter(filter2, context) {
2981
+ switch (filter2.type) {
2982
+ // Props
2576
2983
  case "object": {
2577
- if (context.selectionInverted && filter.id === void 0 && filter.typename === null && Object.keys(filter.props).length === 0) {
2984
+ if (context.selectionInverted && filter2.id === void 0 && filter2.typename === null && Object.keys(filter2.props).length === 0) {
2578
2985
  return QueryPlan.Plan.make([
2579
2986
  {
2580
2987
  _tag: "ClearWorkingSetStep"
@@ -2583,32 +2990,33 @@ var QueryPlanner = class {
2583
2990
  ]);
2584
2991
  }
2585
2992
  if (context.selectionInverted) {
2586
- throw new QueryError("Query too complex", {
2993
+ throw new QueryError({
2994
+ message: "Query too complex",
2587
2995
  context: {
2588
2996
  query: context.originalQuery
2589
2997
  }
2590
2998
  });
2591
2999
  }
2592
- if (filter.id && filter.id?.length > 0) {
3000
+ if (filter2.id && filter2.id?.length > 0) {
2593
3001
  return QueryPlan.Plan.make([
2594
3002
  {
2595
3003
  _tag: "SelectStep",
2596
3004
  spaces: context.selectionSpaces,
2597
3005
  selector: {
2598
3006
  _tag: "IdSelector",
2599
- objectIds: filter.id
3007
+ objectIds: filter2.id
2600
3008
  }
2601
3009
  },
2602
3010
  ...this._generateDeletedHandlingSteps(context),
2603
3011
  {
2604
3012
  _tag: "FilterStep",
2605
3013
  filter: {
2606
- ...filter,
3014
+ ...filter2,
2607
3015
  id: void 0
2608
3016
  }
2609
3017
  }
2610
3018
  ]);
2611
- } else if (filter.typename) {
3019
+ } else if (filter2.typename) {
2612
3020
  return QueryPlan.Plan.make([
2613
3021
  {
2614
3022
  _tag: "SelectStep",
@@ -2616,7 +3024,7 @@ var QueryPlanner = class {
2616
3024
  selector: {
2617
3025
  _tag: "TypeSelector",
2618
3026
  typename: [
2619
- filter.typename
3027
+ filter2.typename
2620
3028
  ],
2621
3029
  inverted: false
2622
3030
  }
@@ -2625,7 +3033,7 @@ var QueryPlanner = class {
2625
3033
  {
2626
3034
  _tag: "FilterStep",
2627
3035
  filter: {
2628
- ...filter,
3036
+ ...filter2,
2629
3037
  typename: null
2630
3038
  }
2631
3039
  }
@@ -2643,12 +3051,32 @@ var QueryPlanner = class {
2643
3051
  {
2644
3052
  _tag: "FilterStep",
2645
3053
  filter: {
2646
- ...filter
3054
+ ...filter2
2647
3055
  }
2648
3056
  }
2649
3057
  ]);
2650
3058
  }
2651
3059
  }
3060
+ // Tag
3061
+ case "tag": {
3062
+ return QueryPlan.Plan.make([
3063
+ {
3064
+ _tag: "SelectStep",
3065
+ spaces: context.selectionSpaces,
3066
+ selector: {
3067
+ _tag: "WildcardSelector"
3068
+ }
3069
+ },
3070
+ ...this._generateDeletedHandlingSteps(context),
3071
+ {
3072
+ _tag: "FilterStep",
3073
+ filter: {
3074
+ ...filter2
3075
+ }
3076
+ }
3077
+ ]);
3078
+ }
3079
+ // Text
2652
3080
  case "text-search": {
2653
3081
  return QueryPlan.Plan.make([
2654
3082
  {
@@ -2656,55 +3084,61 @@ var QueryPlanner = class {
2656
3084
  spaces: context.selectionSpaces,
2657
3085
  selector: {
2658
3086
  _tag: "TextSelector",
2659
- text: filter.text,
2660
- searchKind: filter.searchKind ?? this._options.defaultTextSearchKind
3087
+ text: filter2.text,
3088
+ searchKind: filter2.searchKind ?? this._options.defaultTextSearchKind
2661
3089
  }
2662
3090
  },
2663
3091
  ...this._generateDeletedHandlingSteps(context)
2664
3092
  ]);
2665
3093
  }
3094
+ // Compare
2666
3095
  case "compare":
2667
- throw new QueryError("Query too complex", {
3096
+ throw new QueryError({
3097
+ message: "Query too complex",
2668
3098
  context: {
2669
3099
  query: context.originalQuery
2670
3100
  }
2671
3101
  });
2672
3102
  case "in":
2673
- throw new QueryError("Query too complex", {
3103
+ throw new QueryError({
3104
+ message: "Query too complex",
2674
3105
  context: {
2675
3106
  query: context.originalQuery
2676
3107
  }
2677
3108
  });
2678
3109
  case "range":
2679
- throw new QueryError("Query too complex", {
3110
+ throw new QueryError({
3111
+ message: "Query too complex",
2680
3112
  context: {
2681
3113
  query: context.originalQuery
2682
3114
  }
2683
3115
  });
3116
+ // Boolean
2684
3117
  case "not":
2685
- return this._generateSelectionFromFilter(filter.filter, {
3118
+ return this._generateSelectionFromFilter(filter2.filter, {
2686
3119
  ...context,
2687
3120
  selectionInverted: !context.selectionInverted
2688
3121
  });
2689
3122
  case "and":
2690
- throw new QueryError("Query too complex", {
3123
+ throw new QueryError({
3124
+ message: "Query too complex",
2691
3125
  context: {
2692
3126
  query: context.originalQuery
2693
3127
  }
2694
3128
  });
2695
3129
  case "or":
2696
- if (filter.filters.every(isTrivialTypenameFilter)) {
2697
- const typenames = filter.filters.map((f) => {
2698
- invariant9(f.type === "object" && f.typename !== null, void 0, {
3130
+ if (filter2.filters.every(isTrivialTypenameFilter)) {
3131
+ const typenames = filter2.filters.map((filter3) => {
3132
+ invariant9(filter3.type === "object" && filter3.typename !== null, void 0, {
2699
3133
  F: __dxlog_file10,
2700
- L: 191,
3134
+ L: 224,
2701
3135
  S: this,
2702
3136
  A: [
2703
- "f.type === 'object' && f.typename !== null",
3137
+ "filter.type === 'object' && filter.typename !== null",
2704
3138
  ""
2705
3139
  ]
2706
3140
  });
2707
- return f.typename;
3141
+ return filter3.typename;
2708
3142
  });
2709
3143
  return QueryPlan.Plan.make([
2710
3144
  {
@@ -2719,14 +3153,16 @@ var QueryPlanner = class {
2719
3153
  ...this._generateDeletedHandlingSteps(context)
2720
3154
  ]);
2721
3155
  } else {
2722
- throw new QueryError("Query too complex", {
3156
+ throw new QueryError({
3157
+ message: "Query too complex",
2723
3158
  context: {
2724
3159
  query: context.originalQuery
2725
3160
  }
2726
3161
  });
2727
3162
  }
2728
3163
  default:
2729
- throw new QueryError(`Unsupported filter type: ${filter.type}`, {
3164
+ throw new QueryError({
3165
+ message: `Unsupported filter type: ${filter2.type}`,
2730
3166
  context: {
2731
3167
  query: context.originalQuery
2732
3168
  }
@@ -2926,6 +3362,45 @@ var QueryPlanner = class {
2926
3362
  _optimizeSoloUnions(plan) {
2927
3363
  return plan;
2928
3364
  }
3365
+ _generateOrderClause(query, context) {
3366
+ return QueryPlan.Plan.make([
3367
+ ...this._generate(query.query, context).steps,
3368
+ {
3369
+ _tag: "OrderStep",
3370
+ order: query.order
3371
+ }
3372
+ ]);
3373
+ }
3374
+ // After complete plan is built, inspect it from the end:
3375
+ // - Walk backwards until hitting an object set changer.
3376
+ // - If an order step is found, skip.
3377
+ // - Otherwise append natural order to the end.
3378
+ _ensureOrderStep(plan) {
3379
+ const OBJECT_SET_CHANGERS = /* @__PURE__ */ new Set([
3380
+ "SelectStep",
3381
+ "TraverseStep",
3382
+ "UnionStep",
3383
+ "SetDifferenceStep"
3384
+ ]);
3385
+ for (let i = plan.steps.length - 1; i >= 0; i--) {
3386
+ const step = plan.steps[i];
3387
+ if (step._tag === "OrderStep") {
3388
+ return plan;
3389
+ }
3390
+ if (OBJECT_SET_CHANGERS.has(step._tag)) {
3391
+ break;
3392
+ }
3393
+ }
3394
+ return QueryPlan.Plan.make([
3395
+ ...plan.steps,
3396
+ {
3397
+ _tag: "OrderStep",
3398
+ order: [
3399
+ Order.natural.ast
3400
+ ]
3401
+ }
3402
+ ]);
3403
+ }
2929
3404
  };
2930
3405
  var DEFAULT_CONTEXT = {
2931
3406
  originalQuery: null,
@@ -2946,8 +3421,8 @@ var createRelationTraversalStep = (direction) => ({
2946
3421
  direction
2947
3422
  }
2948
3423
  });
2949
- var isTrivialTypenameFilter = (filter) => {
2950
- return filter.type === "object" && filter.typename !== null && Object.keys(filter.props).length === 0 && (filter.id === void 0 || filter.id.length === 0) && (filter.foreignKeys === void 0 || filter.foreignKeys.length === 0);
3424
+ var isTrivialTypenameFilter = (filter2) => {
3425
+ return filter2.type === "object" && filter2.typename !== null && Object.keys(filter2.props).length === 0 && (filter2.id === void 0 || filter2.id.length === 0) && (filter2.foreignKeys === void 0 || filter2.foreignKeys.length === 0);
2951
3426
  };
2952
3427
 
2953
3428
  // src/query/query-executor.ts
@@ -2959,11 +3434,57 @@ var ExecutionTrace = Object.freeze({
2959
3434
  objectCount: 0,
2960
3435
  documentsLoaded: 0,
2961
3436
  indexHits: 0,
3437
+ beginTs: 0,
3438
+ endTs: 0,
2962
3439
  indexQueryTime: 0,
2963
3440
  documentLoadTime: 0,
2964
3441
  executionTime: 0,
2965
3442
  children: []
2966
3443
  }),
3444
+ markEnd: (trace6) => {
3445
+ trace6.endTs = performance.now();
3446
+ trace6.executionTime = trace6.endTs - trace6.beginTs;
3447
+ },
3448
+ putOnPerformanceTimeline: (trace6) => {
3449
+ performance.measure(trace6.name, {
3450
+ start: trace6.beginTs,
3451
+ end: trace6.endTs,
3452
+ detail: {
3453
+ devtools: {
3454
+ dataType: "track-entry",
3455
+ track: "Query Execution",
3456
+ trackGroup: "ECHO",
3457
+ color: "tertiary-dark",
3458
+ properties: [
3459
+ [
3460
+ "objectCount",
3461
+ trace6.objectCount
3462
+ ],
3463
+ [
3464
+ "documentsLoaded",
3465
+ trace6.documentsLoaded
3466
+ ],
3467
+ [
3468
+ "index hits",
3469
+ trace6.indexHits
3470
+ ],
3471
+ [
3472
+ "indexQueryTime",
3473
+ trace6.indexQueryTime
3474
+ ],
3475
+ [
3476
+ "documentLoadTime",
3477
+ trace6.documentLoadTime
3478
+ ]
3479
+ ],
3480
+ tooltipText: trace6.details
3481
+ }
3482
+ }
3483
+ });
3484
+ for (const child of trace6.children) {
3485
+ ExecutionTrace.putOnPerformanceTimeline(child);
3486
+ }
3487
+ },
2967
3488
  format: (trace6) => {
2968
3489
  const go = (trace7, indent) => {
2969
3490
  return [
@@ -2978,10 +3499,21 @@ var ExecutionTrace = Object.freeze({
2978
3499
  });
2979
3500
  var TRACE_QUERY_EXECUTION = false;
2980
3501
  var QueryExecutor = class extends Resource6 {
3502
+ _indexer;
3503
+ _automergeHost;
3504
+ _spaceStateManager;
3505
+ /**
3506
+ * Id of this query.
3507
+ */
3508
+ _id;
3509
+ _query;
3510
+ // TODO(dmaretskyi): Might be used in the future.
3511
+ _reactivity;
3512
+ _plan;
3513
+ _trace = ExecutionTrace.makeEmpty();
3514
+ _lastResultSet = [];
2981
3515
  constructor(options) {
2982
3516
  super();
2983
- this._trace = ExecutionTrace.makeEmpty();
2984
- this._lastResultSet = [];
2985
3517
  this._indexer = options.indexer;
2986
3518
  this._automergeHost = options.automergeHost;
2987
3519
  this._spaceStateManager = options.spaceStateManager;
@@ -3016,7 +3548,7 @@ var QueryExecutor = class extends Resource6 {
3016
3548
  async execQuery() {
3017
3549
  invariant10(this._lifecycleState === LifecycleState3.OPEN, void 0, {
3018
3550
  F: __dxlog_file11,
3019
- L: 173,
3551
+ L: 211,
3020
3552
  S: this,
3021
3553
  A: [
3022
3554
  "this._lifecycleState === LifecycleState.OPEN",
@@ -3051,7 +3583,7 @@ var QueryExecutor = class extends Resource6 {
3051
3583
  trace6.children.push(result.trace);
3052
3584
  }
3053
3585
  trace6.objectCount = workingSet.length;
3054
- trace6.executionTime = performance.now() - begin;
3586
+ ExecutionTrace.markEnd(trace6);
3055
3587
  return {
3056
3588
  workingSet,
3057
3589
  trace: trace6
@@ -3089,10 +3621,14 @@ var QueryExecutor = class extends Resource6 {
3089
3621
  case "TraverseStep":
3090
3622
  ({ workingSet: newWorkingSet, trace: trace6 } = await this._execTraverseStep(step, workingSet));
3091
3623
  break;
3624
+ case "OrderStep":
3625
+ ({ workingSet: newWorkingSet, trace: trace6 } = await this._execOrderStep(step, workingSet));
3626
+ break;
3092
3627
  default:
3093
3628
  throw new Error(`Unknown step type: ${step._tag}`);
3094
3629
  }
3095
- trace6.executionTime = performance.now() - begin;
3630
+ trace6.beginTs = begin;
3631
+ ExecutionTrace.markEnd(trace6);
3096
3632
  return {
3097
3633
  workingSet: newWorkingSet,
3098
3634
  trace: trace6
@@ -3105,7 +3641,8 @@ var QueryExecutor = class extends Resource6 {
3105
3641
  const trace6 = {
3106
3642
  ...ExecutionTrace.makeEmpty(),
3107
3643
  name: "Select",
3108
- details: JSON.stringify(step.selector)
3644
+ details: JSON.stringify(step.selector),
3645
+ beginTs: performance.now()
3109
3646
  };
3110
3647
  switch (step.selector._tag) {
3111
3648
  case "WildcardSelector": {
@@ -3126,7 +3663,7 @@ var QueryExecutor = class extends Resource6 {
3126
3663
  const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3127
3664
  trace6.documentsLoaded += results.length;
3128
3665
  trace6.documentLoadTime += performance.now() - documentLoadStart;
3129
- workingSet.push(...results.filter(isNonNullable2).filter((item) => step.spaces.includes(item.spaceId)));
3666
+ workingSet.push(...results.filter(isNonNullable3).filter((item) => step.spaces.includes(item.spaceId)));
3130
3667
  trace6.objectCount = workingSet.length;
3131
3668
  break;
3132
3669
  }
@@ -3136,7 +3673,7 @@ var QueryExecutor = class extends Resource6 {
3136
3673
  sourceSpaceId: step.spaces[0]
3137
3674
  })));
3138
3675
  trace6.documentLoadTime += performance.now() - beginLoad;
3139
- workingSet.push(...items.filter(isNonNullable2));
3676
+ workingSet.push(...items.filter(isNonNullable3));
3140
3677
  trace6.objectCount = workingSet.length;
3141
3678
  break;
3142
3679
  }
@@ -3158,7 +3695,7 @@ var QueryExecutor = class extends Resource6 {
3158
3695
  const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3159
3696
  trace6.documentsLoaded += results.length;
3160
3697
  trace6.documentLoadTime += performance.now() - documentLoadStart;
3161
- workingSet.push(...results.filter(isNonNullable2).filter((item) => step.spaces.includes(item.spaceId)));
3698
+ workingSet.push(...results.filter(isNonNullable3).filter((item) => step.spaces.includes(item.spaceId)));
3162
3699
  trace6.objectCount = workingSet.length;
3163
3700
  break;
3164
3701
  }
@@ -3183,7 +3720,7 @@ var QueryExecutor = class extends Resource6 {
3183
3720
  const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3184
3721
  trace6.documentsLoaded += results.length;
3185
3722
  trace6.documentLoadTime += performance.now() - documentLoadStart;
3186
- workingSet.push(...results.filter(isNonNullable2).filter((item) => step.spaces.includes(item.spaceId)));
3723
+ workingSet.push(...results.filter(isNonNullable3).filter((item) => step.spaces.includes(item.spaceId)));
3187
3724
  trace6.objectCount = workingSet.length;
3188
3725
  break;
3189
3726
  }
@@ -3212,17 +3749,6 @@ var QueryExecutor = class extends Resource6 {
3212
3749
  };
3213
3750
  }
3214
3751
  async _execFilterDeletedStep(step, workingSet) {
3215
- if (workingSet.length === 6) {
3216
- log9.info("FilterDeletedStep", {
3217
- step,
3218
- workingSet
3219
- }, {
3220
- F: __dxlog_file11,
3221
- L: 386,
3222
- S: this,
3223
- C: (f, a) => f(...a)
3224
- });
3225
- }
3226
3752
  const expected = step.mode === "only-deleted";
3227
3753
  const result = workingSet.filter((item) => ObjectStructure.isDeleted(item.doc) === expected);
3228
3754
  return {
@@ -3260,24 +3786,24 @@ var QueryExecutor = class extends Resource6 {
3260
3786
  spaceId: item.spaceId
3261
3787
  } : null;
3262
3788
  } catch {
3263
- log9.warn("Invalid reference", {
3789
+ log9.warn("invalid reference", {
3264
3790
  ref: ref2["/"]
3265
3791
  }, {
3266
3792
  F: __dxlog_file11,
3267
- L: 431,
3793
+ L: 477,
3268
3794
  S: this,
3269
3795
  C: (f, a) => f(...a)
3270
3796
  });
3271
3797
  return null;
3272
3798
  }
3273
3799
  });
3274
- }).filter(isNonNullable2);
3800
+ }).filter(isNonNullable3);
3275
3801
  const beginLoad = performance.now();
3276
3802
  const items = await Promise.all(refs.map(({ ref, spaceId }) => this._loadFromDXN(ref, {
3277
3803
  sourceSpaceId: spaceId
3278
3804
  })));
3279
3805
  trace6.documentLoadTime += performance.now() - beginLoad;
3280
- newWorkingSet.push(...items.filter(isNonNullable2));
3806
+ newWorkingSet.push(...items.filter(isNonNullable3));
3281
3807
  trace6.objectCount = newWorkingSet.length;
3282
3808
  break;
3283
3809
  }
@@ -3296,7 +3822,7 @@ var QueryExecutor = class extends Resource6 {
3296
3822
  const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3297
3823
  trace6.documentsLoaded += results.length;
3298
3824
  trace6.documentLoadTime += performance.now() - documentLoadStart;
3299
- newWorkingSet.push(...results.filter(isNonNullable2));
3825
+ newWorkingSet.push(...results.filter(isNonNullable3));
3300
3826
  trace6.objectCount = newWorkingSet.length;
3301
3827
  break;
3302
3828
  }
@@ -3318,23 +3844,23 @@ var QueryExecutor = class extends Resource6 {
3318
3844
  spaceId: item.spaceId
3319
3845
  };
3320
3846
  } catch {
3321
- log9.warn("Invalid reference", {
3847
+ log9.warn("invalid reference", {
3322
3848
  ref: ref["/"]
3323
3849
  }, {
3324
3850
  F: __dxlog_file11,
3325
- L: 494,
3851
+ L: 540,
3326
3852
  S: this,
3327
3853
  C: (f, a) => f(...a)
3328
3854
  });
3329
3855
  return null;
3330
3856
  }
3331
- }).filter(isNonNullable2);
3857
+ }).filter(isNonNullable3);
3332
3858
  const beginLoad = performance.now();
3333
3859
  const items = await Promise.all(refs.map(({ ref, spaceId }) => this._loadFromDXN(ref, {
3334
3860
  sourceSpaceId: spaceId
3335
3861
  })));
3336
3862
  trace6.documentLoadTime += performance.now() - beginLoad;
3337
- newWorkingSet.push(...items.filter(isNonNullable2));
3863
+ newWorkingSet.push(...items.filter(isNonNullable3));
3338
3864
  trace6.objectCount = newWorkingSet.length;
3339
3865
  break;
3340
3866
  }
@@ -3354,7 +3880,7 @@ var QueryExecutor = class extends Resource6 {
3354
3880
  const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3355
3881
  trace6.documentsLoaded += results.length;
3356
3882
  trace6.documentLoadTime += performance.now() - documentLoadStart;
3357
- newWorkingSet.push(...results.filter(isNonNullable2));
3883
+ newWorkingSet.push(...results.filter(isNonNullable3));
3358
3884
  trace6.objectCount = newWorkingSet.length;
3359
3885
  break;
3360
3886
  }
@@ -3411,16 +3937,83 @@ var QueryExecutor = class extends Resource6 {
3411
3937
  trace: trace6
3412
3938
  };
3413
3939
  }
3940
+ async _execOrderStep(step, workingSet) {
3941
+ const sortedWorkingSet = [
3942
+ ...workingSet
3943
+ ].sort((a, b) => this._compareMultiOrder(a, b, step.order));
3944
+ return {
3945
+ workingSet: sortedWorkingSet,
3946
+ trace: {
3947
+ ...ExecutionTrace.makeEmpty(),
3948
+ name: "Order",
3949
+ details: JSON.stringify(step.order),
3950
+ objectCount: sortedWorkingSet.length
3951
+ }
3952
+ };
3953
+ }
3954
+ _compareMultiOrder(a, b, orders) {
3955
+ if (orders.length === 0) {
3956
+ return 0;
3957
+ } else if (orders.length === 1) {
3958
+ return this._compareByOrder(a, b, orders[0]);
3959
+ }
3960
+ for (const order of orders) {
3961
+ const comparison = this._compareByOrder(a, b, order);
3962
+ if (comparison !== 0) {
3963
+ return comparison;
3964
+ }
3965
+ }
3966
+ return 0;
3967
+ }
3968
+ _compareByOrder(a, b, order) {
3969
+ switch (order.kind) {
3970
+ case "natural":
3971
+ return a.objectId.localeCompare(b.objectId);
3972
+ case "property": {
3973
+ const comparison = this._compareByProperty(a, b, order.property);
3974
+ return order.direction === "desc" ? -comparison : comparison;
3975
+ }
3976
+ default:
3977
+ return 0;
3978
+ }
3979
+ }
3980
+ _compareByProperty(a, b, property) {
3981
+ const aValue = a.doc.data[property];
3982
+ const bValue = b.doc.data[property];
3983
+ if (aValue == null && bValue == null) {
3984
+ return 0;
3985
+ }
3986
+ if (aValue == null) {
3987
+ return 1;
3988
+ }
3989
+ if (bValue == null) {
3990
+ return -1;
3991
+ }
3992
+ if (typeof aValue === "string" && typeof bValue === "string") {
3993
+ return aValue.localeCompare(bValue);
3994
+ }
3995
+ if (typeof aValue === "number" && typeof bValue === "number") {
3996
+ return aValue - bValue;
3997
+ }
3998
+ if (typeof aValue === "boolean" && typeof bValue === "boolean") {
3999
+ return aValue === bValue ? 0 : aValue ? 1 : -1;
4000
+ }
4001
+ return String(aValue).localeCompare(String(bValue));
4002
+ }
3414
4003
  async _loadDocumentsAfterIndexQuery(indexHits) {
3415
4004
  return Promise.all(indexHits.map(async (hit) => {
3416
4005
  return this._loadFromIndexHit(hit);
3417
4006
  }));
3418
4007
  }
4008
+ /**
4009
+ * Space key hex -> SpaceId.
4010
+ */
4011
+ _spaceIdCache = /* @__PURE__ */ new Map();
3419
4012
  async _loadFromIndexHit(hit) {
3420
4013
  const { objectId, documentId, spaceKey: spaceKeyInIndex } = objectPointerCodec3.decode(hit.id);
3421
4014
  const handle = await this._automergeHost.loadDoc(Context3.default(void 0, {
3422
4015
  F: __dxlog_file11,
3423
- L: 604
4016
+ L: 738
3424
4017
  }), documentId);
3425
4018
  const doc = handle.doc();
3426
4019
  if (!doc) {
@@ -3430,6 +4023,11 @@ var QueryExecutor = class extends Resource6 {
3430
4023
  if (!spaceKey) {
3431
4024
  return null;
3432
4025
  }
4026
+ let spaceId = this._spaceIdCache.get(spaceKey);
4027
+ if (!spaceId) {
4028
+ spaceId = await createIdFromSpaceKey(PublicKey3.from(spaceKey));
4029
+ this._spaceIdCache.set(spaceKey, spaceId);
4030
+ }
3433
4031
  const object = DatabaseDirectory3.getInlineObject(doc, objectId);
3434
4032
  if (!object) {
3435
4033
  return null;
@@ -3437,7 +4035,7 @@ var QueryExecutor = class extends Resource6 {
3437
4035
  return {
3438
4036
  objectId,
3439
4037
  documentId,
3440
- spaceId: await createIdFromSpaceKey(PublicKey3.from(spaceKey)),
4038
+ spaceId,
3441
4039
  doc: object
3442
4040
  };
3443
4041
  }
@@ -3448,7 +4046,7 @@ var QueryExecutor = class extends Resource6 {
3448
4046
  dxn
3449
4047
  }, {
3450
4048
  F: __dxlog_file11,
3451
- L: 631,
4049
+ L: 771,
3452
4050
  S: this,
3453
4051
  C: (f, a) => f(...a)
3454
4052
  });
@@ -3461,7 +4059,7 @@ var QueryExecutor = class extends Resource6 {
3461
4059
  spaceId
3462
4060
  }, {
3463
4061
  F: __dxlog_file11,
3464
- L: 639,
4062
+ L: 779,
3465
4063
  S: this,
3466
4064
  C: (f, a) => f(...a)
3467
4065
  });
@@ -3473,7 +4071,7 @@ var QueryExecutor = class extends Resource6 {
3473
4071
  spaceId
3474
4072
  }, {
3475
4073
  F: __dxlog_file11,
3476
- L: 644,
4074
+ L: 784,
3477
4075
  S: this,
3478
4076
  C: (f, a) => f(...a)
3479
4077
  });
@@ -3494,7 +4092,7 @@ var QueryExecutor = class extends Resource6 {
3494
4092
  }
3495
4093
  const handle = await this._automergeHost.loadDoc(Context3.default(void 0, {
3496
4094
  F: __dxlog_file11,
3497
- L: 663
4095
+ L: 803
3498
4096
  }), link);
3499
4097
  const doc = handle.doc();
3500
4098
  if (!doc) {
@@ -3522,9 +4120,13 @@ function _ts_decorate5(decorators, target, key, desc) {
3522
4120
  }
3523
4121
  var __dxlog_file12 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/query-service.ts";
3524
4122
  var QueryServiceImpl = class extends Resource7 {
4123
+ _params;
4124
+ // TODO(dmaretskyi): We need to implement query deduping. Idle composer has 80 queries with only 10 being unique.
4125
+ _queries = /* @__PURE__ */ new Set();
4126
+ _updateQueries;
3525
4127
  // TODO(burdon): OK for options, but not params. Pass separately and type readonly here.
3526
4128
  constructor(_params) {
3527
- super(), this._params = _params, this._queries = /* @__PURE__ */ new Set();
4129
+ super(), this._params = _params;
3528
4130
  trace4.diagnostic({
3529
4131
  id: "active-queries",
3530
4132
  name: "Active Queries",
@@ -3541,7 +4143,7 @@ var QueryServiceImpl = class extends Resource7 {
3541
4143
  }
3542
4144
  async _open() {
3543
4145
  this._params.indexer.updated.on(this._ctx, () => this.invalidateQueries());
3544
- this._updateQueries = new DeferredTask(this._ctx, this._executeQueries.bind(this));
4146
+ this._updateQueries = new DeferredTask2(this._ctx, this._executeQueries.bind(this));
3545
4147
  }
3546
4148
  async _close() {
3547
4149
  await this._updateQueries.join();
@@ -3552,6 +4154,18 @@ var QueryServiceImpl = class extends Resource7 {
3552
4154
  }
3553
4155
  execQuery(request) {
3554
4156
  return new Stream2(({ next, close, ctx }) => {
4157
+ if (this._params.indexer.config?.enabled !== true) {
4158
+ log10.error("indexer is disabled", {
4159
+ config: this._params.indexer.config
4160
+ }, {
4161
+ F: __dxlog_file12,
4162
+ L: 102,
4163
+ S: this,
4164
+ C: (f, a) => f(...a)
4165
+ });
4166
+ close();
4167
+ return;
4168
+ }
3555
4169
  const queryEntry = this._createQuery(ctx, request, next, close, close);
3556
4170
  scheduleMicroTask(ctx, async () => {
3557
4171
  await queryEntry.executor.open();
@@ -3567,7 +4181,7 @@ var QueryServiceImpl = class extends Resource7 {
3567
4181
  async reindex() {
3568
4182
  log10("Reindexing all documents...", void 0, {
3569
4183
  F: __dxlog_file12,
3570
- L: 113,
4184
+ L: 121,
3571
4185
  S: this,
3572
4186
  C: (f, a) => f(...a)
3573
4187
  });
@@ -3582,7 +4196,7 @@ var QueryServiceImpl = class extends Resource7 {
3582
4196
  count: ids.size
3583
4197
  }, {
3584
4198
  F: __dxlog_file12,
3585
- L: 121,
4199
+ L: 129,
3586
4200
  S: this,
3587
4201
  C: (f, a) => f(...a)
3588
4202
  });
@@ -3592,7 +4206,7 @@ var QueryServiceImpl = class extends Resource7 {
3592
4206
  count: ids.size
3593
4207
  }, {
3594
4208
  F: __dxlog_file12,
3595
- L: 125,
4209
+ L: 133,
3596
4210
  S: this,
3597
4211
  C: (f, a) => f(...a)
3598
4212
  });
@@ -3658,7 +4272,7 @@ var QueryServiceImpl = class extends Resource7 {
3658
4272
  } catch (err) {
3659
4273
  log10.catch(err, void 0, {
3660
4274
  F: __dxlog_file12,
3661
- L: 196,
4275
+ L: 204,
3662
4276
  S: this,
3663
4277
  C: (f, a) => f(...a)
3664
4278
  });
@@ -3669,7 +4283,7 @@ var QueryServiceImpl = class extends Resource7 {
3669
4283
  duration: performance.now() - begin
3670
4284
  }, {
3671
4285
  F: __dxlog_file12,
3672
- L: 200,
4286
+ L: 208,
3673
4287
  S: this,
3674
4288
  C: (f, a) => f(...a)
3675
4289
  });
@@ -3720,7 +4334,7 @@ var createDocumentsIterator = (automergeHost) => (
3720
4334
  }
3721
4335
  const linkHandle = await automergeHost.loadDoc(Context4.default(void 0, {
3722
4336
  F: __dxlog_file12,
3723
- L: 240
4337
+ L: 248
3724
4338
  }), urlString);
3725
4339
  for await (const result of getObjectsFromHandle(linkHandle)) {
3726
4340
  yield result;
@@ -3745,7 +4359,7 @@ var createDocumentsIterator = (automergeHost) => (
3745
4359
  import { interpretAsDocumentId as interpretAsDocumentId3 } from "@automerge/automerge-repo";
3746
4360
  import isEqual from "lodash.isequal";
3747
4361
  import { Event as Event3, UpdateScheduler as UpdateScheduler3 } from "@dxos/async";
3748
- import { Resource as Resource8, Context as Context5, LifecycleState as LifecycleState4 } from "@dxos/context";
4362
+ import { Context as Context5, LifecycleState as LifecycleState4, Resource as Resource8 } from "@dxos/context";
3749
4363
  import { invariant as invariant12 } from "@dxos/invariant";
3750
4364
 
3751
4365
  // src/db-host/database-root.ts
@@ -3754,17 +4368,17 @@ import { DatabaseDirectory as DatabaseDirectory5, SpaceDocVersion as SpaceDocVer
3754
4368
  import { invariant as invariant11 } from "@dxos/invariant";
3755
4369
 
3756
4370
  // src/db-host/automerge-metrics.ts
3757
- import * as A4 from "@automerge/automerge";
4371
+ import * as A5 from "@automerge/automerge";
3758
4372
  import { log as log11 } from "@dxos/log";
3759
4373
  var __dxlog_file13 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/automerge-metrics.ts";
3760
4374
  var measureDocMetrics = (doc) => {
3761
- const snapshot = A4.save(doc);
4375
+ const snapshot = A5.save(doc);
3762
4376
  const start = Date.now();
3763
- const temp = A4.load(snapshot);
4377
+ const temp = A5.load(snapshot);
3764
4378
  const end = Date.now();
3765
- A4.free(temp);
4379
+ A5.free(temp);
3766
4380
  const getAllChangesStart = Date.now();
3767
- const mutationCount = A4.getAllChanges(doc).length;
4381
+ const mutationCount = A5.getAllChanges(doc).length;
3768
4382
  const getAllChangesEnd = Date.now();
3769
4383
  if (getAllChangesEnd - getAllChangesStart > 300) {
3770
4384
  log11.warn("getAllChanges took too long", {
@@ -3786,6 +4400,7 @@ var measureDocMetrics = (doc) => {
3786
4400
  // src/db-host/database-root.ts
3787
4401
  var __dxlog_file14 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/database-root.ts";
3788
4402
  var DatabaseRoot = class {
4403
+ _rootHandle;
3789
4404
  static mapLinks(doc, mapping) {
3790
4405
  doc.change((d) => {
3791
4406
  if (!d.links) {
@@ -3870,14 +4485,11 @@ var DatabaseRoot = class {
3870
4485
  // src/db-host/space-state-manager.ts
3871
4486
  var __dxlog_file15 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/space-state-manager.ts";
3872
4487
  var SpaceStateManager = class extends Resource8 {
3873
- constructor() {
3874
- super(...arguments);
3875
- this._roots = /* @__PURE__ */ new Map();
3876
- this._rootBySpace = /* @__PURE__ */ new Map();
3877
- this._perRootContext = /* @__PURE__ */ new Map();
3878
- this._lastSpaceDocumentList = /* @__PURE__ */ new Map();
3879
- this.spaceDocumentListUpdated = new Event3();
3880
- }
4488
+ _roots = /* @__PURE__ */ new Map();
4489
+ _rootBySpace = /* @__PURE__ */ new Map();
4490
+ _perRootContext = /* @__PURE__ */ new Map();
4491
+ _lastSpaceDocumentList = /* @__PURE__ */ new Map();
4492
+ spaceDocumentListUpdated = new Event3();
3881
4493
  async _close(ctx) {
3882
4494
  for (const [_, rootCtx] of this._perRootContext) {
3883
4495
  await rootCtx.dispose();
@@ -3952,6 +4564,10 @@ var SpaceStateManager = class extends Resource8 {
3952
4564
  }
3953
4565
  };
3954
4566
  var SpaceDocumentListUpdatedEvent = class {
4567
+ spaceId;
4568
+ spaceRootId;
4569
+ previousRootId;
4570
+ documentIds;
3955
4571
  constructor(spaceId, spaceRootId, previousRootId, documentIds) {
3956
4572
  this.spaceId = spaceId;
3957
4573
  this.spaceRootId = spaceRootId;
@@ -3968,9 +4584,15 @@ var DEFAULT_INDEXING_CONFIG = {
3968
4584
  vector: false
3969
4585
  };
3970
4586
  var EchoHost = class extends Resource9 {
4587
+ _indexMetadataStore;
4588
+ _indexer;
4589
+ _automergeHost;
4590
+ _queryService;
4591
+ _dataService;
4592
+ _spaceStateManager = new SpaceStateManager();
4593
+ _echoDataMonitor;
3971
4594
  constructor({ kv, indexing = {}, peerIdProvider, getSpaceKeyByRootDocumentId }) {
3972
4595
  super();
3973
- this._spaceStateManager = new SpaceStateManager();
3974
4596
  const indexingConfig = {
3975
4597
  ...DEFAULT_INDEXING_CONFIG,
3976
4598
  ...indexing
@@ -4091,7 +4713,6 @@ var EchoHost = class extends Resource9 {
4091
4713
  if (e.previousRootId) {
4092
4714
  void this._automergeHost.clearLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.previousRootId));
4093
4715
  }
4094
- void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId), e.documentIds);
4095
4716
  void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.spaceRootId), e.documentIds);
4096
4717
  });
4097
4718
  this._automergeHost.documentsSaved.on(this._ctx, () => {
@@ -4108,7 +4729,7 @@ var EchoHost = class extends Resource9 {
4108
4729
  * Flush all pending writes to the underlying storage.
4109
4730
  */
4110
4731
  async flush() {
4111
- await this._automergeHost.repo.flush();
4732
+ await this._automergeHost.flush();
4112
4733
  }
4113
4734
  /**
4114
4735
  * Perform any pending index updates.
@@ -4137,7 +4758,7 @@ var EchoHost = class extends Resource9 {
4137
4758
  async createSpaceRoot(spaceKey) {
4138
4759
  invariant13(this._lifecycleState === LifecycleState5.OPEN, void 0, {
4139
4760
  F: __dxlog_file16,
4140
- L: 255,
4761
+ L: 254,
4141
4762
  S: this,
4142
4763
  A: [
4143
4764
  "this._lifecycleState === LifecycleState.OPEN",
@@ -4165,7 +4786,7 @@ var EchoHost = class extends Resource9 {
4165
4786
  async openSpaceRoot(spaceId, automergeUrl) {
4166
4787
  invariant13(this._lifecycleState === LifecycleState5.OPEN, void 0, {
4167
4788
  F: __dxlog_file16,
4168
- L: 274,
4789
+ L: 273,
4169
4790
  S: this,
4170
4791
  A: [
4171
4792
  "this._lifecycleState === LifecycleState.OPEN",
@@ -4195,16 +4816,17 @@ var EchoHost = class extends Resource9 {
4195
4816
  };
4196
4817
 
4197
4818
  // src/edge/echo-edge-replicator.ts
4819
+ import * as Automerge from "@automerge/automerge";
4198
4820
  import { cbor as cbor2 } from "@automerge/automerge-repo";
4199
- import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask as scheduleMicroTask2 } from "@dxos/async";
4821
+ import { Mutex, scheduleMicroTask as scheduleMicroTask2, scheduleTask as scheduleTask2 } from "@dxos/async";
4200
4822
  import { Context as Context6, Resource as Resource11 } from "@dxos/context";
4201
4823
  import { randomUUID } from "@dxos/crypto";
4202
4824
  import { invariant as invariant14 } from "@dxos/invariant";
4203
4825
  import { log as log13 } from "@dxos/log";
4204
- import { EdgeService } from "@dxos/protocols";
4826
+ import { DocumentCodec, EdgeService } from "@dxos/protocols";
4205
4827
  import { buf } from "@dxos/protocols/buf";
4206
4828
  import { MessageSchema as RouterMessageSchema } from "@dxos/protocols/buf/dxos/edge/messenger_pb";
4207
- import { bufferToArray as bufferToArray2 } from "@dxos/util";
4829
+ import { bufferToArray as bufferToArray2, setDeep } from "@dxos/util";
4208
4830
 
4209
4831
  // src/edge/inflight-request-limiter.ts
4210
4832
  import { Trigger as Trigger2 } from "@dxos/async";
@@ -4212,8 +4834,17 @@ import { Resource as Resource10 } from "@dxos/context";
4212
4834
  import { log as log12 } from "@dxos/log";
4213
4835
  var __dxlog_file17 = "/__w/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/inflight-request-limiter.ts";
4214
4836
  var InflightRequestLimiter = class extends Resource10 {
4837
+ _config;
4838
+ /**
4839
+ * Decrement when we receive a sync message, increment when we send one.
4840
+ * Can't exceed _config.maxInflightRequests.
4841
+ * Resets after timeout to avoid replicator being stuck.
4842
+ */
4843
+ _inflightRequestBalance = 0;
4844
+ _requestBarrier = new Trigger2();
4845
+ _resetBalanceTimeout;
4215
4846
  constructor(_config) {
4216
- super(), this._config = _config, this._inflightRequestBalance = 0, this._requestBarrier = new Trigger2();
4847
+ super(), this._config = _config;
4217
4848
  }
4218
4849
  async _open() {
4219
4850
  this._inflightRequestBalance = 0;
@@ -4330,14 +4961,17 @@ var INITIAL_RESTART_DELAY = 500;
4330
4961
  var RESTART_DELAY_JITTER = 250;
4331
4962
  var MAX_RESTART_DELAY = 5e3;
4332
4963
  var EchoEdgeReplicator = class {
4333
- constructor({ edgeConnection, disableSharePolicy }) {
4334
- this._mutex = new Mutex();
4335
- this._ctx = void 0;
4336
- this._context = null;
4337
- this._connectedSpaces = /* @__PURE__ */ new Set();
4338
- this._connections = /* @__PURE__ */ new Map();
4339
- this._sharePolicyEnabled = true;
4964
+ _edgeConnection;
4965
+ _edgeHttpClient;
4966
+ _mutex = new Mutex();
4967
+ _ctx = void 0;
4968
+ _context = null;
4969
+ _connectedSpaces = /* @__PURE__ */ new Set();
4970
+ _connections = /* @__PURE__ */ new Map();
4971
+ _sharePolicyEnabled = true;
4972
+ constructor({ edgeConnection, edgeHttpClient, disableSharePolicy }) {
4340
4973
  this._edgeConnection = edgeConnection;
4974
+ this._edgeHttpClient = edgeHttpClient;
4341
4975
  this._sharePolicyEnabled = !disableSharePolicy;
4342
4976
  }
4343
4977
  async connect(context) {
@@ -4346,14 +4980,14 @@ var EchoEdgeReplicator = class {
4346
4980
  connectedSpaces: this._connectedSpaces.size
4347
4981
  }, {
4348
4982
  F: __dxlog_file18,
4349
- L: 61,
4983
+ L: 73,
4350
4984
  S: this,
4351
4985
  C: (f, a) => f(...a)
4352
4986
  });
4353
4987
  this._context = context;
4354
4988
  this._ctx = Context6.default(void 0, {
4355
4989
  F: __dxlog_file18,
4356
- L: 63
4990
+ L: 75
4357
4991
  });
4358
4992
  this._ctx.onDispose(this._edgeConnection.onReconnected(() => {
4359
4993
  this._ctx && scheduleMicroTask2(this._ctx, () => this._handleReconnect());
@@ -4452,7 +5086,7 @@ var EchoEdgeReplicator = class {
4452
5086
  async _openConnection(spaceId, reconnects = 0) {
4453
5087
  invariant14(this._context, void 0, {
4454
5088
  F: __dxlog_file18,
4455
- L: 124,
5089
+ L: 136,
4456
5090
  S: this,
4457
5091
  A: [
4458
5092
  "this._context",
@@ -4461,7 +5095,7 @@ var EchoEdgeReplicator = class {
4461
5095
  });
4462
5096
  invariant14(!this._connections.has(spaceId), void 0, {
4463
5097
  F: __dxlog_file18,
4464
- L: 125,
5098
+ L: 137,
4465
5099
  S: this,
4466
5100
  A: [
4467
5101
  "!this._connections.has(spaceId)",
@@ -4470,14 +5104,31 @@ var EchoEdgeReplicator = class {
4470
5104
  });
4471
5105
  let restartScheduled = false;
4472
5106
  const connection = new EdgeReplicatorConnection({
5107
+ edgeHttpClient: this._edgeHttpClient,
4473
5108
  edgeConnection: this._edgeConnection,
4474
5109
  spaceId,
4475
5110
  context: this._context,
4476
5111
  sharedPolicyEnabled: this._sharePolicyEnabled,
4477
5112
  onRemoteConnected: async () => {
5113
+ log13.trace("dxos.echo.edge.replicator.onRemoteConnected", {
5114
+ spaceId
5115
+ }, {
5116
+ F: __dxlog_file18,
5117
+ L: 148,
5118
+ S: this,
5119
+ C: (f, a) => f(...a)
5120
+ });
4478
5121
  this._context?.onConnectionOpen(connection);
4479
5122
  },
4480
5123
  onRemoteDisconnected: async () => {
5124
+ log13.trace("dxos.echo.edge.replicator.onRemoteDisconnected", {
5125
+ spaceId
5126
+ }, {
5127
+ F: __dxlog_file18,
5128
+ L: 152,
5129
+ S: this,
5130
+ C: (f, a) => f(...a)
5131
+ });
4481
5132
  this._context?.onConnectionClosed(connection);
4482
5133
  },
4483
5134
  onRestartRequested: async () => {
@@ -4491,7 +5142,7 @@ var EchoEdgeReplicator = class {
4491
5142
  restartDelay
4492
5143
  }, {
4493
5144
  F: __dxlog_file18,
4494
- L: 148,
5145
+ L: 163,
4495
5146
  S: this,
4496
5147
  C: (f, a) => f(...a)
4497
5148
  });
@@ -4513,6 +5164,16 @@ var EchoEdgeReplicator = class {
4513
5164
  if (ctx?.disposed) {
4514
5165
  return;
4515
5166
  }
5167
+ log13.trace("dxos.echo.edge.replicator.restart", {
5168
+ spaceId,
5169
+ reconnects,
5170
+ restartDelay
5171
+ }, {
5172
+ F: __dxlog_file18,
5173
+ L: 180,
5174
+ S: this,
5175
+ C: (f, a) => f(...a)
5176
+ });
4516
5177
  await this._openConnection(spaceId, reconnects + 1);
4517
5178
  } catch (e) {
4518
5179
  env.error = e;
@@ -4530,17 +5191,32 @@ var EchoEdgeReplicator = class {
4530
5191
  var MAX_INFLIGHT_REQUESTS = 5;
4531
5192
  var MAX_RATE_LIMIT_WAIT_TIME_MS = 3e3;
4532
5193
  var EdgeReplicatorConnection = class extends Resource11 {
4533
- constructor({ edgeConnection, spaceId, context, sharedPolicyEnabled, onRemoteConnected, onRemoteDisconnected, onRestartRequested }) {
5194
+ _connectionId = randomUUID();
5195
+ _edgeConnection;
5196
+ _edgeHttpClient;
5197
+ _remotePeerId = null;
5198
+ _targetServiceId;
5199
+ _spaceId;
5200
+ _context;
5201
+ _sharedPolicyEnabled;
5202
+ _onRemoteConnected;
5203
+ _onRemoteDisconnected;
5204
+ _onRestartRequested;
5205
+ _sequence = 0;
5206
+ _requestLimiter = new InflightRequestLimiter({
5207
+ maxInflightRequests: MAX_INFLIGHT_REQUESTS,
5208
+ resetBalanceTimeoutMs: MAX_RATE_LIMIT_WAIT_TIME_MS
5209
+ });
5210
+ _readableStreamController;
5211
+ readable;
5212
+ writable;
5213
+ constructor({ edgeConnection, edgeHttpClient, spaceId, context, sharedPolicyEnabled, onRemoteConnected, onRemoteDisconnected, onRestartRequested }) {
4534
5214
  super();
4535
- this._remotePeerId = null;
4536
- this._requestLimiter = new InflightRequestLimiter({
4537
- maxInflightRequests: MAX_INFLIGHT_REQUESTS,
4538
- resetBalanceTimeoutMs: MAX_RATE_LIMIT_WAIT_TIME_MS
4539
- });
4540
5215
  this._edgeConnection = edgeConnection;
5216
+ this._edgeHttpClient = edgeHttpClient;
4541
5217
  this._spaceId = spaceId;
4542
5218
  this._context = context;
4543
- this._remotePeerId = `${EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}-${randomUUID()}`;
5219
+ this._remotePeerId = `${EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}-${this._connectionId}`;
4544
5220
  this._targetServiceId = `${EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}`;
4545
5221
  this._sharedPolicyEnabled = sharedPolicyEnabled;
4546
5222
  this._onRemoteConnected = onRemoteConnected;
@@ -4561,7 +5237,7 @@ var EdgeReplicatorConnection = class extends Resource11 {
4561
5237
  async _open(ctx) {
4562
5238
  log13("opening...", void 0, {
4563
5239
  F: __dxlog_file18,
4564
- L: 251,
5240
+ L: 273,
4565
5241
  S: this,
4566
5242
  C: (f, a) => f(...a)
4567
5243
  });
@@ -4569,12 +5245,29 @@ var EdgeReplicatorConnection = class extends Resource11 {
4569
5245
  this._ctx.onDispose(this._edgeConnection.onMessage((msg) => {
4570
5246
  this._onMessage(msg);
4571
5247
  }));
5248
+ let firstReconnect = true;
5249
+ this._ctx.onDispose(
5250
+ // NOTE: This will fire immediately if the connection is already open.
5251
+ this._edgeConnection.onReconnected(async () => {
5252
+ if (firstReconnect) {
5253
+ log13.verbose("first reconnect skipped", void 0, {
5254
+ F: __dxlog_file18,
5255
+ L: 288,
5256
+ S: this,
5257
+ C: (f, a) => f(...a)
5258
+ });
5259
+ firstReconnect = false;
5260
+ return;
5261
+ }
5262
+ this._onRestartRequested();
5263
+ })
5264
+ );
4572
5265
  await this._onRemoteConnected();
4573
5266
  }
4574
5267
  async _close() {
4575
5268
  log13("closing...", void 0, {
4576
5269
  F: __dxlog_file18,
4577
- L: 266,
5270
+ L: 301,
4578
5271
  S: this,
4579
5272
  C: (f, a) => f(...a)
4580
5273
  });
@@ -4585,7 +5278,7 @@ var EdgeReplicatorConnection = class extends Resource11 {
4585
5278
  get peerId() {
4586
5279
  invariant14(this._remotePeerId, "Not connected", {
4587
5280
  F: __dxlog_file18,
4588
- L: 275,
5281
+ L: 310,
4589
5282
  S: this,
4590
5283
  A: [
4591
5284
  "this._remotePeerId",
@@ -4610,7 +5303,7 @@ var EdgeReplicatorConnection = class extends Resource11 {
4610
5303
  remoteId: this._remotePeerId
4611
5304
  }, {
4612
5305
  F: __dxlog_file18,
4613
- L: 290,
5306
+ L: 325,
4614
5307
  S: this,
4615
5308
  C: (f, a) => f(...a)
4616
5309
  });
@@ -4631,20 +5324,42 @@ var EdgeReplicatorConnection = class extends Resource11 {
4631
5324
  }
4632
5325
  const payload = cbor2.decode(message.payload.value);
4633
5326
  log13.verbose("received", {
4634
- type: payload.type,
4635
- documentId: payload.type === "sync" && payload.documentId,
5327
+ ...getMessageInfo(payload),
4636
5328
  remoteId: this._remotePeerId
4637
5329
  }, {
4638
5330
  F: __dxlog_file18,
4639
- L: 319,
5331
+ L: 354,
4640
5332
  S: this,
4641
5333
  C: (f, a) => f(...a)
4642
5334
  });
4643
5335
  payload.senderId = this._remotePeerId;
4644
5336
  this._processMessage(payload);
4645
5337
  }
5338
+ get bundleSyncEnabled() {
5339
+ return true;
5340
+ }
5341
+ async pushBundle(bundle) {
5342
+ const request = {
5343
+ bundle: bundle.map(({ documentId, data, heads }) => ({
5344
+ documentId,
5345
+ mutation: DocumentCodec.encode(data),
5346
+ heads
5347
+ }))
5348
+ };
5349
+ await this._edgeHttpClient.importBundle(this._spaceId, request);
5350
+ }
5351
+ async pullBundle(docHeads) {
5352
+ const request = {
5353
+ docHeads
5354
+ };
5355
+ const response = await this._edgeHttpClient.exportBundle(this._spaceId, request);
5356
+ return Object.fromEntries(response.bundle.map((doc) => [
5357
+ doc.documentId,
5358
+ DocumentCodec.decode(doc.mutation)
5359
+ ]));
5360
+ }
4646
5361
  _processMessage(message) {
4647
- if (isForbiddenErrorMessage(message)) {
5362
+ if (isErrorMessage(message)) {
4648
5363
  this._onRestartRequested();
4649
5364
  return;
4650
5365
  }
@@ -4653,33 +5368,66 @@ var EdgeReplicatorConnection = class extends Resource11 {
4653
5368
  }
4654
5369
  async _sendMessage(message) {
4655
5370
  message.targetId = this._targetServiceId;
5371
+ setDeep(message, [
5372
+ "metadata",
5373
+ "dxos_sequence"
5374
+ ], this._getSequence());
5375
+ setDeep(message, [
5376
+ "metadata",
5377
+ "dxos_connectionId"
5378
+ ], this._connectionId);
4656
5379
  log13.verbose("sending...", {
4657
- type: message.type,
4658
- documentId: message.type === "sync" && message.documentId,
5380
+ ...getMessageInfo(message),
4659
5381
  remoteId: this._remotePeerId
4660
5382
  }, {
4661
5383
  F: __dxlog_file18,
4662
- L: 348,
5384
+ L: 407,
4663
5385
  S: this,
4664
5386
  C: (f, a) => f(...a)
4665
5387
  });
4666
5388
  const encoded = cbor2.encode(message);
4667
- await this._edgeConnection.send(buf.create(RouterMessageSchema, {
4668
- serviceId: this._targetServiceId,
4669
- source: {
4670
- identityKey: this._edgeConnection.identityKey,
4671
- peerKey: this._edgeConnection.peerKey
4672
- },
4673
- payload: {
4674
- value: bufferToArray2(encoded)
4675
- }
4676
- }));
5389
+ try {
5390
+ await this._edgeConnection.send(buf.create(RouterMessageSchema, {
5391
+ serviceId: this._targetServiceId,
5392
+ source: {
5393
+ identityKey: this._edgeConnection.identityKey,
5394
+ peerKey: this._edgeConnection.peerKey
5395
+ },
5396
+ payload: {
5397
+ value: bufferToArray2(encoded)
5398
+ }
5399
+ }));
5400
+ } catch (err) {
5401
+ log13.error("failed to send message", {
5402
+ err
5403
+ }, {
5404
+ F: __dxlog_file18,
5405
+ L: 426,
5406
+ S: this,
5407
+ C: (f, a) => f(...a)
5408
+ });
5409
+ }
5410
+ }
5411
+ _getSequence() {
5412
+ return this._sequence++;
4677
5413
  }
4678
5414
  };
4679
- var isForbiddenErrorMessage = (message) => message.type === "error" && message.message === "Forbidden";
5415
+ var isErrorMessage = (message) => message.type === "error";
5416
+ var getMessageInfo = (msg) => {
5417
+ const { have, heads, need, changes } = msg.type === "sync" ? Automerge.decodeSyncMessage(msg.data) : {};
5418
+ return {
5419
+ type: msg.type,
5420
+ documentId: "documentId" in msg ? msg.documentId : void 0,
5421
+ have,
5422
+ heads,
5423
+ need,
5424
+ changes: changes?.length,
5425
+ sequence: msg.metadata?.dxos_sequence
5426
+ };
5427
+ };
4680
5428
 
4681
5429
  // src/util.ts
4682
- import { decodeReference, ObjectStructure as ObjectStructure2 } from "@dxos/echo-protocol";
5430
+ import { ObjectStructure as ObjectStructure2, decodeReference } from "@dxos/echo-protocol";
4683
5431
  var findInlineObjectOfType = (spaceDoc, typename) => {
4684
5432
  for (const id in spaceDoc.objects ?? {}) {
4685
5433
  const obj = spaceDoc.objects[id];
@@ -4731,6 +5479,7 @@ export {
4731
5479
  diffCollectionState,
4732
5480
  encodingOptions,
4733
5481
  filterMatchObject,
5482
+ filterMatchObjectJSON,
4734
5483
  filterMatchValue,
4735
5484
  findInlineObjectOfType,
4736
5485
  getSpaceIdFromCollectionId,