@dxos/echo-pipeline 0.8.3 → 0.8.4-main.84f28bd

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/lib/browser/{chunk-TQJTKNMS.mjs → chunk-ANZAS5CC.mjs} +2 -2
  2. package/dist/lib/browser/{chunk-35I6ERLG.mjs → chunk-GBFX5J5B.mjs} +27 -27
  3. package/dist/lib/browser/filter/index.mjs +1 -1
  4. package/dist/lib/browser/index.mjs +61 -61
  5. package/dist/lib/browser/index.mjs.map +3 -3
  6. package/dist/lib/browser/meta.json +1 -1
  7. package/dist/lib/browser/testing/index.mjs +13 -13
  8. package/dist/lib/node-esm/{chunk-RVK35BS7.mjs → chunk-2SAZ7CCF.mjs} +2 -2
  9. package/dist/lib/node-esm/{chunk-5BHLPT24.mjs → chunk-FQFKWA3X.mjs} +27 -27
  10. package/dist/lib/node-esm/filter/index.mjs +1 -1
  11. package/dist/lib/node-esm/index.mjs +61 -61
  12. package/dist/lib/node-esm/index.mjs.map +3 -3
  13. package/dist/lib/node-esm/meta.json +1 -1
  14. package/dist/lib/node-esm/testing/index.mjs +13 -13
  15. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  16. package/dist/types/tsconfig.tsbuildinfo +1 -1
  17. package/package.json +38 -38
  18. package/src/automerge/echo-network-adapter.ts +2 -1
  19. package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
  20. package/dist/lib/node/chunk-HOPOFWAL.cjs.map +0 -7
  21. package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
  22. package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
  23. package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
  24. package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
  25. package/dist/lib/node/filter/index.cjs +0 -32
  26. package/dist/lib/node/filter/index.cjs.map +0 -7
  27. package/dist/lib/node/index.cjs +0 -4699
  28. package/dist/lib/node/index.cjs.map +0 -7
  29. package/dist/lib/node/meta.json +0 -1
  30. package/dist/lib/node/testing/index.cjs +0 -753
  31. package/dist/lib/node/testing/index.cjs.map +0 -7
  32. /package/dist/lib/browser/{chunk-TQJTKNMS.mjs.map → chunk-ANZAS5CC.mjs.map} +0 -0
  33. /package/dist/lib/browser/{chunk-35I6ERLG.mjs.map → chunk-GBFX5J5B.mjs.map} +0 -0
  34. /package/dist/lib/node-esm/{chunk-RVK35BS7.mjs.map → chunk-2SAZ7CCF.mjs.map} +0 -0
  35. /package/dist/lib/node-esm/{chunk-5BHLPT24.mjs.map → chunk-FQFKWA3X.mjs.map} +0 -0
@@ -1,4699 +0,0 @@
1
- "use strict";
2
- var __create = Object.create;
3
- var __defProp = Object.defineProperty;
4
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
- var __hasOwnProp = Object.prototype.hasOwnProperty;
8
- var __export = (target, all) => {
9
- for (var name in all)
10
- __defProp(target, name, { get: all[name], enumerable: true });
11
- };
12
- var __copyProps = (to, from, except, desc) => {
13
- if (from && typeof from === "object" || typeof from === "function") {
14
- for (let key of __getOwnPropNames(from))
15
- if (!__hasOwnProp.call(to, key) && key !== except)
16
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
- }
18
- return to;
19
- };
20
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
- // If the importer is in node compatibility mode or this is not an ESM
22
- // file that has been converted to a CommonJS file using a Babel-
23
- // compatible transform (i.e. "__esModule" has not been set), then set
24
- // "default" to the CommonJS "module.exports" for node compatibility.
25
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
- mod
27
- ));
28
- var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
- var node_exports = {};
30
- __export(node_exports, {
31
- AuthExtension: () => import_chunk_JXX6LF5U.AuthExtension,
32
- AuthStatus: () => import_chunk_JXX6LF5U.AuthStatus,
33
- AutomergeHost: () => AutomergeHost,
34
- CredentialRetrieverExtension: () => import_chunk_JXX6LF5U.CredentialRetrieverExtension,
35
- CredentialServerExtension: () => import_chunk_JXX6LF5U.CredentialServerExtension,
36
- DataServiceImpl: () => DataServiceImpl,
37
- DatabaseRoot: () => DatabaseRoot,
38
- DocumentsSynchronizer: () => DocumentsSynchronizer,
39
- EchoDataMonitor: () => EchoDataMonitor,
40
- EchoEdgeReplicator: () => EchoEdgeReplicator,
41
- EchoHost: () => EchoHost,
42
- ExecutionTrace: () => ExecutionTrace,
43
- FIND_PARAMS: () => FIND_PARAMS,
44
- LevelDBStorageAdapter: () => LevelDBStorageAdapter,
45
- MOCK_AUTH_PROVIDER: () => import_chunk_JXX6LF5U.MOCK_AUTH_PROVIDER,
46
- MOCK_AUTH_VERIFIER: () => import_chunk_JXX6LF5U.MOCK_AUTH_VERIFIER,
47
- MeshEchoReplicator: () => MeshEchoReplicator,
48
- MetadataStore: () => import_chunk_JXX6LF5U.MetadataStore,
49
- Pipeline: () => import_chunk_JXX6LF5U.Pipeline,
50
- QueryExecutor: () => QueryExecutor,
51
- QueryPlan: () => QueryPlan,
52
- QueryPlanner: () => QueryPlanner,
53
- QueryServiceImpl: () => QueryServiceImpl,
54
- Space: () => import_chunk_JXX6LF5U.Space,
55
- SpaceDocumentListUpdatedEvent: () => SpaceDocumentListUpdatedEvent,
56
- SpaceManager: () => import_chunk_JXX6LF5U.SpaceManager,
57
- SpaceProtocol: () => import_chunk_JXX6LF5U.SpaceProtocol,
58
- SpaceProtocolSession: () => import_chunk_JXX6LF5U.SpaceProtocolSession,
59
- SpaceStateManager: () => SpaceStateManager,
60
- TimeframeClock: () => import_chunk_JXX6LF5U.TimeframeClock,
61
- codec: () => import_chunk_JXX6LF5U.codec,
62
- createIdFromSpaceKey: () => import_chunk_JXX6LF5U.createIdFromSpaceKey,
63
- createMappedFeedWriter: () => import_chunk_JXX6LF5U.createMappedFeedWriter,
64
- deriveCollectionIdFromSpaceId: () => deriveCollectionIdFromSpaceId,
65
- diffCollectionState: () => diffCollectionState,
66
- encodingOptions: () => encodingOptions,
67
- filterMatchObject: () => import_chunk_HOPOFWAL.filterMatchObject,
68
- filterMatchValue: () => import_chunk_HOPOFWAL.filterMatchValue,
69
- findInlineObjectOfType: () => findInlineObjectOfType,
70
- getSpaceIdFromCollectionId: () => getSpaceIdFromCollectionId,
71
- hasInvitationExpired: () => import_chunk_JXX6LF5U.hasInvitationExpired,
72
- mapFeedIndexesToTimeframe: () => import_chunk_JXX6LF5U.mapFeedIndexesToTimeframe,
73
- mapTimeframeToFeedIndexes: () => import_chunk_JXX6LF5U.mapTimeframeToFeedIndexes,
74
- startAfter: () => import_chunk_JXX6LF5U.startAfter,
75
- valueEncoding: () => import_chunk_JXX6LF5U.valueEncoding
76
- });
77
- module.exports = __toCommonJS(node_exports);
78
- var import_chunk_HOPOFWAL = require("./chunk-HOPOFWAL.cjs");
79
- var import_chunk_JXX6LF5U = require("./chunk-JXX6LF5U.cjs");
80
- var import_chunk_Q7SFCCGT = require("./chunk-Q7SFCCGT.cjs");
81
- var import_async = require("@dxos/async");
82
- var import_stream = require("@dxos/codec-protobuf/stream");
83
- var import_invariant = require("@dxos/invariant");
84
- var import_keys = require("@dxos/keys");
85
- var import_log = require("@dxos/log");
86
- var import_automerge = require("@automerge/automerge");
87
- var import_async2 = require("@dxos/async");
88
- var import_context = require("@dxos/context");
89
- var import_invariant2 = require("@dxos/invariant");
90
- var import_log2 = require("@dxos/log");
91
- var import_automerge2 = require("@automerge/automerge");
92
- var import_automerge_repo = require("@automerge/automerge-repo");
93
- var import_async3 = require("@dxos/async");
94
- var import_context2 = require("@dxos/context");
95
- var import_echo_protocol = require("@dxos/echo-protocol");
96
- var import_invariant3 = require("@dxos/invariant");
97
- var import_keys2 = require("@dxos/keys");
98
- var import_log3 = require("@dxos/log");
99
- var import_protocols = require("@dxos/protocols");
100
- var import_tracing = require("@dxos/tracing");
101
- var import_util = require("@dxos/util");
102
- var import_automerge3 = require("@automerge/automerge");
103
- var import_async4 = require("@dxos/async");
104
- var import_context3 = require("@dxos/context");
105
- var import_log4 = require("@dxos/log");
106
- var import_tracing2 = require("@dxos/tracing");
107
- var import_util2 = require("@dxos/util");
108
- var import_automerge_repo2 = require("@automerge/automerge-repo");
109
- var import_async5 = require("@dxos/async");
110
- var import_context4 = require("@dxos/context");
111
- var import_invariant4 = require("@dxos/invariant");
112
- var import_log5 = require("@dxos/log");
113
- var import_util3 = require("@dxos/util");
114
- var import_protocols2 = require("@dxos/protocols");
115
- var import_indexing = require("@dxos/indexing");
116
- var import_context5 = require("@dxos/context");
117
- var import_invariant5 = require("@dxos/invariant");
118
- var import_keys3 = require("@dxos/keys");
119
- var import_log6 = require("@dxos/log");
120
- var import_util4 = require("@dxos/util");
121
- var A = __toESM(require("@automerge/automerge"));
122
- var import_automerge_repo3 = require("@automerge/automerge-repo");
123
- var import_context6 = require("@dxos/context");
124
- var import_invariant6 = require("@dxos/invariant");
125
- var import_log7 = require("@dxos/log");
126
- var import_teleport_extension_automerge_replicator = require("@dxos/teleport-extension-automerge-replicator");
127
- var import_invariant7 = require("@dxos/invariant");
128
- var import_keys4 = require("@dxos/keys");
129
- var import_tracing3 = require("@dxos/tracing");
130
- var import_util5 = require("@dxos/util");
131
- var import_context7 = require("@dxos/context");
132
- var import_debug = require("@dxos/debug");
133
- var import_echo_protocol2 = require("@dxos/echo-protocol");
134
- var import_indexing2 = require("@dxos/indexing");
135
- var import_invariant8 = require("@dxos/invariant");
136
- var import_indexing3 = require("@dxos/protocols/proto/dxos/echo/indexing");
137
- var import_tracing4 = require("@dxos/tracing");
138
- var A3 = __toESM(require("@automerge/automerge"));
139
- var import_context8 = require("@dxos/context");
140
- var import_echo_protocol3 = require("@dxos/echo-protocol");
141
- var import_invariant9 = require("@dxos/invariant");
142
- var import_log8 = require("@dxos/log");
143
- var import_protocols3 = require("@dxos/protocols");
144
- var import_automerge4 = require("@automerge/automerge");
145
- var import_effect = require("effect");
146
- var import_async6 = require("@dxos/async");
147
- var import_stream2 = require("@dxos/codec-protobuf/stream");
148
- var import_context9 = require("@dxos/context");
149
- var import_debug2 = require("@dxos/debug");
150
- var import_echo_protocol4 = require("@dxos/echo-protocol");
151
- var import_log9 = require("@dxos/log");
152
- var import_protocols4 = require("@dxos/protocols");
153
- var import_tracing5 = require("@dxos/tracing");
154
- var import_effect2 = require("effect");
155
- var import_context10 = require("@dxos/context");
156
- var import_echo_protocol5 = require("@dxos/echo-protocol");
157
- var import_indexing4 = require("@dxos/indexing");
158
- var import_invariant10 = require("@dxos/invariant");
159
- var import_keys5 = require("@dxos/keys");
160
- var import_log10 = require("@dxos/log");
161
- var import_protocols5 = require("@dxos/protocols");
162
- var import_util6 = require("@dxos/util");
163
- var import_invariant11 = require("@dxos/invariant");
164
- var import_errors = require("@dxos/errors");
165
- var import_automerge_repo4 = require("@automerge/automerge-repo");
166
- var import_lodash = __toESM(require("lodash.isequal"));
167
- var import_async7 = require("@dxos/async");
168
- var import_context11 = require("@dxos/context");
169
- var import_invariant12 = require("@dxos/invariant");
170
- var import_automerge_repo5 = require("@automerge/automerge-repo");
171
- var import_echo_protocol6 = require("@dxos/echo-protocol");
172
- var import_invariant13 = require("@dxos/invariant");
173
- var A4 = __toESM(require("@automerge/automerge"));
174
- var import_log11 = require("@dxos/log");
175
- var import_automerge_repo6 = require("@automerge/automerge-repo");
176
- var import_async8 = require("@dxos/async");
177
- var import_context12 = require("@dxos/context");
178
- var import_crypto = require("@dxos/crypto");
179
- var import_invariant14 = require("@dxos/invariant");
180
- var import_log12 = require("@dxos/log");
181
- var import_protocols6 = require("@dxos/protocols");
182
- var import_buf = require("@dxos/protocols/buf");
183
- var import_messenger_pb = require("@dxos/protocols/buf/dxos/edge/messenger_pb");
184
- var import_util7 = require("@dxos/util");
185
- var import_async9 = require("@dxos/async");
186
- var import_context13 = require("@dxos/context");
187
- var import_log13 = require("@dxos/log");
188
- var import_echo_protocol7 = require("@dxos/echo-protocol");
189
- function _ts_decorate(decorators, target, key, desc) {
190
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
191
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
192
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
193
- return c > 3 && r && Object.defineProperty(target, key, r), r;
194
- }
195
- var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/collection-synchronizer.ts";
196
- var MIN_QUERY_INTERVAL = 5e3;
197
- var POLL_INTERVAL = 3e4;
198
- var CollectionSynchronizer = class extends import_context3.Resource {
199
- constructor(params) {
200
- super();
201
- this._perCollectionStates = /* @__PURE__ */ new Map();
202
- this._activeCollections = /* @__PURE__ */ new Set();
203
- this._connectedPeers = /* @__PURE__ */ new Set();
204
- this.remoteStateUpdated = new import_async4.Event();
205
- this._sendCollectionState = params.sendCollectionState;
206
- this._queryCollectionState = params.queryCollectionState;
207
- this._shouldSyncCollection = params.shouldSyncCollection;
208
- }
209
- async _open(ctx) {
210
- (0, import_async4.scheduleTaskInterval)(this._ctx, async () => {
211
- for (const collectionId of this._perCollectionStates.keys()) {
212
- if (this._activeCollections.has(collectionId)) {
213
- this.refreshCollection(collectionId);
214
- await (0, import_async4.asyncReturn)();
215
- }
216
- }
217
- }, POLL_INTERVAL);
218
- }
219
- getRegisteredCollectionIds() {
220
- return [
221
- ...this._activeCollections
222
- ];
223
- }
224
- getLocalCollectionState(collectionId) {
225
- return this._perCollectionStates.get(collectionId)?.localState;
226
- }
227
- setLocalCollectionState(collectionId, state) {
228
- this._activeCollections.add(collectionId);
229
- (0, import_log4.log)("setLocalCollectionState", {
230
- collectionId,
231
- state
232
- }, {
233
- F: __dxlog_file,
234
- L: 76,
235
- S: this,
236
- C: (f, a) => f(...a)
237
- });
238
- this._getOrCreatePerCollectionState(collectionId).localState = state;
239
- queueMicrotask(async () => {
240
- if (!this._ctx.disposed && this._activeCollections.has(collectionId)) {
241
- this._refreshInterestedPeers(collectionId);
242
- this.refreshCollection(collectionId);
243
- }
244
- });
245
- }
246
- clearLocalCollectionState(collectionId) {
247
- this._activeCollections.delete(collectionId);
248
- this._perCollectionStates.delete(collectionId);
249
- (0, import_log4.log)("clearLocalCollectionState", {
250
- collectionId
251
- }, {
252
- F: __dxlog_file,
253
- L: 90,
254
- S: this,
255
- C: (f, a) => f(...a)
256
- });
257
- }
258
- getRemoteCollectionStates(collectionId) {
259
- return this._getOrCreatePerCollectionState(collectionId).remoteStates;
260
- }
261
- refreshCollection(collectionId) {
262
- let scheduleAnotherRefresh = false;
263
- const state = this._getOrCreatePerCollectionState(collectionId);
264
- for (const peerId of this._connectedPeers) {
265
- if (state.interestedPeers.has(peerId)) {
266
- const lastQueried = state.lastQueried.get(peerId) ?? 0;
267
- if (Date.now() - lastQueried > MIN_QUERY_INTERVAL) {
268
- state.lastQueried.set(peerId, Date.now());
269
- this._queryCollectionState(collectionId, peerId);
270
- } else {
271
- scheduleAnotherRefresh = true;
272
- }
273
- }
274
- }
275
- if (scheduleAnotherRefresh) {
276
- (0, import_async4.scheduleTask)(this._ctx, () => this.refreshCollection(collectionId), MIN_QUERY_INTERVAL);
277
- }
278
- }
279
- /**
280
- * Callback when a connection to a peer is established.
281
- */
282
- onConnectionOpen(peerId) {
283
- const spanId = getSpanName(peerId);
284
- import_tracing2.trace.spanStart({
285
- id: spanId,
286
- methodName: spanId,
287
- instance: this,
288
- parentCtx: this._ctx,
289
- showInBrowserTimeline: true,
290
- attributes: {
291
- peerId
292
- }
293
- });
294
- this._connectedPeers.add(peerId);
295
- queueMicrotask(async () => {
296
- if (this._ctx.disposed) {
297
- return;
298
- }
299
- for (const [collectionId, state] of this._perCollectionStates.entries()) {
300
- if (this._activeCollections.has(collectionId) && this._shouldSyncCollection(collectionId, peerId)) {
301
- state.interestedPeers.add(peerId);
302
- state.lastQueried.set(peerId, Date.now());
303
- this._queryCollectionState(collectionId, peerId);
304
- }
305
- }
306
- });
307
- }
308
- /**
309
- * Callback when a connection to a peer is closed.
310
- */
311
- onConnectionClosed(peerId) {
312
- this._connectedPeers.delete(peerId);
313
- for (const perCollectionState of this._perCollectionStates.values()) {
314
- perCollectionState.remoteStates.delete(peerId);
315
- }
316
- }
317
- /**
318
- * Callback when a peer queries the state of a collection.
319
- */
320
- onCollectionStateQueried(collectionId, peerId) {
321
- const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
322
- if (perCollectionState.localState) {
323
- this._sendCollectionState(collectionId, peerId, perCollectionState.localState);
324
- }
325
- }
326
- /**
327
- * Callback when a peer sends the state of a collection.
328
- */
329
- onRemoteStateReceived(collectionId, peerId, state) {
330
- (0, import_log4.log)("onRemoteStateReceived", {
331
- collectionId,
332
- peerId,
333
- state
334
- }, {
335
- F: __dxlog_file,
336
- L: 171,
337
- S: this,
338
- C: (f, a) => f(...a)
339
- });
340
- validateCollectionState(state);
341
- const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
342
- const existingState = perCollectionState.remoteStates.get(peerId) ?? {
343
- documents: {}
344
- };
345
- const diff = diffCollectionState(existingState, state);
346
- const spanId = getSpanName(peerId);
347
- if (diff.different.length === 0) {
348
- import_tracing2.trace.spanEnd(spanId);
349
- } else {
350
- import_tracing2.trace.spanStart({
351
- id: spanId,
352
- methodName: spanId,
353
- instance: this,
354
- parentCtx: this._ctx,
355
- showInBrowserTimeline: true,
356
- attributes: {
357
- peerId
358
- }
359
- });
360
- }
361
- if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
362
- perCollectionState.remoteStates.set(peerId, state);
363
- this.remoteStateUpdated.emit({
364
- peerId,
365
- collectionId,
366
- newDocsAppeared: diff.missingOnLocal.length > 0
367
- });
368
- }
369
- }
370
- _getOrCreatePerCollectionState(collectionId) {
371
- return (0, import_util2.defaultMap)(this._perCollectionStates, collectionId, () => ({
372
- localState: void 0,
373
- remoteStates: /* @__PURE__ */ new Map(),
374
- interestedPeers: /* @__PURE__ */ new Set(),
375
- lastQueried: /* @__PURE__ */ new Map()
376
- }));
377
- }
378
- _refreshInterestedPeers(collectionId) {
379
- for (const peerId of this._connectedPeers) {
380
- if (this._shouldSyncCollection(collectionId, peerId)) {
381
- this._getOrCreatePerCollectionState(collectionId).interestedPeers.add(peerId);
382
- } else {
383
- this._getOrCreatePerCollectionState(collectionId).interestedPeers.delete(peerId);
384
- }
385
- }
386
- }
387
- };
388
- CollectionSynchronizer = _ts_decorate([
389
- import_tracing2.trace.resource()
390
- ], CollectionSynchronizer);
391
- var diffCollectionState = (local, remote) => {
392
- const allDocuments = /* @__PURE__ */ new Set([
393
- ...Object.keys(local.documents),
394
- ...Object.keys(remote.documents)
395
- ]);
396
- const missingOnRemote = [];
397
- const missingOnLocal = [];
398
- const different = [];
399
- for (const documentId of allDocuments) {
400
- if (!local.documents[documentId]) {
401
- missingOnLocal.push(documentId);
402
- } else if (!remote.documents[documentId]) {
403
- missingOnRemote.push(documentId);
404
- } else if (!import_automerge3.next.equals(local.documents[documentId], remote.documents[documentId])) {
405
- different.push(documentId);
406
- }
407
- }
408
- return {
409
- missingOnRemote,
410
- missingOnLocal,
411
- different
412
- };
413
- };
414
- var validateCollectionState = (state) => {
415
- Object.entries(state.documents).forEach(([documentId, heads]) => {
416
- if (!isValidDocumentId(documentId)) {
417
- throw new Error(`Invalid documentId: ${documentId}`);
418
- }
419
- if (Array.isArray(heads) && heads.some((head) => typeof head !== "string")) {
420
- throw new Error(`Invalid heads: ${heads}`);
421
- }
422
- });
423
- };
424
- var isValidDocumentId = (documentId) => {
425
- return typeof documentId === "string" && !documentId.includes(":");
426
- };
427
- var getSpanName = (peerId) => {
428
- return `collection-sync-${peerId}`;
429
- };
430
- var isCollectionQueryMessage = (message) => message.type === import_protocols2.MESSAGE_TYPE_COLLECTION_QUERY;
431
- var isCollectionStateMessage = (message) => message.type === import_protocols2.MESSAGE_TYPE_COLLECTION_STATE;
432
- function _ts_decorate2(decorators, target, key, desc) {
433
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
434
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
435
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
436
- return c > 3 && r && Object.defineProperty(target, key, r), r;
437
- }
438
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
439
- var EchoNetworkAdapter = class extends import_automerge_repo2.NetworkAdapter {
440
- constructor(_params) {
441
- super(), this._params = _params, this._replicators = /* @__PURE__ */ new Set(), this._connections = /* @__PURE__ */ new Map(), this._lifecycleState = import_context4.LifecycleState.CLOSED, this._connected = new import_async5.Trigger(), this._ready = new import_async5.Trigger();
442
- }
443
- isReady() {
444
- return this._lifecycleState === import_context4.LifecycleState.OPEN;
445
- }
446
- whenReady() {
447
- return this._ready.wait();
448
- }
449
- connect(peerId, peerMetadata) {
450
- this.peerId = peerId;
451
- this.peerMetadata = peerMetadata;
452
- this._connected.wake();
453
- }
454
- send(message) {
455
- this._send(message);
456
- }
457
- disconnect() {
458
- }
459
- async open() {
460
- if (this._lifecycleState === import_context4.LifecycleState.OPEN) {
461
- return;
462
- }
463
- this._lifecycleState = import_context4.LifecycleState.OPEN;
464
- this._ready.wake();
465
- }
466
- async close() {
467
- if (this._lifecycleState === import_context4.LifecycleState.CLOSED) {
468
- return this;
469
- }
470
- for (const replicator of this._replicators) {
471
- await replicator.disconnect();
472
- }
473
- this._replicators.clear();
474
- this._ready.reset();
475
- this._lifecycleState = import_context4.LifecycleState.CLOSED;
476
- }
477
- async whenConnected() {
478
- await this._connected.wait({
479
- timeout: 1e4
480
- });
481
- }
482
- onConnectionAuthScopeChanged(peer) {
483
- const entry = this._connections.get(peer);
484
- if (entry) {
485
- this._onConnectionAuthScopeChanged(entry.connection);
486
- }
487
- }
488
- async addReplicator(replicator) {
489
- (0, import_invariant4.invariant)(this._lifecycleState === import_context4.LifecycleState.OPEN, void 0, {
490
- F: __dxlog_file2,
491
- L: 129,
492
- S: this,
493
- A: [
494
- "this._lifecycleState === LifecycleState.OPEN",
495
- ""
496
- ]
497
- });
498
- (0, import_invariant4.invariant)(this.peerId, void 0, {
499
- F: __dxlog_file2,
500
- L: 130,
501
- S: this,
502
- A: [
503
- "this.peerId",
504
- ""
505
- ]
506
- });
507
- (0, import_invariant4.invariant)(!this._replicators.has(replicator), void 0, {
508
- F: __dxlog_file2,
509
- L: 131,
510
- S: this,
511
- A: [
512
- "!this._replicators.has(replicator)",
513
- ""
514
- ]
515
- });
516
- this._replicators.add(replicator);
517
- await replicator.connect({
518
- peerId: this.peerId,
519
- onConnectionOpen: this._onConnectionOpen.bind(this),
520
- onConnectionClosed: this._onConnectionClosed.bind(this),
521
- onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
522
- isDocumentInRemoteCollection: this._params.isDocumentInRemoteCollection,
523
- getContainingSpaceForDocument: this._params.getContainingSpaceForDocument,
524
- getContainingSpaceIdForDocument: async (documentId) => {
525
- const key = await this._params.getContainingSpaceForDocument(documentId);
526
- return key ? (0, import_chunk_JXX6LF5U.createIdFromSpaceKey)(key) : null;
527
- }
528
- });
529
- }
530
- async removeReplicator(replicator) {
531
- (0, import_invariant4.invariant)(this._lifecycleState === import_context4.LifecycleState.OPEN, void 0, {
532
- F: __dxlog_file2,
533
- L: 150,
534
- S: this,
535
- A: [
536
- "this._lifecycleState === LifecycleState.OPEN",
537
- ""
538
- ]
539
- });
540
- (0, import_invariant4.invariant)(this._replicators.has(replicator), void 0, {
541
- F: __dxlog_file2,
542
- L: 151,
543
- S: this,
544
- A: [
545
- "this._replicators.has(replicator)",
546
- ""
547
- ]
548
- });
549
- await replicator.disconnect();
550
- this._replicators.delete(replicator);
551
- }
552
- async shouldAdvertise(peerId, params) {
553
- const connection = this._connections.get(peerId);
554
- if (!connection) {
555
- return false;
556
- }
557
- return connection.connection.shouldAdvertise(params);
558
- }
559
- shouldSyncCollection(peerId, params) {
560
- const connection = this._connections.get(peerId);
561
- if (!connection) {
562
- return false;
563
- }
564
- return connection.connection.shouldSyncCollection(params);
565
- }
566
- queryCollectionState(collectionId, targetId) {
567
- const message = {
568
- type: "collection-query",
569
- senderId: this.peerId,
570
- targetId,
571
- collectionId
572
- };
573
- this._send(message);
574
- }
575
- sendCollectionState(collectionId, targetId, state) {
576
- const message = {
577
- type: "collection-state",
578
- senderId: this.peerId,
579
- targetId,
580
- collectionId,
581
- state
582
- };
583
- this._send(message);
584
- }
585
- // TODO(dmaretskyi): Remove.
586
- getPeersInterestedInCollection(collectionId) {
587
- return Array.from(this._connections.values()).map((connection) => {
588
- return connection.connection.shouldSyncCollection({
589
- collectionId
590
- }) ? connection.connection.peerId : null;
591
- }).filter(import_util3.isNonNullable);
592
- }
593
- _send(message) {
594
- const connectionEntry = this._connections.get(message.targetId);
595
- if (!connectionEntry) {
596
- throw new Error("Connection not found.");
597
- }
598
- const start = Date.now();
599
- connectionEntry.writer.write(message).then(() => {
600
- this._params.monitor?.recordMessageSent(message, Date.now() - start);
601
- }).catch((err) => {
602
- if (connectionEntry.isOpen) {
603
- import_log5.log.catch(err, void 0, {
604
- F: __dxlog_file2,
605
- L: 221,
606
- S: this,
607
- C: (f, a) => f(...a)
608
- });
609
- }
610
- this._params.monitor?.recordMessageSendingFailed(message);
611
- });
612
- }
613
- _onConnectionOpen(connection) {
614
- (0, import_log5.log)("connection opened", {
615
- peerId: connection.peerId
616
- }, {
617
- F: __dxlog_file2,
618
- L: 229,
619
- S: this,
620
- C: (f, a) => f(...a)
621
- });
622
- (0, import_invariant4.invariant)(!this._connections.has(connection.peerId), void 0, {
623
- F: __dxlog_file2,
624
- L: 230,
625
- S: this,
626
- A: [
627
- "!this._connections.has(connection.peerId as PeerId)",
628
- ""
629
- ]
630
- });
631
- const connectionEntry = {
632
- isOpen: true,
633
- connection,
634
- reader: connection.readable.getReader(),
635
- writer: connection.writable.getWriter()
636
- };
637
- this._connections.set(connection.peerId, connectionEntry);
638
- queueMicrotask(async () => {
639
- try {
640
- while (true) {
641
- const { done, value } = await connectionEntry.reader.read();
642
- if (done) {
643
- break;
644
- }
645
- this._onMessage(value);
646
- }
647
- } catch (err) {
648
- if (connectionEntry.isOpen) {
649
- import_log5.log.catch(err, void 0, {
650
- F: __dxlog_file2,
651
- L: 254,
652
- S: this,
653
- C: (f, a) => f(...a)
654
- });
655
- }
656
- }
657
- });
658
- (0, import_log5.log)("emit peer-candidate", {
659
- peerId: connection.peerId
660
- }, {
661
- F: __dxlog_file2,
662
- L: 259,
663
- S: this,
664
- C: (f, a) => f(...a)
665
- });
666
- this._emitPeerCandidate(connection);
667
- this._params.monitor?.recordPeerConnected(connection.peerId);
668
- }
669
- _onMessage(message) {
670
- if (isCollectionQueryMessage(message)) {
671
- this._params.onCollectionStateQueried(message.collectionId, message.senderId);
672
- } else if (isCollectionStateMessage(message)) {
673
- this._params.onCollectionStateReceived(message.collectionId, message.senderId, message.state);
674
- } else {
675
- this.emit("message", message);
676
- }
677
- this._params.monitor?.recordMessageReceived(message);
678
- }
679
- _onConnectionClosed(connection) {
680
- (0, import_log5.log)("connection closed", {
681
- peerId: connection.peerId
682
- }, {
683
- F: __dxlog_file2,
684
- L: 276,
685
- S: this,
686
- C: (f, a) => f(...a)
687
- });
688
- const entry = this._connections.get(connection.peerId);
689
- (0, import_invariant4.invariant)(entry, void 0, {
690
- F: __dxlog_file2,
691
- L: 278,
692
- S: this,
693
- A: [
694
- "entry",
695
- ""
696
- ]
697
- });
698
- entry.isOpen = false;
699
- this.emit("peer-disconnected", {
700
- peerId: connection.peerId
701
- });
702
- this._params.monitor?.recordPeerDisconnected(connection.peerId);
703
- void entry.reader.cancel().catch((err) => import_log5.log.catch(err, void 0, {
704
- F: __dxlog_file2,
705
- L: 284,
706
- S: this,
707
- C: (f, a) => f(...a)
708
- }));
709
- void entry.writer.abort().catch((err) => import_log5.log.catch(err, void 0, {
710
- F: __dxlog_file2,
711
- L: 285,
712
- S: this,
713
- C: (f, a) => f(...a)
714
- }));
715
- this._connections.delete(connection.peerId);
716
- }
717
- /**
718
- * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
719
- * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
720
- */
721
- _onConnectionAuthScopeChanged(connection) {
722
- (0, import_log5.log)("Connection auth scope changed", {
723
- peerId: connection.peerId
724
- }, {
725
- F: __dxlog_file2,
726
- L: 294,
727
- S: this,
728
- C: (f, a) => f(...a)
729
- });
730
- const entry = this._connections.get(connection.peerId);
731
- (0, import_invariant4.invariant)(entry, void 0, {
732
- F: __dxlog_file2,
733
- L: 296,
734
- S: this,
735
- A: [
736
- "entry",
737
- ""
738
- ]
739
- });
740
- this.emit("peer-disconnected", {
741
- peerId: connection.peerId
742
- });
743
- this._emitPeerCandidate(connection);
744
- }
745
- _emitPeerCandidate(connection) {
746
- this.emit("peer-candidate", {
747
- peerId: connection.peerId,
748
- peerMetadata: createEchoPeerMetadata()
749
- });
750
- }
751
- };
752
- _ts_decorate2([
753
- import_async5.synchronized
754
- ], EchoNetworkAdapter.prototype, "open", null);
755
- _ts_decorate2([
756
- import_async5.synchronized
757
- ], EchoNetworkAdapter.prototype, "close", null);
758
- _ts_decorate2([
759
- import_async5.synchronized
760
- ], EchoNetworkAdapter.prototype, "addReplicator", null);
761
- _ts_decorate2([
762
- import_async5.synchronized
763
- ], EchoNetworkAdapter.prototype, "removeReplicator", null);
764
- var createEchoPeerMetadata = () => ({
765
- // TODO(dmaretskyi): Refactor this.
766
- dxos_peerSource: "EchoNetworkAdapter"
767
- });
768
- var isEchoPeerMetadata = (metadata) => metadata?.dxos_peerSource === "EchoNetworkAdapter";
769
- var HeadsStore = class {
770
- constructor({ db }) {
771
- this._db = db;
772
- }
773
- setHeads(documentId, heads, batch) {
774
- batch.put(documentId, heads, {
775
- sublevel: this._db,
776
- keyEncoding: "utf8",
777
- valueEncoding: import_indexing.headsEncoding
778
- });
779
- }
780
- // TODO(dmaretskyi): Make batched.
781
- async getHeads(documentIds) {
782
- return this._db.getMany(documentIds, {
783
- keyEncoding: "utf8",
784
- valueEncoding: import_indexing.headsEncoding
785
- });
786
- }
787
- };
788
- var LevelDBStorageAdapter = class extends import_context5.Resource {
789
- constructor(_params) {
790
- super(), this._params = _params;
791
- }
792
- async load(keyArray) {
793
- try {
794
- if (this._lifecycleState !== import_context5.LifecycleState.OPEN) {
795
- return void 0;
796
- }
797
- const startMs = Date.now();
798
- const chunk = await this._params.db.get(keyArray, {
799
- ...encodingOptions
800
- });
801
- this._params.monitor?.recordBytesLoaded(chunk.byteLength);
802
- this._params.monitor?.recordLoadDuration(Date.now() - startMs);
803
- return chunk;
804
- } catch (err) {
805
- if (isLevelDbNotFoundError(err)) {
806
- return void 0;
807
- }
808
- throw err;
809
- }
810
- }
811
- async save(keyArray, binary) {
812
- if (this._lifecycleState !== import_context5.LifecycleState.OPEN) {
813
- return void 0;
814
- }
815
- const startMs = Date.now();
816
- const batch = this._params.db.batch();
817
- await this._params.callbacks?.beforeSave?.({
818
- path: keyArray,
819
- batch
820
- });
821
- batch.put(keyArray, Buffer.from(binary), {
822
- ...encodingOptions
823
- });
824
- await batch.write();
825
- this._params.monitor?.recordBytesStored(binary.byteLength);
826
- await this._params.callbacks?.afterSave?.(keyArray);
827
- this._params.monitor?.recordStoreDuration(Date.now() - startMs);
828
- }
829
- async remove(keyArray) {
830
- if (this._lifecycleState !== import_context5.LifecycleState.OPEN) {
831
- return void 0;
832
- }
833
- await this._params.db.del(keyArray, {
834
- ...encodingOptions
835
- });
836
- }
837
- async loadRange(keyPrefix) {
838
- if (this._lifecycleState !== import_context5.LifecycleState.OPEN) {
839
- return [];
840
- }
841
- const startMs = Date.now();
842
- const result = [];
843
- for await (const [key, value] of this._params.db.iterator({
844
- gte: keyPrefix,
845
- lte: [
846
- ...keyPrefix,
847
- "\uFFFF"
848
- ],
849
- ...encodingOptions
850
- })) {
851
- result.push({
852
- key,
853
- data: value
854
- });
855
- this._params.monitor?.recordBytesLoaded(value.byteLength);
856
- }
857
- this._params.monitor?.recordLoadDuration(Date.now() - startMs);
858
- return result;
859
- }
860
- async removeRange(keyPrefix) {
861
- if (this._lifecycleState !== import_context5.LifecycleState.OPEN) {
862
- return void 0;
863
- }
864
- const batch = this._params.db.batch();
865
- for await (const [key] of this._params.db.iterator({
866
- gte: keyPrefix,
867
- lte: [
868
- ...keyPrefix,
869
- "\uFFFF"
870
- ],
871
- ...encodingOptions
872
- })) {
873
- batch.del(key, {
874
- ...encodingOptions
875
- });
876
- }
877
- await batch.write();
878
- }
879
- };
880
- var keyEncoder = {
881
- encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
882
- decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
883
- format: "buffer"
884
- };
885
- var encodingOptions = {
886
- keyEncoding: keyEncoder,
887
- valueEncoding: "buffer"
888
- };
889
- var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
890
- function _ts_decorate3(decorators, target, key, desc) {
891
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
892
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
893
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
894
- return c > 3 && r && Object.defineProperty(target, key, r), r;
895
- }
896
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
897
- var FIND_PARAMS = {
898
- allowableStates: [
899
- "ready",
900
- "requesting"
901
- ]
902
- };
903
- var AutomergeHost = class extends import_context2.Resource {
904
- constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider, getSpaceKeyByRootDocumentId }) {
905
- super();
906
- this._collectionSynchronizer = new CollectionSynchronizer({
907
- queryCollectionState: this._queryCollectionState.bind(this),
908
- sendCollectionState: this._sendCollectionState.bind(this),
909
- shouldSyncCollection: this._shouldSyncCollection.bind(this)
910
- });
911
- this.collectionStateUpdated = new import_async3.Event();
912
- this.documentsSaved = new import_async3.Event();
913
- this._db = db;
914
- this._storage = new LevelDBStorageAdapter({
915
- db: db.sublevel("automerge"),
916
- callbacks: {
917
- beforeSave: async (params) => this._beforeSave(params),
918
- afterSave: async (key) => this._afterSave(key)
919
- },
920
- monitor: dataMonitor
921
- });
922
- this._echoNetworkAdapter = new EchoNetworkAdapter({
923
- getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
924
- isDocumentInRemoteCollection: this._isDocumentInRemoteCollection.bind(this),
925
- onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
926
- onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
927
- monitor: dataMonitor
928
- });
929
- this._headsStore = new HeadsStore({
930
- db: db.sublevel("heads")
931
- });
932
- this._indexMetadataStore = indexMetadataStore;
933
- this._peerIdProvider = peerIdProvider;
934
- this._getSpaceKeyByRootDocumentId = getSpaceKeyByRootDocumentId;
935
- }
936
- async _open() {
937
- this._peerId = `host-${this._peerIdProvider?.() ?? import_keys2.PublicKey.random().toHex()}`;
938
- await this._storage.open?.();
939
- this._repo = new import_automerge_repo.Repo({
940
- peerId: this._peerId,
941
- sharePolicy: this._sharePolicy.bind(this),
942
- storage: this._storage,
943
- network: [
944
- // Upstream swarm.
945
- this._echoNetworkAdapter
946
- ]
947
- });
948
- let updatingAuthScope = false;
949
- import_async3.Event.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => !updatingAuthScope && this._onPeerConnected(e.peerId));
950
- import_async3.Event.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => !updatingAuthScope && this._onPeerDisconnected(e.peerId));
951
- this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId, newDocsAppeared }) => {
952
- this._onRemoteCollectionStateUpdated(collectionId, peerId);
953
- this.collectionStateUpdated.emit({
954
- collectionId
955
- });
956
- if (newDocsAppeared) {
957
- updatingAuthScope = true;
958
- try {
959
- this._echoNetworkAdapter.onConnectionAuthScopeChanged(peerId);
960
- } finally {
961
- updatingAuthScope = false;
962
- }
963
- }
964
- });
965
- await this._echoNetworkAdapter.open();
966
- await this._collectionSynchronizer.open();
967
- await this._echoNetworkAdapter.open();
968
- await this._echoNetworkAdapter.whenConnected();
969
- }
970
- async _close() {
971
- await this._collectionSynchronizer.close();
972
- await this._storage.close?.();
973
- await this._echoNetworkAdapter.close();
974
- await this._ctx.dispose();
975
- }
976
- /**
977
- * @deprecated To be abstracted away.
978
- */
979
- get repo() {
980
- return this._repo;
981
- }
982
- get peerId() {
983
- return this._peerId;
984
- }
985
- get loadedDocsCount() {
986
- return Object.keys(this._repo.handles).length;
987
- }
988
- async addReplicator(replicator) {
989
- await this._echoNetworkAdapter.addReplicator(replicator);
990
- }
991
- async removeReplicator(replicator) {
992
- await this._echoNetworkAdapter.removeReplicator(replicator);
993
- }
994
- /**
995
- * Loads the document handle from the repo and waits for it to be ready.
996
- */
997
- async loadDoc(ctx, documentId, opts) {
998
- let handle;
999
- if (typeof documentId === "string") {
1000
- handle = this._repo.handles[documentId];
1001
- }
1002
- if (!handle) {
1003
- handle = await this._repo.find(documentId, FIND_PARAMS);
1004
- }
1005
- if (!handle.isReady()) {
1006
- if (!opts?.timeout) {
1007
- await (0, import_context2.cancelWithContext)(ctx, handle.whenReady());
1008
- } else {
1009
- await (0, import_context2.cancelWithContext)(ctx, (0, import_async3.asyncTimeout)(handle.whenReady(), opts.timeout));
1010
- }
1011
- }
1012
- return handle;
1013
- }
1014
- async exportDoc(ctx, id) {
1015
- const documentId = (0, import_automerge_repo.interpretAsDocumentId)(id);
1016
- const chunks = await this._storage.loadRange([
1017
- documentId
1018
- ]);
1019
- return (0, import_util.bufferToArray)(Buffer.concat(chunks.map((c) => c.data)));
1020
- }
1021
- /**
1022
- * Create new persisted document.
1023
- */
1024
- createDoc(initialValue, opts) {
1025
- if (opts?.preserveHistory) {
1026
- if (initialValue instanceof Uint8Array) {
1027
- return this._repo.import(initialValue);
1028
- }
1029
- if (!(0, import_automerge2.isAutomerge)(initialValue)) {
1030
- throw new TypeError("Initial value must be an Automerge document");
1031
- }
1032
- return this._repo.import((0, import_automerge2.save)(initialValue));
1033
- } else {
1034
- if (initialValue instanceof Uint8Array) {
1035
- throw new Error("Cannot create document from Uint8Array without preserving history");
1036
- }
1037
- return this._repo.create(initialValue);
1038
- }
1039
- }
1040
- async waitUntilHeadsReplicated(heads) {
1041
- const entries = heads.entries;
1042
- if (!entries?.length) {
1043
- return;
1044
- }
1045
- const documentIds = entries.map((entry) => entry.documentId);
1046
- const documentHeads = await this.getHeads(documentIds);
1047
- const headsToWait = entries.filter((entry, index) => {
1048
- const targetHeads = entry.heads;
1049
- if (!targetHeads || targetHeads.length === 0) {
1050
- return false;
1051
- }
1052
- const currentHeads = documentHeads[index];
1053
- return !(currentHeads !== null && (0, import_automerge2.equals)(currentHeads, targetHeads));
1054
- });
1055
- if (headsToWait.length > 0) {
1056
- await Promise.all(headsToWait.map(async (entry, index) => {
1057
- const handle = await this.loadDoc(import_context2.Context.default(void 0, {
1058
- F: __dxlog_file3,
1059
- L: 293
1060
- }), entry.documentId);
1061
- await waitForHeads(handle, entry.heads);
1062
- }));
1063
- }
1064
- await this._repo.flush(documentIds.filter((documentId) => this._repo.handles[documentId] && this._repo.handles[documentId].isReady()));
1065
- }
1066
- async reIndexHeads(documentIds) {
1067
- for (const documentId of documentIds) {
1068
- (0, import_log3.log)("re-indexing heads for document", {
1069
- documentId
1070
- }, {
1071
- F: __dxlog_file3,
1072
- L: 307,
1073
- S: this,
1074
- C: (f, a) => f(...a)
1075
- });
1076
- const handle = await this._repo.find(documentId, FIND_PARAMS);
1077
- if (!handle.isReady()) {
1078
- import_log3.log.warn("document is not available locally, skipping", {
1079
- documentId
1080
- }, {
1081
- F: __dxlog_file3,
1082
- L: 310,
1083
- S: this,
1084
- C: (f, a) => f(...a)
1085
- });
1086
- continue;
1087
- }
1088
- const heads = handle.heads();
1089
- const batch = this._db.batch();
1090
- this._headsStore.setHeads(documentId, heads, batch);
1091
- await batch.write();
1092
- }
1093
- (0, import_log3.log)("done re-indexing heads", void 0, {
1094
- F: __dxlog_file3,
1095
- L: 319,
1096
- S: this,
1097
- C: (f, a) => f(...a)
1098
- });
1099
- }
1100
- // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
1101
- // Hosts, running in the worker, don't share documents unless requested by other peers.
1102
- // NOTE: If both peers return sharePolicy=false the replication will not happen
1103
- // https://github.com/automerge/automerge-repo/pull/292
1104
- async _sharePolicy(peerId, documentId) {
1105
- if (peerId.startsWith("client-")) {
1106
- return false;
1107
- }
1108
- if (!documentId) {
1109
- return false;
1110
- }
1111
- const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
1112
- if (isEchoPeerMetadata(peerMetadata)) {
1113
- return this._echoNetworkAdapter.shouldAdvertise(peerId, {
1114
- documentId
1115
- });
1116
- }
1117
- return false;
1118
- }
1119
- async _beforeSave({ path, batch }) {
1120
- const handle = this._repo.handles[path[0]];
1121
- if (!handle || !handle.isReady()) {
1122
- return;
1123
- }
1124
- const doc = handle.doc();
1125
- if (!doc) {
1126
- return;
1127
- }
1128
- const heads = (0, import_automerge2.getHeads)(doc);
1129
- this._headsStore.setHeads(handle.documentId, heads, batch);
1130
- const spaceKey = import_echo_protocol.DatabaseDirectory.getSpaceKey(doc) ?? void 0;
1131
- const objectIds = Object.keys(doc.objects ?? {});
1132
- const encodedIds = objectIds.map((objectId) => import_protocols.objectPointerCodec.encode({
1133
- documentId: handle.documentId,
1134
- objectId,
1135
- spaceKey
1136
- }));
1137
- const idToLastHash = new Map(encodedIds.map((id) => [
1138
- id,
1139
- heads
1140
- ]));
1141
- this._indexMetadataStore.markDirty(idToLastHash, batch);
1142
- }
1143
- _shouldSyncCollection(collectionId, peerId) {
1144
- const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
1145
- if (isEchoPeerMetadata(peerMetadata)) {
1146
- return this._echoNetworkAdapter.shouldSyncCollection(peerId, {
1147
- collectionId
1148
- });
1149
- }
1150
- return false;
1151
- }
1152
- /**
1153
- * Called by AutomergeStorageAdapter after levelDB batch commit.
1154
- */
1155
- async _afterSave(path) {
1156
- this._indexMetadataStore.notifyMarkedDirty();
1157
- const documentId = path[0];
1158
- const document = this._repo.handles[documentId]?.doc();
1159
- if (document) {
1160
- const heads = (0, import_automerge2.getHeads)(document);
1161
- this._onHeadsChanged(documentId, heads);
1162
- }
1163
- this.documentsSaved.emit();
1164
- }
1165
- _automergePeers() {
1166
- return this._repo.peers;
1167
- }
1168
- async _isDocumentInRemoteCollection(params) {
1169
- for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
1170
- const remoteCollections = this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
1171
- const remotePeerDocs = remoteCollections.get(params.peerId)?.documents;
1172
- if (remotePeerDocs && params.documentId in remotePeerDocs) {
1173
- return true;
1174
- }
1175
- }
1176
- return false;
1177
- }
1178
- async _getContainingSpaceForDocument(documentId) {
1179
- const handle = this._repo.handles[documentId];
1180
- if (handle.state === "loading") {
1181
- await handle.whenReady();
1182
- }
1183
- if (handle && handle.isReady() && handle.doc()) {
1184
- const spaceKeyHex = import_echo_protocol.DatabaseDirectory.getSpaceKey(handle.doc());
1185
- if (spaceKeyHex) {
1186
- return import_keys2.PublicKey.from(spaceKeyHex);
1187
- }
1188
- }
1189
- const rootDocSpaceKey = this._getSpaceKeyByRootDocumentId?.(documentId);
1190
- if (rootDocSpaceKey) {
1191
- return rootDocSpaceKey;
1192
- }
1193
- return null;
1194
- }
1195
- /**
1196
- * Flush documents to disk.
1197
- */
1198
- async flush({ documentIds } = {}) {
1199
- const loadedDocuments = documentIds?.filter((documentId) => {
1200
- const handle = this._repo.handles[documentId];
1201
- return handle && handle.isReady();
1202
- });
1203
- await this._repo.flush(loadedDocuments);
1204
- }
1205
- async getHeads(documentIds) {
1206
- const result = [];
1207
- const storeRequestIds = [];
1208
- const storeResultIndices = [];
1209
- for (const documentId of documentIds) {
1210
- const handle = this._repo.handles[documentId];
1211
- if (handle && handle.isReady() && handle.doc()) {
1212
- result.push((0, import_automerge2.getHeads)(handle.doc()));
1213
- } else {
1214
- storeRequestIds.push(documentId);
1215
- storeResultIndices.push(result.length);
1216
- result.push(void 0);
1217
- }
1218
- }
1219
- if (storeRequestIds.length > 0) {
1220
- const storedHeads = await this._headsStore.getHeads(storeRequestIds);
1221
- for (let i = 0; i < storedHeads.length; i++) {
1222
- result[storeResultIndices[i]] = storedHeads[i];
1223
- }
1224
- }
1225
- return result;
1226
- }
1227
- //
1228
- // Collection sync.
1229
- //
1230
- getLocalCollectionState(collectionId) {
1231
- return this._collectionSynchronizer.getLocalCollectionState(collectionId);
1232
- }
1233
- getRemoteCollectionStates(collectionId) {
1234
- return this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
1235
- }
1236
- refreshCollection(collectionId) {
1237
- this._collectionSynchronizer.refreshCollection(collectionId);
1238
- }
1239
- async getCollectionSyncState(collectionId) {
1240
- const result = {
1241
- peers: []
1242
- };
1243
- const localState = this.getLocalCollectionState(collectionId);
1244
- const remoteState = this.getRemoteCollectionStates(collectionId);
1245
- if (!localState) {
1246
- return result;
1247
- }
1248
- for (const [peerId, state] of remoteState) {
1249
- const diff = diffCollectionState(localState, state);
1250
- result.peers.push({
1251
- peerId,
1252
- missingOnRemote: diff.missingOnRemote.length,
1253
- missingOnLocal: diff.missingOnLocal.length,
1254
- differentDocuments: diff.different.length,
1255
- localDocumentCount: Object.keys(localState.documents).length,
1256
- remoteDocumentCount: Object.keys(state.documents).length
1257
- });
1258
- }
1259
- return result;
1260
- }
1261
- /**
1262
- * Update the local collection state based on the locally stored document heads.
1263
- */
1264
- async updateLocalCollectionState(collectionId, documentIds) {
1265
- const heads = await this.getHeads(documentIds);
1266
- const documents = Object.fromEntries(heads.map((heads2, index) => [
1267
- documentIds[index],
1268
- heads2 ?? []
1269
- ]));
1270
- this._collectionSynchronizer.setLocalCollectionState(collectionId, {
1271
- documents
1272
- });
1273
- }
1274
- async clearLocalCollectionState(collectionId) {
1275
- this._collectionSynchronizer.clearLocalCollectionState(collectionId);
1276
- }
1277
- _onCollectionStateQueried(collectionId, peerId) {
1278
- this._collectionSynchronizer.onCollectionStateQueried(collectionId, peerId);
1279
- }
1280
- _onCollectionStateReceived(collectionId, peerId, state) {
1281
- this._collectionSynchronizer.onRemoteStateReceived(collectionId, peerId, decodeCollectionState(state));
1282
- }
1283
- _queryCollectionState(collectionId, peerId) {
1284
- this._echoNetworkAdapter.queryCollectionState(collectionId, peerId);
1285
- }
1286
- _sendCollectionState(collectionId, peerId, state) {
1287
- this._echoNetworkAdapter.sendCollectionState(collectionId, peerId, encodeCollectionState(state));
1288
- }
1289
- _onPeerConnected(peerId) {
1290
- this._collectionSynchronizer.onConnectionOpen(peerId);
1291
- }
1292
- _onPeerDisconnected(peerId) {
1293
- this._collectionSynchronizer.onConnectionClosed(peerId);
1294
- }
1295
- _onRemoteCollectionStateUpdated(collectionId, peerId) {
1296
- const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
1297
- const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
1298
- if (!localState || !remoteState) {
1299
- return;
1300
- }
1301
- const { different, missingOnLocal, missingOnRemote } = diffCollectionState(localState, remoteState);
1302
- const toReplicate = [
1303
- ...missingOnLocal,
1304
- ...missingOnRemote,
1305
- ...different
1306
- ];
1307
- if (toReplicate.length === 0) {
1308
- return;
1309
- }
1310
- (0, import_log3.log)("replicating documents after collection sync", {
1311
- collectionId,
1312
- peerId,
1313
- toReplicate,
1314
- count: toReplicate.length
1315
- }, {
1316
- F: __dxlog_file3,
1317
- L: 563,
1318
- S: this,
1319
- C: (f, a) => f(...a)
1320
- });
1321
- for (const documentId of toReplicate) {
1322
- this._repo.findWithProgress(documentId);
1323
- }
1324
- }
1325
- _onHeadsChanged(documentId, heads) {
1326
- const collectionsChanged = /* @__PURE__ */ new Set();
1327
- for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
1328
- const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
1329
- if (state?.documents[documentId]) {
1330
- const newState = structuredClone(state);
1331
- newState.documents[documentId] = heads;
1332
- this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
1333
- collectionsChanged.add(collectionId);
1334
- }
1335
- }
1336
- for (const collectionId of collectionsChanged) {
1337
- this.collectionStateUpdated.emit({
1338
- collectionId
1339
- });
1340
- }
1341
- }
1342
- };
1343
- _ts_decorate3([
1344
- import_tracing.trace.info()
1345
- ], AutomergeHost.prototype, "_peerId", void 0);
1346
- _ts_decorate3([
1347
- import_tracing.trace.info({
1348
- depth: null
1349
- })
1350
- ], AutomergeHost.prototype, "_automergePeers", null);
1351
- _ts_decorate3([
1352
- import_tracing.trace.span({
1353
- showInBrowserTimeline: true
1354
- })
1355
- ], AutomergeHost.prototype, "flush", null);
1356
- AutomergeHost = _ts_decorate3([
1357
- import_tracing.trace.resource()
1358
- ], AutomergeHost);
1359
- var waitForHeads = async (handle, heads) => {
1360
- const unavailableHeads = new Set(heads);
1361
- await handle.whenReady();
1362
- await import_async3.Event.wrap(handle, "change").waitForCondition(() => {
1363
- for (const changeHash of unavailableHeads.values()) {
1364
- if (changeIsPresentInDoc(handle.doc(), changeHash)) {
1365
- unavailableHeads.delete(changeHash);
1366
- }
1367
- }
1368
- return unavailableHeads.size === 0;
1369
- });
1370
- };
1371
- var changeIsPresentInDoc = (doc, changeHash) => {
1372
- return !!(0, import_automerge2.getBackend)(doc).getChangeByHash(changeHash);
1373
- };
1374
- var decodeCollectionState = (state) => {
1375
- (0, import_invariant3.invariant)(typeof state === "object" && state !== null, "Invalid state", {
1376
- F: __dxlog_file3,
1377
- L: 614,
1378
- S: void 0,
1379
- A: [
1380
- "typeof state === 'object' && state !== null",
1381
- "'Invalid state'"
1382
- ]
1383
- });
1384
- return state;
1385
- };
1386
- var encodeCollectionState = (state) => {
1387
- return state;
1388
- };
1389
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
1390
- var DEFAULT_FACTORY = (params) => new import_teleport_extension_automerge_replicator.AutomergeReplicator(...params);
1391
- var MeshReplicatorConnection = class extends import_context6.Resource {
1392
- constructor(_params) {
1393
- super(), this._params = _params, this.remoteDeviceKey = null, this._remotePeerId = null, this._isEnabled = false;
1394
- let readableStreamController;
1395
- this.readable = new ReadableStream({
1396
- start: (controller) => {
1397
- readableStreamController = controller;
1398
- this._ctx.onDispose(() => controller.close());
1399
- }
1400
- });
1401
- this.writable = new WritableStream({
1402
- write: async (message, controller) => {
1403
- (0, import_invariant6.invariant)(this._isEnabled, "Writing to a disabled connection", {
1404
- F: __dxlog_file4,
1405
- L: 51,
1406
- S: this,
1407
- A: [
1408
- "this._isEnabled",
1409
- "'Writing to a disabled connection'"
1410
- ]
1411
- });
1412
- try {
1413
- logSendSync(message);
1414
- await this.replicatorExtension.sendSyncMessage({
1415
- payload: import_automerge_repo3.cbor.encode(message)
1416
- });
1417
- } catch (err) {
1418
- controller.error(err);
1419
- this._disconnectIfEnabled();
1420
- }
1421
- }
1422
- });
1423
- const createAutomergeReplicator = this._params.replicatorFactory ?? DEFAULT_FACTORY;
1424
- this.replicatorExtension = createAutomergeReplicator([
1425
- {
1426
- peerId: this._params.ownPeerId
1427
- },
1428
- {
1429
- onStartReplication: async (info, remotePeerId) => {
1430
- this.remoteDeviceKey = remotePeerId;
1431
- this._remotePeerId = info.id;
1432
- (0, import_log7.log)("onStartReplication", {
1433
- id: info.id,
1434
- thisPeerId: this.peerId,
1435
- remotePeerId: remotePeerId.toHex()
1436
- }, {
1437
- F: __dxlog_file4,
1438
- L: 80,
1439
- S: this,
1440
- C: (f, a) => f(...a)
1441
- });
1442
- this._params.onRemoteConnected();
1443
- },
1444
- onSyncMessage: async ({ payload }) => {
1445
- if (!this._isEnabled) {
1446
- return;
1447
- }
1448
- const message = import_automerge_repo3.cbor.decode(payload);
1449
- readableStreamController.enqueue(message);
1450
- },
1451
- onClose: async () => {
1452
- this._disconnectIfEnabled();
1453
- }
1454
- }
1455
- ]);
1456
- }
1457
- _disconnectIfEnabled() {
1458
- if (this._isEnabled) {
1459
- this._params.onRemoteDisconnected();
1460
- }
1461
- }
1462
- get peerId() {
1463
- (0, import_invariant6.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
1464
- F: __dxlog_file4,
1465
- L: 106,
1466
- S: this,
1467
- A: [
1468
- "this._remotePeerId != null",
1469
- "'Remote peer has not connected yet.'"
1470
- ]
1471
- });
1472
- return this._remotePeerId;
1473
- }
1474
- get isEnabled() {
1475
- return this._isEnabled;
1476
- }
1477
- async shouldAdvertise(params) {
1478
- return this._params.shouldAdvertise(params);
1479
- }
1480
- shouldSyncCollection(params) {
1481
- return this._params.shouldSyncCollection(params);
1482
- }
1483
- /**
1484
- * Start exchanging messages with the remote peer.
1485
- * Call after the remote peer has connected.
1486
- */
1487
- enable() {
1488
- (0, import_invariant6.invariant)(this._remotePeerId != null, "Remote peer has not connected yet.", {
1489
- F: __dxlog_file4,
1490
- L: 127,
1491
- S: this,
1492
- A: [
1493
- "this._remotePeerId != null",
1494
- "'Remote peer has not connected yet.'"
1495
- ]
1496
- });
1497
- this._isEnabled = true;
1498
- }
1499
- /**
1500
- * Stop exchanging messages with the remote peer.
1501
- */
1502
- disable() {
1503
- this._isEnabled = false;
1504
- }
1505
- };
1506
- var logSendSync = (message) => {
1507
- (0, import_log7.log)("sendSyncMessage", () => {
1508
- const decodedSyncMessage = message.type === "sync" && message.data ? A.decodeSyncMessage(message.data) : void 0;
1509
- return {
1510
- sync: decodedSyncMessage && {
1511
- headsLength: decodedSyncMessage.heads.length,
1512
- requesting: decodedSyncMessage.need.length > 0,
1513
- sendingChanges: decodedSyncMessage.changes.length > 0
1514
- },
1515
- type: message.type,
1516
- from: message.senderId,
1517
- to: message.targetId
1518
- };
1519
- }, {
1520
- F: __dxlog_file4,
1521
- L: 140,
1522
- S: void 0,
1523
- C: (f, a) => f(...a)
1524
- });
1525
- };
1526
- var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
1527
- var deriveCollectionIdFromSpaceId = (spaceId, rootDocumentId) => rootDocumentId ? `space:${spaceId}:${rootDocumentId}` : `space:${spaceId}`;
1528
- var getSpaceIdFromCollectionId = (collectionId) => {
1529
- const spaceId = collectionId.split(":")[1];
1530
- (0, import_invariant7.invariant)(import_keys4.SpaceId.isValid(spaceId), void 0, {
1531
- F: __dxlog_file5,
1532
- L: 16,
1533
- S: void 0,
1534
- A: [
1535
- "SpaceId.isValid(spaceId)",
1536
- ""
1537
- ]
1538
- });
1539
- return spaceId;
1540
- };
1541
- var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts";
1542
- var MeshEchoReplicator = class {
1543
- constructor() {
1544
- this._connectionsPerPeer = /* @__PURE__ */ new Map();
1545
- this._connections = /* @__PURE__ */ new Set();
1546
- this._authorizedDevices = /* @__PURE__ */ new Map();
1547
- this._context = null;
1548
- }
1549
- async connect(context) {
1550
- this._context = context;
1551
- }
1552
- async disconnect() {
1553
- for (const connection of this._connections) {
1554
- if (connection.isEnabled) {
1555
- this._context?.onConnectionClosed(connection);
1556
- }
1557
- }
1558
- for (const connection of this._connections) {
1559
- await connection.close();
1560
- }
1561
- this._connections.clear();
1562
- this._connectionsPerPeer.clear();
1563
- this._context = null;
1564
- }
1565
- createExtension(extensionFactory) {
1566
- (0, import_invariant5.invariant)(this._context, void 0, {
1567
- F: __dxlog_file6,
1568
- L: 67,
1569
- S: this,
1570
- A: [
1571
- "this._context",
1572
- ""
1573
- ]
1574
- });
1575
- const connection = new MeshReplicatorConnection({
1576
- ownPeerId: this._context.peerId,
1577
- replicatorFactory: extensionFactory,
1578
- onRemoteConnected: async () => {
1579
- (0, import_log6.log)("onRemoteConnected", {
1580
- peerId: connection.peerId
1581
- }, {
1582
- F: __dxlog_file6,
1583
- L: 73,
1584
- S: this,
1585
- C: (f, a) => f(...a)
1586
- });
1587
- (0, import_invariant5.invariant)(this._context, void 0, {
1588
- F: __dxlog_file6,
1589
- L: 74,
1590
- S: this,
1591
- A: [
1592
- "this._context",
1593
- ""
1594
- ]
1595
- });
1596
- const existingConnections = this._connectionsPerPeer.get(connection.peerId);
1597
- if (existingConnections?.length) {
1598
- const enabledConnection = existingConnections[0];
1599
- this._context.onConnectionAuthScopeChanged(enabledConnection);
1600
- existingConnections.push(connection);
1601
- } else {
1602
- this._connectionsPerPeer.set(connection.peerId, [
1603
- connection
1604
- ]);
1605
- this._context.onConnectionOpen(connection);
1606
- connection.enable();
1607
- }
1608
- },
1609
- onRemoteDisconnected: async () => {
1610
- (0, import_log6.log)("onRemoteDisconnected", {
1611
- peerId: connection.peerId
1612
- }, {
1613
- F: __dxlog_file6,
1614
- L: 88,
1615
- S: this,
1616
- C: (f, a) => f(...a)
1617
- });
1618
- this._connections.delete(connection);
1619
- const existingConnections = this._connectionsPerPeer.get(connection.peerId) ?? [];
1620
- const index = existingConnections.indexOf(connection);
1621
- if (index < 0) {
1622
- import_log6.log.warn("disconnected connection not found", {
1623
- peerId: connection.peerId
1624
- }, {
1625
- F: __dxlog_file6,
1626
- L: 96,
1627
- S: this,
1628
- C: (f, a) => f(...a)
1629
- });
1630
- return;
1631
- }
1632
- existingConnections.splice(index, 1);
1633
- if (connection.isEnabled) {
1634
- this._context?.onConnectionClosed(connection);
1635
- connection.disable();
1636
- if (existingConnections.length > 0) {
1637
- this._context?.onConnectionOpen(existingConnections[0]);
1638
- existingConnections[0].enable();
1639
- }
1640
- }
1641
- },
1642
- shouldAdvertise: async (params) => {
1643
- (0, import_log6.log)("shouldAdvertise", {
1644
- peerId: connection.peerId,
1645
- documentId: params.documentId
1646
- }, {
1647
- F: __dxlog_file6,
1648
- L: 114,
1649
- S: this,
1650
- C: (f, a) => f(...a)
1651
- });
1652
- (0, import_invariant5.invariant)(this._context, void 0, {
1653
- F: __dxlog_file6,
1654
- L: 115,
1655
- S: this,
1656
- A: [
1657
- "this._context",
1658
- ""
1659
- ]
1660
- });
1661
- try {
1662
- const spaceKey = await this._context.getContainingSpaceForDocument(params.documentId);
1663
- if (!spaceKey) {
1664
- const remoteDocumentExists = await this._context.isDocumentInRemoteCollection({
1665
- documentId: params.documentId,
1666
- peerId: connection.peerId
1667
- });
1668
- (0, import_log6.log)("document not found locally for share policy check", {
1669
- peerId: connection.peerId,
1670
- documentId: params.documentId,
1671
- acceptDocument: remoteDocumentExists
1672
- }, {
1673
- F: __dxlog_file6,
1674
- L: 123,
1675
- S: this,
1676
- C: (f, a) => f(...a)
1677
- });
1678
- return remoteDocumentExists;
1679
- }
1680
- const spaceId = await (0, import_chunk_JXX6LF5U.createIdFromSpaceKey)(spaceKey);
1681
- const authorizedDevices = this._authorizedDevices.get(spaceId);
1682
- if (!connection.remoteDeviceKey) {
1683
- (0, import_log6.log)("device key not found for share policy check", {
1684
- peerId: connection.peerId,
1685
- documentId: params.documentId
1686
- }, {
1687
- F: __dxlog_file6,
1688
- L: 140,
1689
- S: this,
1690
- C: (f, a) => f(...a)
1691
- });
1692
- return false;
1693
- }
1694
- const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
1695
- (0, import_log6.log)("share policy check", {
1696
- localPeer: this._context.peerId,
1697
- remotePeer: connection.peerId,
1698
- documentId: params.documentId,
1699
- deviceKey: connection.remoteDeviceKey,
1700
- spaceKey,
1701
- isAuthorized
1702
- }, {
1703
- F: __dxlog_file6,
1704
- L: 148,
1705
- S: this,
1706
- C: (f, a) => f(...a)
1707
- });
1708
- return isAuthorized;
1709
- } catch (err) {
1710
- import_log6.log.catch(err, void 0, {
1711
- F: __dxlog_file6,
1712
- L: 158,
1713
- S: this,
1714
- C: (f, a) => f(...a)
1715
- });
1716
- return false;
1717
- }
1718
- },
1719
- shouldSyncCollection: ({ collectionId }) => {
1720
- const spaceId = getSpaceIdFromCollectionId(collectionId);
1721
- const authorizedDevices = this._authorizedDevices.get(spaceId);
1722
- if (!connection.remoteDeviceKey) {
1723
- (0, import_log6.log)("device key not found for collection sync check", {
1724
- peerId: connection.peerId,
1725
- collectionId
1726
- }, {
1727
- F: __dxlog_file6,
1728
- L: 168,
1729
- S: this,
1730
- C: (f, a) => f(...a)
1731
- });
1732
- return false;
1733
- }
1734
- const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
1735
- return isAuthorized;
1736
- }
1737
- });
1738
- this._connections.add(connection);
1739
- return connection.replicatorExtension;
1740
- }
1741
- async authorizeDevice(spaceKey, deviceKey) {
1742
- (0, import_log6.log)("authorizeDevice", {
1743
- spaceKey,
1744
- deviceKey
1745
- }, {
1746
- F: __dxlog_file6,
1747
- L: 185,
1748
- S: this,
1749
- C: (f, a) => f(...a)
1750
- });
1751
- const spaceId = await (0, import_chunk_JXX6LF5U.createIdFromSpaceKey)(spaceKey);
1752
- (0, import_util4.defaultMap)(this._authorizedDevices, spaceId, () => new import_util4.ComplexSet(import_keys3.PublicKey.hash)).add(deviceKey);
1753
- for (const connection of this._connections) {
1754
- if (connection.isEnabled && connection.remoteDeviceKey && connection.remoteDeviceKey.equals(deviceKey)) {
1755
- if (this._connectionsPerPeer.has(connection.peerId)) {
1756
- this._context?.onConnectionAuthScopeChanged(connection);
1757
- }
1758
- }
1759
- }
1760
- }
1761
- };
1762
- function _ts_decorate4(decorators, target, key, desc) {
1763
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1764
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
1765
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1766
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1767
- }
1768
- var PER_SECOND_RATE_AVG_WINDOW_SIZE = 5;
1769
- var DEFAULT_AVG_WINDOW_SIZE = 25;
1770
- var EchoDataMonitor = class {
1771
- constructor(_params = {
1772
- timeSeriesLength: 30
1773
- }) {
1774
- this._params = _params;
1775
- this._lastTick = 0;
1776
- this._activeCounters = createLocalCounters();
1777
- this._localTimeSeries = createLocalTimeSeries();
1778
- this._storageAverages = createStorageAverages();
1779
- this._replicationAverages = createNetworkAverages();
1780
- this._sizeByMessage = {};
1781
- this._lastReceivedMessages = new import_util5.CircularBuffer(100);
1782
- this._lastSentMessages = new import_util5.CircularBuffer(100);
1783
- this._connectionsCount = 0;
1784
- }
1785
- tick(timeMs) {
1786
- this._advanceTimeWindow(timeMs - this._lastTick);
1787
- this._lastTick = timeMs;
1788
- }
1789
- computeStats() {
1790
- return {
1791
- meta: {
1792
- rateAverageOverSeconds: PER_SECOND_RATE_AVG_WINDOW_SIZE
1793
- },
1794
- storage: {
1795
- reads: {
1796
- payloadSize: this._storageAverages.loadedChunkSize.average(),
1797
- opDuration: this._storageAverages.loadDuration.average(),
1798
- countPerSecond: this._storageAverages.loadsPerSecond.average()
1799
- },
1800
- writes: {
1801
- payloadSize: this._storageAverages.storedChunkSize.average(),
1802
- opDuration: this._storageAverages.storeDuration.average(),
1803
- countPerSecond: this._storageAverages.storesPerSecond.average()
1804
- }
1805
- },
1806
- replicator: {
1807
- connections: this._connectionsCount,
1808
- receivedMessages: {
1809
- payloadSize: this._replicationAverages.receivedMessageSize.average(),
1810
- countPerSecond: this._replicationAverages.receivedPerSecond.average()
1811
- },
1812
- sentMessages: {
1813
- payloadSize: this._replicationAverages.sentMessageSize.average(),
1814
- opDuration: this._replicationAverages.sendDuration.average(),
1815
- countPerSecond: this._replicationAverages.sentPerSecond.average(),
1816
- failedPerSecond: this._replicationAverages.sendsFailedPerSecond.average()
1817
- },
1818
- countByMessage: this._computeMessageHistogram("type"),
1819
- avgSizeByMessage: (0, import_util5.mapValues)(this._sizeByMessage, (summary) => summary.average())
1820
- }
1821
- };
1822
- }
1823
- get connectionsCount() {
1824
- return this._connectionsCount;
1825
- }
1826
- /**
1827
- * @internal
1828
- */
1829
- get lastPerSecondStats() {
1830
- return this._lastCompleteCounters;
1831
- }
1832
- /**
1833
- * @internal
1834
- */
1835
- get timeSeries() {
1836
- return {
1837
- ...this._localTimeSeries.storage,
1838
- ...this._localTimeSeries.replication
1839
- };
1840
- }
1841
- /**
1842
- * @internal
1843
- */
1844
- get messagesByPeerId() {
1845
- return this._computeMessageHistogram("peerId");
1846
- }
1847
- _advanceTimeWindow(millisPassed) {
1848
- const oldMetrics = Object.freeze(this._activeCounters);
1849
- this._activeCounters = createLocalCounters();
1850
- this._lastCompleteCounters = oldMetrics;
1851
- for (const peerId of Object.keys(oldMetrics.byPeerId)) {
1852
- this._activeCounters.byPeerId[peerId] = createMessageCounter();
1853
- }
1854
- this._addToTimeSeries(oldMetrics.replication, this._localTimeSeries.replication);
1855
- this._addToTimeSeries(oldMetrics.storage, this._localTimeSeries.storage);
1856
- if (Math.abs(millisPassed - 1e3) < 100) {
1857
- this._reportPerSecondRate(oldMetrics);
1858
- }
1859
- }
1860
- _addToTimeSeries(values, timeSeries) {
1861
- for (const [key, value] of Object.entries(values)) {
1862
- const values2 = timeSeries[key];
1863
- values2.push(value);
1864
- if (values2.length > this._params.timeSeriesLength) {
1865
- values2.shift();
1866
- }
1867
- }
1868
- }
1869
- _reportPerSecondRate(metrics) {
1870
- const toReport = [
1871
- [
1872
- "storage.load",
1873
- metrics.storage.loadedChunks,
1874
- this._storageAverages.loadsPerSecond
1875
- ],
1876
- [
1877
- "storage.store",
1878
- metrics.storage.storedChunks,
1879
- this._storageAverages.storesPerSecond
1880
- ],
1881
- [
1882
- "network.receive",
1883
- metrics.replication.received,
1884
- this._replicationAverages.receivedPerSecond
1885
- ],
1886
- [
1887
- "network.send",
1888
- metrics.replication.sent,
1889
- this._replicationAverages.sentPerSecond
1890
- ]
1891
- ];
1892
- for (const [metricName, metric, summary] of toReport) {
1893
- summary.record(metric);
1894
- if (metric > 0) {
1895
- import_tracing3.trace.metrics.distribution(`dxos.echo.${metricName}-rate`, metric);
1896
- import_tracing3.trace.metrics.increment(`dxos.echo.${metricName}`, 1, {
1897
- tags: {
1898
- status: "busy"
1899
- }
1900
- });
1901
- } else {
1902
- import_tracing3.trace.metrics.increment(`dxos.echo.${metricName}`, 1, {
1903
- tags: {
1904
- status: "idle"
1905
- }
1906
- });
1907
- }
1908
- }
1909
- this._replicationAverages.sendsFailedPerSecond.record(metrics.replication.failed);
1910
- }
1911
- recordPeerConnected(peerId) {
1912
- this._activeCounters.byPeerId[peerId] = createMessageCounter();
1913
- this._connectionsCount++;
1914
- }
1915
- recordPeerDisconnected(peerId) {
1916
- this._connectionsCount--;
1917
- delete this._activeCounters.byPeerId[peerId];
1918
- }
1919
- recordBytesStored(count) {
1920
- this._activeCounters.storage.storedChunks++;
1921
- this._activeCounters.storage.storedBytes += count;
1922
- this._storageAverages.storedChunkSize.record(count);
1923
- import_tracing3.trace.metrics.distribution("dxos.echo.storage.bytes-stored", count, {
1924
- unit: "bytes"
1925
- });
1926
- }
1927
- recordLoadDuration(durationMs) {
1928
- this._storageAverages.loadDuration.record(durationMs);
1929
- }
1930
- recordStoreDuration(durationMs) {
1931
- this._storageAverages.storeDuration.record(durationMs);
1932
- }
1933
- recordBytesLoaded(count) {
1934
- this._activeCounters.storage.loadedChunks++;
1935
- this._activeCounters.storage.loadedBytes += count;
1936
- this._storageAverages.loadedChunkSize.record(count);
1937
- import_tracing3.trace.metrics.distribution("dxos.echo.storage.bytes-loaded", count, {
1938
- unit: "bytes"
1939
- });
1940
- }
1941
- recordMessageSent(message, duration) {
1942
- let metricsGroupName;
1943
- const bytes = getByteCount(message);
1944
- const tags = {
1945
- type: message.type
1946
- };
1947
- if (isAutomergeProtocolMessage(message)) {
1948
- this._activeCounters.replication.sent++;
1949
- this._replicationAverages.sendDuration.record(duration);
1950
- this._replicationAverages.sentMessageSize.record(bytes);
1951
- metricsGroupName = "replication";
1952
- } else {
1953
- metricsGroupName = "collection-sync";
1954
- }
1955
- import_tracing3.trace.metrics.distribution(`dxos.echo.${metricsGroupName}.bytes-sent`, bytes, {
1956
- unit: "bytes",
1957
- tags
1958
- });
1959
- import_tracing3.trace.metrics.distribution(`dxos.echo.${metricsGroupName}.send-duration`, duration, {
1960
- unit: "millisecond",
1961
- tags
1962
- });
1963
- import_tracing3.trace.metrics.increment(`dxos.echo.${metricsGroupName}.send-status`, 1, {
1964
- tags: {
1965
- ...tags,
1966
- success: true
1967
- }
1968
- });
1969
- const { messageSize, messageCounts } = this._getStatsForType(message);
1970
- messageSize.record(bytes);
1971
- messageCounts.sent++;
1972
- this._lastSentMessages.push({
1973
- type: message.type,
1974
- peerId: message.targetId
1975
- });
1976
- }
1977
- recordMessageReceived(message) {
1978
- const bytes = getByteCount(message);
1979
- const tags = {
1980
- type: message.type
1981
- };
1982
- if (isAutomergeProtocolMessage(message)) {
1983
- this._activeCounters.replication.received++;
1984
- this._replicationAverages.receivedMessageSize.record(bytes);
1985
- import_tracing3.trace.metrics.distribution("dxos.echo.replication.bytes-received", bytes, {
1986
- unit: "bytes",
1987
- tags
1988
- });
1989
- } else {
1990
- import_tracing3.trace.metrics.distribution("dxos.echo.collection-sync.bytes-received", bytes, {
1991
- unit: "bytes",
1992
- tags
1993
- });
1994
- }
1995
- const { messageSize, messageCounts } = this._getStatsForType(message);
1996
- messageSize.record(bytes);
1997
- messageCounts.received++;
1998
- this._lastReceivedMessages.push({
1999
- type: message.type,
2000
- peerId: message.senderId
2001
- });
2002
- }
2003
- recordMessageSendingFailed(message) {
2004
- const tags = {
2005
- type: message.type,
2006
- success: false
2007
- };
2008
- if (isAutomergeProtocolMessage(message)) {
2009
- this._activeCounters.replication.failed++;
2010
- import_tracing3.trace.metrics.increment("dxos.echo.replication.send-status", 1, {
2011
- unit: "bytes",
2012
- tags
2013
- });
2014
- } else {
2015
- import_tracing3.trace.metrics.increment("dxos.echo.collection-sync.send-status", 1, {
2016
- unit: "bytes",
2017
- tags
2018
- });
2019
- }
2020
- const { messageCounts } = this._getStatsForType(message);
2021
- messageCounts.failed++;
2022
- }
2023
- _getStatsForType(message) {
2024
- const messageSize = this._sizeByMessage[message.type] ??= createSlidingWindow();
2025
- const messageCounts = this._activeCounters.byType[message.type] ??= createMessageCounter();
2026
- return {
2027
- messageCounts,
2028
- messageSize
2029
- };
2030
- }
2031
- _computeMessageHistogram(groupKey) {
2032
- const result = {};
2033
- for (const receivedMessage of this._lastReceivedMessages) {
2034
- const counters = result[receivedMessage[groupKey]] ??= {
2035
- received: 0,
2036
- sent: 0
2037
- };
2038
- counters.received++;
2039
- }
2040
- for (const receivedMessage of this._lastSentMessages) {
2041
- const counters = result[receivedMessage[groupKey]] ??= {
2042
- received: 0,
2043
- sent: 0
2044
- };
2045
- counters.sent++;
2046
- }
2047
- return result;
2048
- }
2049
- };
2050
- EchoDataMonitor = _ts_decorate4([
2051
- import_tracing3.trace.resource()
2052
- ], EchoDataMonitor);
2053
- var isAutomergeProtocolMessage = (message) => {
2054
- return !(isCollectionQueryMessage(message) || isCollectionStateMessage(message));
2055
- };
2056
- var createSlidingWindow = (overrides) => new import_util5.SlidingWindowSummary({
2057
- dataPoints: DEFAULT_AVG_WINDOW_SIZE,
2058
- precision: 2,
2059
- ...overrides
2060
- });
2061
- var createLocalCounters = () => ({
2062
- storage: {
2063
- loadedBytes: 0,
2064
- storedBytes: 0,
2065
- storedChunks: 0,
2066
- loadedChunks: 0
2067
- },
2068
- replication: createMessageCounter(),
2069
- byPeerId: {},
2070
- byType: {}
2071
- });
2072
- var createLocalTimeSeries = () => ({
2073
- storage: {
2074
- loadedBytes: [],
2075
- storedBytes: [],
2076
- storedChunks: [],
2077
- loadedChunks: []
2078
- },
2079
- replication: {
2080
- sent: [],
2081
- failed: [],
2082
- received: []
2083
- }
2084
- });
2085
- var createMessageCounter = () => ({
2086
- sent: 0,
2087
- received: 0,
2088
- failed: 0
2089
- });
2090
- var createNetworkAverages = () => ({
2091
- receivedMessageSize: createSlidingWindow(),
2092
- sentMessageSize: createSlidingWindow(),
2093
- sendDuration: createSlidingWindow(),
2094
- receivedPerSecond: createSlidingWindow({
2095
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
2096
- }),
2097
- sentPerSecond: createSlidingWindow({
2098
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
2099
- }),
2100
- sendsFailedPerSecond: createSlidingWindow({
2101
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
2102
- })
2103
- });
2104
- var createStorageAverages = () => ({
2105
- storedChunkSize: createSlidingWindow(),
2106
- loadedChunkSize: createSlidingWindow(),
2107
- loadDuration: createSlidingWindow(),
2108
- storeDuration: createSlidingWindow(),
2109
- loadsPerSecond: createSlidingWindow({
2110
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
2111
- }),
2112
- storesPerSecond: createSlidingWindow({
2113
- dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
2114
- })
2115
- });
2116
- var getByteCount = (message) => {
2117
- return message.type.length + message.senderId.length + message.targetId.length + (message.data?.byteLength ?? 0) + (message.documentId?.length ?? 0);
2118
- };
2119
- var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/documents-synchronizer.ts";
2120
- var MAX_UPDATE_FREQ = 10;
2121
- var DocumentsSynchronizer = class extends import_context.Resource {
2122
- constructor(_params) {
2123
- super(), this._params = _params, this._syncStates = /* @__PURE__ */ new Map(), this._pendingUpdates = /* @__PURE__ */ new Set(), this._sendUpdatesJob = void 0;
2124
- }
2125
- addDocuments(documentIds, retryCounter = 0) {
2126
- if (retryCounter > 3) {
2127
- import_log2.log.warn("Failed to load document, retry limit reached", {
2128
- documentIds
2129
- }, {
2130
- F: __dxlog_file7,
2131
- L: 52,
2132
- S: this,
2133
- C: (f, a) => f(...a)
2134
- });
2135
- return;
2136
- }
2137
- for (const documentId of documentIds) {
2138
- this._params.repo.find(documentId).then(async (doc) => {
2139
- await doc.whenReady();
2140
- this._startSync(doc);
2141
- this._pendingUpdates.add(doc.documentId);
2142
- this._sendUpdatesJob.trigger();
2143
- }).catch((error) => {
2144
- import_log2.log.warn("Failed to load document, wraparound", {
2145
- documentId,
2146
- error
2147
- }, {
2148
- F: __dxlog_file7,
2149
- L: 66,
2150
- S: this,
2151
- C: (f, a) => f(...a)
2152
- });
2153
- this.addDocuments([
2154
- documentId
2155
- ], retryCounter + 1);
2156
- });
2157
- }
2158
- }
2159
- removeDocuments(documentIds) {
2160
- for (const documentId of documentIds) {
2161
- this._syncStates.get(documentId)?.clearSubscriptions?.();
2162
- this._syncStates.delete(documentId);
2163
- this._pendingUpdates.delete(documentId);
2164
- }
2165
- }
2166
- async _open() {
2167
- this._sendUpdatesJob = new import_async2.UpdateScheduler(this._ctx, this._checkAndSendUpdates.bind(this), {
2168
- maxFrequency: MAX_UPDATE_FREQ
2169
- });
2170
- }
2171
- async _close() {
2172
- await this._sendUpdatesJob.join();
2173
- this._syncStates.clear();
2174
- }
2175
- async update(updates) {
2176
- for (const { documentId, mutation, isNew } of updates) {
2177
- if (isNew) {
2178
- const doc = await this._params.repo.find(documentId, FIND_PARAMS);
2179
- doc.update((doc2) => import_automerge.next.loadIncremental(doc2, mutation));
2180
- this._startSync(doc);
2181
- } else {
2182
- this._writeMutation(documentId, mutation);
2183
- }
2184
- }
2185
- }
2186
- _startSync(doc) {
2187
- if (this._syncStates.has(doc.documentId)) {
2188
- (0, import_log2.log)("Document already being synced", {
2189
- documentId: doc.documentId
2190
- }, {
2191
- F: __dxlog_file7,
2192
- L: 105,
2193
- S: this,
2194
- C: (f, a) => f(...a)
2195
- });
2196
- return;
2197
- }
2198
- const syncState = {
2199
- handle: doc
2200
- };
2201
- this._subscribeForChanges(syncState);
2202
- this._syncStates.set(doc.documentId, syncState);
2203
- }
2204
- _subscribeForChanges(syncState) {
2205
- const handler = () => {
2206
- this._pendingUpdates.add(syncState.handle.documentId);
2207
- this._sendUpdatesJob.trigger();
2208
- };
2209
- syncState.handle.on("heads-changed", handler);
2210
- syncState.clearSubscriptions = () => syncState.handle.off("heads-changed", handler);
2211
- }
2212
- async _checkAndSendUpdates() {
2213
- const updates = [];
2214
- const docsWithPendingUpdates = Array.from(this._pendingUpdates);
2215
- this._pendingUpdates.clear();
2216
- for (const documentId of docsWithPendingUpdates) {
2217
- const update = this._getPendingChanges(documentId);
2218
- if (update) {
2219
- updates.push({
2220
- documentId,
2221
- mutation: update
2222
- });
2223
- }
2224
- }
2225
- if (updates.length > 0) {
2226
- this._params.sendUpdates({
2227
- updates
2228
- });
2229
- }
2230
- }
2231
- _getPendingChanges(documentId) {
2232
- const syncState = this._syncStates.get(documentId);
2233
- (0, import_invariant2.invariant)(syncState, "Sync state for document not found", {
2234
- F: __dxlog_file7,
2235
- L: 146,
2236
- S: this,
2237
- A: [
2238
- "syncState",
2239
- "'Sync state for document not found'"
2240
- ]
2241
- });
2242
- const handle = syncState.handle;
2243
- if (!handle || !handle.isReady() || !handle.doc()) {
2244
- return;
2245
- }
2246
- const doc = handle.doc();
2247
- const mutation = syncState.lastSentHead ? import_automerge.next.saveSince(doc, syncState.lastSentHead) : import_automerge.next.save(doc);
2248
- if (mutation.length === 0) {
2249
- return;
2250
- }
2251
- syncState.lastSentHead = import_automerge.next.getHeads(doc);
2252
- return mutation;
2253
- }
2254
- _writeMutation(documentId, mutation) {
2255
- const syncState = this._syncStates.get(documentId);
2256
- (0, import_invariant2.invariant)(syncState, "Sync state for document not found", {
2257
- F: __dxlog_file7,
2258
- L: 162,
2259
- S: this,
2260
- A: [
2261
- "syncState",
2262
- "'Sync state for document not found'"
2263
- ]
2264
- });
2265
- syncState.handle.update((doc) => {
2266
- const headsBefore = import_automerge.next.getHeads(doc);
2267
- const newDoc = import_automerge.next.loadIncremental(doc, mutation);
2268
- if (import_automerge.next.equals(headsBefore, syncState.lastSentHead)) {
2269
- syncState.lastSentHead = import_automerge.next.getHeads(newDoc);
2270
- }
2271
- return newDoc;
2272
- });
2273
- }
2274
- };
2275
- var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
2276
- var DataServiceImpl = class {
2277
- constructor(params) {
2278
- this._subscriptions = /* @__PURE__ */ new Map();
2279
- this._automergeHost = params.automergeHost;
2280
- this._spaceStateManager = params.spaceStateManager;
2281
- this._updateIndexes = params.updateIndexes;
2282
- }
2283
- subscribe(request) {
2284
- return new import_stream.Stream(({ next, ready }) => {
2285
- const synchronizer = new DocumentsSynchronizer({
2286
- repo: this._automergeHost.repo,
2287
- sendUpdates: (updates) => next(updates)
2288
- });
2289
- synchronizer.open().then(() => {
2290
- this._subscriptions.set(request.subscriptionId, synchronizer);
2291
- ready();
2292
- }).catch((err) => import_log.log.catch(err, void 0, {
2293
- F: __dxlog_file8,
2294
- L: 71,
2295
- S: this,
2296
- C: (f, a) => f(...a)
2297
- }));
2298
- return () => synchronizer.close();
2299
- });
2300
- }
2301
- async updateSubscription(request) {
2302
- const synchronizer = this._subscriptions.get(request.subscriptionId);
2303
- (0, import_invariant.invariant)(synchronizer, "Subscription not found", {
2304
- F: __dxlog_file8,
2305
- L: 78,
2306
- S: this,
2307
- A: [
2308
- "synchronizer",
2309
- "'Subscription not found'"
2310
- ]
2311
- });
2312
- if (request.addIds?.length) {
2313
- await synchronizer.addDocuments(request.addIds);
2314
- }
2315
- if (request.removeIds?.length) {
2316
- await synchronizer.removeDocuments(request.removeIds);
2317
- }
2318
- }
2319
- async update(request) {
2320
- if (!request.updates) {
2321
- return;
2322
- }
2323
- const synchronizer = this._subscriptions.get(request.subscriptionId);
2324
- (0, import_invariant.invariant)(synchronizer, "Subscription not found", {
2325
- F: __dxlog_file8,
2326
- L: 93,
2327
- S: this,
2328
- A: [
2329
- "synchronizer",
2330
- "'Subscription not found'"
2331
- ]
2332
- });
2333
- await synchronizer.update(request.updates);
2334
- }
2335
- async flush(request) {
2336
- await this._automergeHost.flush(request);
2337
- }
2338
- async getDocumentHeads(request) {
2339
- const documentIds = request.documentIds;
2340
- if (!documentIds) {
2341
- return {
2342
- heads: {
2343
- entries: []
2344
- }
2345
- };
2346
- }
2347
- const heads = await this._automergeHost.getHeads(documentIds);
2348
- return {
2349
- heads: {
2350
- entries: heads.map((heads2, idx) => ({
2351
- documentId: documentIds[idx],
2352
- heads: heads2
2353
- }))
2354
- }
2355
- };
2356
- }
2357
- async waitUntilHeadsReplicated(request, options) {
2358
- await this._automergeHost.waitUntilHeadsReplicated(request.heads);
2359
- }
2360
- async reIndexHeads(request, options) {
2361
- await this._automergeHost.reIndexHeads(request.documentIds ?? []);
2362
- }
2363
- async updateIndexes() {
2364
- await this._updateIndexes();
2365
- }
2366
- subscribeSpaceSyncState(request) {
2367
- return new import_stream.Stream(({ ctx, next, ready }) => {
2368
- const spaceId = request.spaceId;
2369
- (0, import_invariant.invariant)(import_keys.SpaceId.isValid(spaceId), void 0, {
2370
- F: __dxlog_file8,
2371
- L: 133,
2372
- S: this,
2373
- A: [
2374
- "SpaceId.isValid(spaceId)",
2375
- ""
2376
- ]
2377
- });
2378
- const rootDocumentId = this._spaceStateManager.getSpaceRootDocumentId(spaceId);
2379
- let collectionId = rootDocumentId && deriveCollectionIdFromSpaceId(spaceId, rootDocumentId);
2380
- this._spaceStateManager.spaceDocumentListUpdated.on(ctx, (event) => {
2381
- const newId = deriveCollectionIdFromSpaceId(spaceId, event.spaceRootId);
2382
- if (newId !== collectionId) {
2383
- collectionId = newId;
2384
- scheduler.trigger();
2385
- }
2386
- });
2387
- const scheduler = new import_async.UpdateScheduler(ctx, async () => {
2388
- const state = collectionId ? await this._automergeHost.getCollectionSyncState(collectionId) : {
2389
- peers: []
2390
- };
2391
- next({
2392
- peers: state.peers.map((peer) => ({
2393
- peerId: peer.peerId,
2394
- missingOnRemote: peer.missingOnRemote,
2395
- missingOnLocal: peer.missingOnLocal,
2396
- differentDocuments: peer.differentDocuments,
2397
- localDocumentCount: peer.localDocumentCount,
2398
- remoteDocumentCount: peer.remoteDocumentCount
2399
- }))
2400
- });
2401
- });
2402
- this._automergeHost.collectionStateUpdated.on(ctx, (e) => {
2403
- if (e.collectionId === collectionId) {
2404
- scheduler.trigger();
2405
- }
2406
- });
2407
- scheduler.trigger();
2408
- });
2409
- }
2410
- };
2411
- var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/documents-iterator.ts";
2412
- var LOG_VIEW_OPERATION_THRESHOLD = 300;
2413
- var createSelectedDocumentsIterator = (automergeHost) => (
2414
- /**
2415
- * Get object data blobs from Automerge Repo by ids.
2416
- */
2417
- // TODO(mykola): Unload automerge handles after usage.
2418
- async function* loadDocuments(objects) {
2419
- for (const [id, heads] of objects.entries()) {
2420
- try {
2421
- const { documentId, objectId } = import_protocols3.objectPointerCodec.decode(id);
2422
- const handle = await automergeHost.loadDoc(import_context8.Context.default(void 0, {
2423
- F: __dxlog_file9,
2424
- L: 31
2425
- }), documentId);
2426
- let doc = handle.doc();
2427
- (0, import_invariant9.invariant)(doc, void 0, {
2428
- F: __dxlog_file9,
2429
- L: 34,
2430
- S: this,
2431
- A: [
2432
- "doc",
2433
- ""
2434
- ]
2435
- });
2436
- const currentHeads = A3.getHeads(doc);
2437
- if (!A3.equals(currentHeads, heads)) {
2438
- const begin = Date.now();
2439
- doc = A3.view(doc, heads);
2440
- const end = Date.now();
2441
- if (end - begin > LOG_VIEW_OPERATION_THRESHOLD) {
2442
- (0, import_log8.log)("Checking out document version is taking too long", {
2443
- duration: end - begin,
2444
- requestedHeads: heads,
2445
- originalHeads: currentHeads
2446
- }, {
2447
- F: __dxlog_file9,
2448
- L: 45,
2449
- S: this,
2450
- C: (f, a) => f(...a)
2451
- });
2452
- }
2453
- }
2454
- if (doc.version !== import_echo_protocol3.SpaceDocVersion.CURRENT) {
2455
- continue;
2456
- }
2457
- if (!doc.objects?.[objectId]) {
2458
- continue;
2459
- }
2460
- let newId = id;
2461
- if (import_protocols3.objectPointerCodec.getVersion(id) === import_protocols3.ObjectPointerVersion.V0) {
2462
- const spaceKey = import_echo_protocol3.DatabaseDirectory.getSpaceKey(doc) ?? void 0;
2463
- newId = import_protocols3.objectPointerCodec.encode({
2464
- documentId,
2465
- objectId,
2466
- spaceKey
2467
- });
2468
- }
2469
- yield [
2470
- {
2471
- id: newId,
2472
- object: doc.objects[objectId],
2473
- heads
2474
- }
2475
- ];
2476
- } catch (error) {
2477
- import_log8.log.error("Error loading document", {
2478
- heads,
2479
- id,
2480
- error
2481
- }, {
2482
- F: __dxlog_file9,
2483
- L: 71,
2484
- S: this,
2485
- C: (f, a) => f(...a)
2486
- });
2487
- }
2488
- }
2489
- }
2490
- );
2491
- var QueryError = class extends import_errors.BaseError.extend("QUERY_ERROR") {
2492
- };
2493
- (function(QueryPlan2) {
2494
- QueryPlan2.Plan = Object.freeze({
2495
- make: (steps) => ({
2496
- steps
2497
- })
2498
- });
2499
- QueryPlan2.FilterStep = Object.freeze({
2500
- isNoop: (step) => {
2501
- switch (step.filter.type) {
2502
- case "object": {
2503
- return step.filter.typename === null && (step.filter.id === void 0 || step.filter.id.length === 0) && (step.filter.props === void 0 || Object.keys(step.filter.props).length === 0) && (step.filter.foreignKeys === void 0 || step.filter.foreignKeys.length === 0);
2504
- }
2505
- default:
2506
- return false;
2507
- }
2508
- }
2509
- });
2510
- })(QueryPlan || (QueryPlan = {}));
2511
- var QueryPlan;
2512
- var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/query/query-planner.ts";
2513
- var DEFAULT_OPTIONS = {
2514
- defaultTextSearchKind: "full-text"
2515
- };
2516
- var QueryPlanner = class {
2517
- constructor(options) {
2518
- this._options = {
2519
- ...DEFAULT_OPTIONS,
2520
- ...options
2521
- };
2522
- }
2523
- createPlan(query) {
2524
- let plan = this._generate(query, {
2525
- ...DEFAULT_CONTEXT,
2526
- originalQuery: query
2527
- });
2528
- plan = this._optimizeEmptyFilters(plan);
2529
- plan = this._optimizeSoloUnions(plan);
2530
- return plan;
2531
- }
2532
- _generate(query, context) {
2533
- switch (query.type) {
2534
- case "options":
2535
- return this._generateOptionsClause(query, context);
2536
- case "select":
2537
- return this._generateSelectClause(query, context);
2538
- case "filter":
2539
- return this._generateFilterClause(query, context);
2540
- case "incoming-references":
2541
- return this._generateIncomingReferencesClause(query, context);
2542
- case "relation":
2543
- return this._generateRelationClause(query, context);
2544
- case "relation-traversal":
2545
- return this._generateRelationTraversalClause(query, context);
2546
- case "reference-traversal":
2547
- return this._generateReferenceTraversalClause(query, context);
2548
- case "union":
2549
- return this._generateUnionClause(query, context);
2550
- case "set-difference":
2551
- return this._generateSetDifferenceClause(query, context);
2552
- default:
2553
- throw new QueryError(`Unsupported query type: ${query.type}`, {
2554
- context: {
2555
- query: context.originalQuery
2556
- }
2557
- });
2558
- }
2559
- }
2560
- _generateOptionsClause(query, context) {
2561
- const newContext = {
2562
- ...context
2563
- };
2564
- if (query.options.spaceIds) {
2565
- newContext.selectionSpaces = query.options.spaceIds;
2566
- }
2567
- if (query.options.deleted) {
2568
- newContext.deletedHandling = query.options.deleted;
2569
- }
2570
- return this._generate(query.query, newContext);
2571
- }
2572
- _generateSelectClause(query, context) {
2573
- return this._generateSelectionFromFilter(query.filter, context);
2574
- }
2575
- // TODO(dmaretskyi): This can be rewritten as a function of (filter[]) -> (selection ? undefined, rest: filter[]) that recurses onto itself.
2576
- // TODO(dmaretskyi): If the tip of the query ast is a [select, ...filter] shape we can reorder the filters so the query is most efficient.
2577
- _generateSelectionFromFilter(filter, context) {
2578
- switch (filter.type) {
2579
- case "object": {
2580
- if (context.selectionInverted && filter.id === void 0 && filter.typename === null && Object.keys(filter.props).length === 0) {
2581
- return QueryPlan.Plan.make([
2582
- {
2583
- _tag: "ClearWorkingSetStep"
2584
- },
2585
- ...this._generateDeletedHandlingSteps(context)
2586
- ]);
2587
- }
2588
- if (context.selectionInverted) {
2589
- throw new QueryError("Query too complex", {
2590
- context: {
2591
- query: context.originalQuery
2592
- }
2593
- });
2594
- }
2595
- if (filter.id && filter.id?.length > 0) {
2596
- return QueryPlan.Plan.make([
2597
- {
2598
- _tag: "SelectStep",
2599
- spaces: context.selectionSpaces,
2600
- selector: {
2601
- _tag: "IdSelector",
2602
- objectIds: filter.id
2603
- }
2604
- },
2605
- ...this._generateDeletedHandlingSteps(context),
2606
- {
2607
- _tag: "FilterStep",
2608
- filter: {
2609
- ...filter,
2610
- id: void 0
2611
- }
2612
- }
2613
- ]);
2614
- } else if (filter.typename) {
2615
- return QueryPlan.Plan.make([
2616
- {
2617
- _tag: "SelectStep",
2618
- spaces: context.selectionSpaces,
2619
- selector: {
2620
- _tag: "TypeSelector",
2621
- typename: [
2622
- filter.typename
2623
- ],
2624
- inverted: false
2625
- }
2626
- },
2627
- ...this._generateDeletedHandlingSteps(context),
2628
- {
2629
- _tag: "FilterStep",
2630
- filter: {
2631
- ...filter,
2632
- typename: null
2633
- }
2634
- }
2635
- ]);
2636
- } else {
2637
- return QueryPlan.Plan.make([
2638
- {
2639
- _tag: "SelectStep",
2640
- spaces: context.selectionSpaces,
2641
- selector: {
2642
- _tag: "WildcardSelector"
2643
- }
2644
- },
2645
- ...this._generateDeletedHandlingSteps(context),
2646
- {
2647
- _tag: "FilterStep",
2648
- filter: {
2649
- ...filter
2650
- }
2651
- }
2652
- ]);
2653
- }
2654
- }
2655
- case "text-search": {
2656
- return QueryPlan.Plan.make([
2657
- {
2658
- _tag: "SelectStep",
2659
- spaces: context.selectionSpaces,
2660
- selector: {
2661
- _tag: "TextSelector",
2662
- text: filter.text,
2663
- searchKind: filter.searchKind ?? this._options.defaultTextSearchKind
2664
- }
2665
- },
2666
- ...this._generateDeletedHandlingSteps(context)
2667
- ]);
2668
- }
2669
- case "compare":
2670
- throw new QueryError("Query too complex", {
2671
- context: {
2672
- query: context.originalQuery
2673
- }
2674
- });
2675
- case "in":
2676
- throw new QueryError("Query too complex", {
2677
- context: {
2678
- query: context.originalQuery
2679
- }
2680
- });
2681
- case "range":
2682
- throw new QueryError("Query too complex", {
2683
- context: {
2684
- query: context.originalQuery
2685
- }
2686
- });
2687
- case "not":
2688
- return this._generateSelectionFromFilter(filter.filter, {
2689
- ...context,
2690
- selectionInverted: !context.selectionInverted
2691
- });
2692
- case "and":
2693
- throw new QueryError("Query too complex", {
2694
- context: {
2695
- query: context.originalQuery
2696
- }
2697
- });
2698
- case "or":
2699
- if (filter.filters.every(isTrivialTypenameFilter)) {
2700
- const typenames = filter.filters.map((f) => {
2701
- (0, import_invariant11.invariant)(f.type === "object" && f.typename !== null, void 0, {
2702
- F: __dxlog_file10,
2703
- L: 191,
2704
- S: this,
2705
- A: [
2706
- "f.type === 'object' && f.typename !== null",
2707
- ""
2708
- ]
2709
- });
2710
- return f.typename;
2711
- });
2712
- return QueryPlan.Plan.make([
2713
- {
2714
- _tag: "SelectStep",
2715
- spaces: context.selectionSpaces,
2716
- selector: {
2717
- _tag: "TypeSelector",
2718
- typename: typenames,
2719
- inverted: context.selectionInverted
2720
- }
2721
- },
2722
- ...this._generateDeletedHandlingSteps(context)
2723
- ]);
2724
- } else {
2725
- throw new QueryError("Query too complex", {
2726
- context: {
2727
- query: context.originalQuery
2728
- }
2729
- });
2730
- }
2731
- default:
2732
- throw new QueryError(`Unsupported filter type: ${filter.type}`, {
2733
- context: {
2734
- query: context.originalQuery
2735
- }
2736
- });
2737
- }
2738
- }
2739
- _generateDeletedHandlingSteps(context) {
2740
- switch (context.deletedHandling) {
2741
- case "include":
2742
- return [];
2743
- case "exclude":
2744
- return [
2745
- {
2746
- _tag: "FilterDeletedStep",
2747
- mode: "only-non-deleted"
2748
- }
2749
- ];
2750
- case "only":
2751
- return [
2752
- {
2753
- _tag: "FilterDeletedStep",
2754
- mode: "only-deleted"
2755
- }
2756
- ];
2757
- }
2758
- }
2759
- _generateUnionClause(query, context) {
2760
- return QueryPlan.Plan.make([
2761
- {
2762
- _tag: "UnionStep",
2763
- plans: query.queries.map((query2) => this._generate(query2, context))
2764
- }
2765
- ]);
2766
- }
2767
- _generateSetDifferenceClause(query, context) {
2768
- return QueryPlan.Plan.make([
2769
- {
2770
- _tag: "SetDifferenceStep",
2771
- source: this._generate(query.source, context),
2772
- exclude: this._generate(query.exclude, context)
2773
- }
2774
- ]);
2775
- }
2776
- _generateReferenceTraversalClause(query, context) {
2777
- return QueryPlan.Plan.make([
2778
- ...this._generate(query.anchor, context).steps,
2779
- {
2780
- _tag: "TraverseStep",
2781
- traversal: {
2782
- _tag: "ReferenceTraversal",
2783
- direction: "outgoing",
2784
- property: query.property
2785
- }
2786
- },
2787
- ...this._generateDeletedHandlingSteps(context)
2788
- ]);
2789
- }
2790
- _generateIncomingReferencesClause(query, context) {
2791
- return QueryPlan.Plan.make([
2792
- ...this._generate(query.anchor, context).steps,
2793
- {
2794
- _tag: "TraverseStep",
2795
- traversal: {
2796
- _tag: "ReferenceTraversal",
2797
- direction: "incoming",
2798
- property: query.property
2799
- }
2800
- },
2801
- ...this._generateDeletedHandlingSteps(context),
2802
- {
2803
- _tag: "FilterStep",
2804
- filter: {
2805
- type: "object",
2806
- typename: query.typename,
2807
- props: {}
2808
- }
2809
- }
2810
- ]);
2811
- }
2812
- _generateRelationTraversalClause(query, context) {
2813
- switch (query.direction) {
2814
- case "source": {
2815
- return QueryPlan.Plan.make([
2816
- ...this._generate(query.anchor, context).steps,
2817
- createRelationTraversalStep("relation-to-source"),
2818
- ...this._generateDeletedHandlingSteps(context)
2819
- ]);
2820
- }
2821
- case "target": {
2822
- return QueryPlan.Plan.make([
2823
- ...this._generate(query.anchor, context).steps,
2824
- createRelationTraversalStep("relation-to-target"),
2825
- ...this._generateDeletedHandlingSteps(context)
2826
- ]);
2827
- }
2828
- case "both": {
2829
- const anchorPlan = this._generate(query.anchor, context);
2830
- return QueryPlan.Plan.make([
2831
- ...anchorPlan.steps,
2832
- {
2833
- _tag: "UnionStep",
2834
- plans: [
2835
- QueryPlan.Plan.make([
2836
- createRelationTraversalStep("relation-to-source")
2837
- ]),
2838
- QueryPlan.Plan.make([
2839
- createRelationTraversalStep("relation-to-target")
2840
- ])
2841
- ]
2842
- },
2843
- ...this._generateDeletedHandlingSteps(context)
2844
- ]);
2845
- }
2846
- }
2847
- }
2848
- _generateRelationClause(query, context) {
2849
- switch (query.direction) {
2850
- case "outgoing": {
2851
- return QueryPlan.Plan.make([
2852
- ...this._generate(query.anchor, context).steps,
2853
- createRelationTraversalStep("source-to-relation"),
2854
- ...this._generateDeletedHandlingSteps(context),
2855
- {
2856
- _tag: "FilterStep",
2857
- filter: query.filter ?? NOOP_FILTER
2858
- }
2859
- ]);
2860
- }
2861
- case "incoming": {
2862
- return QueryPlan.Plan.make([
2863
- ...this._generate(query.anchor, context).steps,
2864
- createRelationTraversalStep("target-to-relation"),
2865
- ...this._generateDeletedHandlingSteps(context),
2866
- {
2867
- _tag: "FilterStep",
2868
- filter: query.filter ?? NOOP_FILTER
2869
- }
2870
- ]);
2871
- }
2872
- case "both": {
2873
- const anchorPlan = this._generate(query.anchor, context);
2874
- return QueryPlan.Plan.make([
2875
- ...anchorPlan.steps,
2876
- {
2877
- _tag: "UnionStep",
2878
- plans: [
2879
- QueryPlan.Plan.make([
2880
- createRelationTraversalStep("source-to-relation")
2881
- ]),
2882
- QueryPlan.Plan.make([
2883
- createRelationTraversalStep("target-to-relation")
2884
- ])
2885
- ]
2886
- },
2887
- ...this._generateDeletedHandlingSteps(context),
2888
- {
2889
- _tag: "FilterStep",
2890
- filter: query.filter ?? NOOP_FILTER
2891
- }
2892
- ]);
2893
- }
2894
- }
2895
- }
2896
- _generateFilterClause(query, context) {
2897
- return QueryPlan.Plan.make([
2898
- ...this._generate(query.selection, context).steps,
2899
- {
2900
- _tag: "FilterStep",
2901
- filter: query.filter
2902
- }
2903
- ]);
2904
- }
2905
- /**
2906
- * Removes filter steps that have no predicates.
2907
- */
2908
- _optimizeEmptyFilters(plan) {
2909
- return QueryPlan.Plan.make(plan.steps.filter((step) => {
2910
- if (step._tag === "FilterStep") {
2911
- return !QueryPlan.FilterStep.isNoop(step);
2912
- } else {
2913
- return true;
2914
- }
2915
- }).map((step) => {
2916
- if (step._tag === "UnionStep") {
2917
- return {
2918
- _tag: "UnionStep",
2919
- plans: step.plans.map((plan2) => this._optimizeEmptyFilters(plan2))
2920
- };
2921
- } else {
2922
- return step;
2923
- }
2924
- }));
2925
- }
2926
- /**
2927
- * Removes union steps that have only one child.
2928
- */
2929
- _optimizeSoloUnions(plan) {
2930
- return plan;
2931
- }
2932
- };
2933
- var DEFAULT_CONTEXT = {
2934
- originalQuery: null,
2935
- selectionSpaces: [],
2936
- deletedHandling: "exclude",
2937
- selectionInverted: false
2938
- };
2939
- var NOOP_FILTER = {
2940
- type: "object",
2941
- typename: null,
2942
- id: [],
2943
- props: {}
2944
- };
2945
- var createRelationTraversalStep = (direction) => ({
2946
- _tag: "TraverseStep",
2947
- traversal: {
2948
- _tag: "RelationTraversal",
2949
- direction
2950
- }
2951
- });
2952
- var isTrivialTypenameFilter = (filter) => {
2953
- return filter.type === "object" && filter.typename !== null && Object.keys(filter.props).length === 0 && (filter.id === void 0 || filter.id.length === 0) && (filter.foreignKeys === void 0 || filter.foreignKeys.length === 0);
2954
- };
2955
- var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/query/query-executor.ts";
2956
- var ExecutionTrace = Object.freeze({
2957
- makeEmpty: () => ({
2958
- name: "Empty",
2959
- details: "",
2960
- objectCount: 0,
2961
- documentsLoaded: 0,
2962
- indexHits: 0,
2963
- indexQueryTime: 0,
2964
- documentLoadTime: 0,
2965
- executionTime: 0,
2966
- children: []
2967
- }),
2968
- format: (trace6) => {
2969
- const go = (trace7, indent) => {
2970
- return [
2971
- `${" ".repeat(indent)} - ${trace7.name}(${trace7.details})`,
2972
- `${" ".repeat(indent)} objects: ${trace7.objectCount} docs: ${trace7.documentsLoaded} index hits: ${trace7.indexHits} | total: ${trace7.executionTime.toFixed(0)}ms index: ${trace7.indexQueryTime.toFixed(0)}ms load: ${trace7.documentLoadTime.toFixed(0)}ms`,
2973
- "",
2974
- ...trace7.children.map((child) => go(child, indent + 2))
2975
- ].join("\n");
2976
- };
2977
- return go(trace6, 0);
2978
- }
2979
- });
2980
- var TRACE_QUERY_EXECUTION = false;
2981
- var QueryExecutor = class extends import_context10.Resource {
2982
- constructor(options) {
2983
- super();
2984
- this._trace = ExecutionTrace.makeEmpty();
2985
- this._lastResultSet = [];
2986
- this._indexer = options.indexer;
2987
- this._automergeHost = options.automergeHost;
2988
- this._spaceStateManager = options.spaceStateManager;
2989
- this._id = options.queryId;
2990
- this._query = options.query;
2991
- this._reactivity = options.reactivity;
2992
- const queryPlanner = new QueryPlanner();
2993
- this._plan = queryPlanner.createPlan(this._query);
2994
- }
2995
- get query() {
2996
- return this._query;
2997
- }
2998
- get plan() {
2999
- return this._plan;
3000
- }
3001
- get trace() {
3002
- return this._trace;
3003
- }
3004
- async _open(ctx) {
3005
- }
3006
- async _close(ctx) {
3007
- }
3008
- getResults() {
3009
- return this._lastResultSet.map((item) => ({
3010
- id: item.objectId,
3011
- documentId: item.documentId,
3012
- spaceId: item.spaceId,
3013
- // TODO(dmaretskyi): Plumb through the rank.
3014
- rank: 0
3015
- }));
3016
- }
3017
- async execQuery() {
3018
- (0, import_invariant10.invariant)(this._lifecycleState === import_context10.LifecycleState.OPEN, void 0, {
3019
- F: __dxlog_file11,
3020
- L: 173,
3021
- S: this,
3022
- A: [
3023
- "this._lifecycleState === LifecycleState.OPEN",
3024
- ""
3025
- ]
3026
- });
3027
- const prevResultSet = this._lastResultSet;
3028
- const { workingSet, trace: trace6 } = await this._execPlan(this._plan, []);
3029
- this._lastResultSet = workingSet;
3030
- trace6.name = "Root";
3031
- trace6.details = JSON.stringify({
3032
- id: this._id
3033
- });
3034
- this._trace = trace6;
3035
- const changed = prevResultSet.length !== workingSet.length || prevResultSet.some((item, index) => workingSet[index].objectId !== item.objectId || workingSet[index].spaceId !== item.spaceId || workingSet[index].documentId !== item.documentId);
3036
- if (TRACE_QUERY_EXECUTION) {
3037
- console.log(ExecutionTrace.format(trace6));
3038
- }
3039
- return {
3040
- changed
3041
- };
3042
- }
3043
- async _execPlan(plan, workingSet) {
3044
- const trace6 = ExecutionTrace.makeEmpty();
3045
- const begin = performance.now();
3046
- for (const step of plan.steps) {
3047
- if (this._ctx.disposed) {
3048
- throw new import_context10.ContextDisposedError();
3049
- }
3050
- const result = await this._execStep(step, workingSet);
3051
- workingSet = result.workingSet;
3052
- trace6.children.push(result.trace);
3053
- }
3054
- trace6.objectCount = workingSet.length;
3055
- trace6.executionTime = performance.now() - begin;
3056
- return {
3057
- workingSet,
3058
- trace: trace6
3059
- };
3060
- }
3061
- async _execStep(step, workingSet) {
3062
- if (this._ctx.disposed) {
3063
- return {
3064
- workingSet,
3065
- trace: ExecutionTrace.makeEmpty()
3066
- };
3067
- }
3068
- let newWorkingSet, trace6;
3069
- const begin = performance.now();
3070
- switch (step._tag) {
3071
- case "ClearWorkingSetStep":
3072
- newWorkingSet = [];
3073
- trace6 = ExecutionTrace.makeEmpty();
3074
- break;
3075
- case "SelectStep":
3076
- ({ workingSet: newWorkingSet, trace: trace6 } = await this._execSelectStep(step, workingSet));
3077
- break;
3078
- case "FilterStep":
3079
- ({ workingSet: newWorkingSet, trace: trace6 } = await this._execFilterStep(step, workingSet));
3080
- break;
3081
- case "FilterDeletedStep":
3082
- ({ workingSet: newWorkingSet, trace: trace6 } = await this._execFilterDeletedStep(step, workingSet));
3083
- break;
3084
- case "UnionStep":
3085
- ({ workingSet: newWorkingSet, trace: trace6 } = await this._execUnionStep(step, workingSet));
3086
- break;
3087
- case "SetDifferenceStep":
3088
- ({ workingSet: newWorkingSet, trace: trace6 } = await this._execSetDifferenceStep(step, workingSet));
3089
- break;
3090
- case "TraverseStep":
3091
- ({ workingSet: newWorkingSet, trace: trace6 } = await this._execTraverseStep(step, workingSet));
3092
- break;
3093
- default:
3094
- throw new Error(`Unknown step type: ${step._tag}`);
3095
- }
3096
- trace6.executionTime = performance.now() - begin;
3097
- return {
3098
- workingSet: newWorkingSet,
3099
- trace: trace6
3100
- };
3101
- }
3102
- async _execSelectStep(step, workingSet) {
3103
- workingSet = [
3104
- ...workingSet
3105
- ];
3106
- const trace6 = {
3107
- ...ExecutionTrace.makeEmpty(),
3108
- name: "Select",
3109
- details: JSON.stringify(step.selector)
3110
- };
3111
- switch (step.selector._tag) {
3112
- case "WildcardSelector": {
3113
- const beginIndexQuery = performance.now();
3114
- const indexHits = await this._indexer.execQuery({
3115
- typenames: [],
3116
- inverted: false
3117
- });
3118
- trace6.indexHits = +indexHits.length;
3119
- trace6.indexQueryTime += performance.now() - beginIndexQuery;
3120
- if (this._ctx.disposed) {
3121
- return {
3122
- workingSet,
3123
- trace: trace6
3124
- };
3125
- }
3126
- const documentLoadStart = performance.now();
3127
- const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3128
- trace6.documentsLoaded += results.length;
3129
- trace6.documentLoadTime += performance.now() - documentLoadStart;
3130
- workingSet.push(...results.filter(import_util6.isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
3131
- trace6.objectCount = workingSet.length;
3132
- break;
3133
- }
3134
- case "IdSelector": {
3135
- const beginLoad = performance.now();
3136
- const items = await Promise.all(step.selector.objectIds.map((id) => this._loadFromDXN(import_keys5.DXN.fromLocalObjectId(id), {
3137
- sourceSpaceId: step.spaces[0]
3138
- })));
3139
- trace6.documentLoadTime += performance.now() - beginLoad;
3140
- workingSet.push(...items.filter(import_util6.isNonNullable));
3141
- trace6.objectCount = workingSet.length;
3142
- break;
3143
- }
3144
- case "TypeSelector": {
3145
- const beginIndexQuery = performance.now();
3146
- const indexHits = await this._indexer.execQuery({
3147
- typenames: step.selector.typename,
3148
- inverted: step.selector.inverted
3149
- });
3150
- trace6.indexHits = +indexHits.length;
3151
- trace6.indexQueryTime += performance.now() - beginIndexQuery;
3152
- if (this._ctx.disposed) {
3153
- return {
3154
- workingSet,
3155
- trace: trace6
3156
- };
3157
- }
3158
- const documentLoadStart = performance.now();
3159
- const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3160
- trace6.documentsLoaded += results.length;
3161
- trace6.documentLoadTime += performance.now() - documentLoadStart;
3162
- workingSet.push(...results.filter(import_util6.isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
3163
- trace6.objectCount = workingSet.length;
3164
- break;
3165
- }
3166
- case "TextSelector": {
3167
- const beginIndexQuery = performance.now();
3168
- const indexHits = await this._indexer.execQuery({
3169
- typenames: [],
3170
- text: {
3171
- query: step.selector.text,
3172
- kind: import_effect2.Match.type().pipe(import_effect2.Match.withReturnType(), import_effect2.Match.when("full-text", () => "text"), import_effect2.Match.when("vector", () => "vector"), import_effect2.Match.orElseAbsurd)(step.selector.searchKind)
3173
- }
3174
- });
3175
- trace6.indexHits = +indexHits.length;
3176
- trace6.indexQueryTime += performance.now() - beginIndexQuery;
3177
- if (this._ctx.disposed) {
3178
- return {
3179
- workingSet,
3180
- trace: trace6
3181
- };
3182
- }
3183
- const documentLoadStart = performance.now();
3184
- const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3185
- trace6.documentsLoaded += results.length;
3186
- trace6.documentLoadTime += performance.now() - documentLoadStart;
3187
- workingSet.push(...results.filter(import_util6.isNonNullable).filter((item) => step.spaces.includes(item.spaceId)));
3188
- trace6.objectCount = workingSet.length;
3189
- break;
3190
- }
3191
- default:
3192
- throw new Error(`Unknown selector type: ${step.selector._tag}`);
3193
- }
3194
- return {
3195
- workingSet,
3196
- trace: trace6
3197
- };
3198
- }
3199
- async _execFilterStep(step, workingSet) {
3200
- const result = workingSet.filter((item) => (0, import_chunk_HOPOFWAL.filterMatchObject)(step.filter, {
3201
- id: item.objectId,
3202
- spaceId: item.spaceId,
3203
- doc: item.doc
3204
- }));
3205
- return {
3206
- workingSet: result,
3207
- trace: {
3208
- ...ExecutionTrace.makeEmpty(),
3209
- name: "Filter",
3210
- details: JSON.stringify(step.filter),
3211
- objectCount: result.length
3212
- }
3213
- };
3214
- }
3215
- async _execFilterDeletedStep(step, workingSet) {
3216
- if (workingSet.length === 6) {
3217
- import_log10.log.info("FilterDeletedStep", {
3218
- step,
3219
- workingSet
3220
- }, {
3221
- F: __dxlog_file11,
3222
- L: 386,
3223
- S: this,
3224
- C: (f, a) => f(...a)
3225
- });
3226
- }
3227
- const expected = step.mode === "only-deleted";
3228
- const result = workingSet.filter((item) => import_echo_protocol5.ObjectStructure.isDeleted(item.doc) === expected);
3229
- return {
3230
- workingSet: result,
3231
- trace: {
3232
- ...ExecutionTrace.makeEmpty(),
3233
- name: "FilterDeleted",
3234
- details: step.mode,
3235
- objectCount: result.length
3236
- }
3237
- };
3238
- }
3239
- // TODO(dmaretskyi): This needs to be completed.
3240
- async _execTraverseStep(step, workingSet) {
3241
- const trace6 = {
3242
- ...ExecutionTrace.makeEmpty(),
3243
- name: "Traverse",
3244
- details: JSON.stringify(step.traversal)
3245
- };
3246
- const newWorkingSet = [];
3247
- switch (step.traversal._tag) {
3248
- case "ReferenceTraversal": {
3249
- switch (step.traversal.direction) {
3250
- case "outgoing": {
3251
- const property = import_indexing4.EscapedPropPath.unescape(step.traversal.property);
3252
- const refs = workingSet.flatMap((item) => {
3253
- const ref = (0, import_util6.getDeep)(item.doc.data, property);
3254
- const refs2 = Array.isArray(ref) ? ref : [
3255
- ref
3256
- ];
3257
- return refs2.map((ref2) => {
3258
- try {
3259
- return (0, import_echo_protocol5.isEncodedReference)(ref2) ? {
3260
- ref: import_keys5.DXN.parse(ref2["/"]),
3261
- spaceId: item.spaceId
3262
- } : null;
3263
- } catch {
3264
- import_log10.log.warn("Invalid reference", {
3265
- ref: ref2["/"]
3266
- }, {
3267
- F: __dxlog_file11,
3268
- L: 431,
3269
- S: this,
3270
- C: (f, a) => f(...a)
3271
- });
3272
- return null;
3273
- }
3274
- });
3275
- }).filter(import_util6.isNonNullable);
3276
- const beginLoad = performance.now();
3277
- const items = await Promise.all(refs.map(({ ref, spaceId }) => this._loadFromDXN(ref, {
3278
- sourceSpaceId: spaceId
3279
- })));
3280
- trace6.documentLoadTime += performance.now() - beginLoad;
3281
- newWorkingSet.push(...items.filter(import_util6.isNonNullable));
3282
- trace6.objectCount = newWorkingSet.length;
3283
- break;
3284
- }
3285
- case "incoming": {
3286
- const indexHits = await this._indexer.execQuery({
3287
- typenames: [],
3288
- inverted: false,
3289
- graph: {
3290
- kind: "inbound-reference",
3291
- property: step.traversal.property,
3292
- anchors: workingSet.map((item) => item.objectId)
3293
- }
3294
- });
3295
- trace6.indexHits += indexHits.length;
3296
- const documentLoadStart = performance.now();
3297
- const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3298
- trace6.documentsLoaded += results.length;
3299
- trace6.documentLoadTime += performance.now() - documentLoadStart;
3300
- newWorkingSet.push(...results.filter(import_util6.isNonNullable));
3301
- trace6.objectCount = newWorkingSet.length;
3302
- break;
3303
- }
3304
- }
3305
- break;
3306
- }
3307
- case "RelationTraversal": {
3308
- switch (step.traversal.direction) {
3309
- case "relation-to-source":
3310
- case "relation-to-target": {
3311
- const refs = workingSet.map((item) => {
3312
- const ref = step.traversal.direction === "relation-to-source" ? import_echo_protocol5.ObjectStructure.getRelationSource(item.doc) : import_echo_protocol5.ObjectStructure.getRelationTarget(item.doc);
3313
- if (!(0, import_echo_protocol5.isEncodedReference)(ref)) {
3314
- return null;
3315
- }
3316
- try {
3317
- return {
3318
- ref: import_keys5.DXN.parse(ref["/"]),
3319
- spaceId: item.spaceId
3320
- };
3321
- } catch {
3322
- import_log10.log.warn("Invalid reference", {
3323
- ref: ref["/"]
3324
- }, {
3325
- F: __dxlog_file11,
3326
- L: 494,
3327
- S: this,
3328
- C: (f, a) => f(...a)
3329
- });
3330
- return null;
3331
- }
3332
- }).filter(import_util6.isNonNullable);
3333
- const beginLoad = performance.now();
3334
- const items = await Promise.all(refs.map(({ ref, spaceId }) => this._loadFromDXN(ref, {
3335
- sourceSpaceId: spaceId
3336
- })));
3337
- trace6.documentLoadTime += performance.now() - beginLoad;
3338
- newWorkingSet.push(...items.filter(import_util6.isNonNullable));
3339
- trace6.objectCount = newWorkingSet.length;
3340
- break;
3341
- }
3342
- case "source-to-relation":
3343
- case "target-to-relation": {
3344
- const indexHits = await this._indexer.execQuery({
3345
- typenames: [],
3346
- inverted: false,
3347
- graph: {
3348
- kind: step.traversal.direction === "source-to-relation" ? "relation-source" : "relation-target",
3349
- anchors: workingSet.map((item) => item.objectId),
3350
- property: null
3351
- }
3352
- });
3353
- trace6.indexHits += indexHits.length;
3354
- const documentLoadStart = performance.now();
3355
- const results = await this._loadDocumentsAfterIndexQuery(indexHits);
3356
- trace6.documentsLoaded += results.length;
3357
- trace6.documentLoadTime += performance.now() - documentLoadStart;
3358
- newWorkingSet.push(...results.filter(import_util6.isNonNullable));
3359
- trace6.objectCount = newWorkingSet.length;
3360
- break;
3361
- }
3362
- }
3363
- break;
3364
- }
3365
- default:
3366
- throw new Error(`Unknown traversal type: ${step.traversal._tag}`);
3367
- }
3368
- return {
3369
- workingSet: newWorkingSet,
3370
- trace: trace6
3371
- };
3372
- }
3373
- async _execUnionStep(step, workingSet) {
3374
- const results = /* @__PURE__ */ new Map();
3375
- const resultSets = await Promise.all(step.plans.map((plan) => this._execPlan(plan, [
3376
- ...workingSet
3377
- ])));
3378
- const trace6 = {
3379
- ...ExecutionTrace.makeEmpty(),
3380
- name: "Union"
3381
- };
3382
- for (const resultSet of resultSets) {
3383
- for (const item of resultSet.workingSet) {
3384
- results.set(`${item.spaceId}:${item.documentId}:${item.objectId}`, item);
3385
- }
3386
- trace6.children.push(resultSet.trace);
3387
- }
3388
- return {
3389
- workingSet: [
3390
- ...results.values()
3391
- ],
3392
- trace: trace6
3393
- };
3394
- }
3395
- async _execSetDifferenceStep(step, workingSet) {
3396
- const trace6 = {
3397
- ...ExecutionTrace.makeEmpty(),
3398
- name: "SetDifference"
3399
- };
3400
- const sourceResult = await this._execPlan(step.source, [
3401
- ...workingSet
3402
- ]);
3403
- const excludeResult = await this._execPlan(step.exclude, [
3404
- ...workingSet
3405
- ]);
3406
- trace6.children.push(sourceResult.trace, excludeResult.trace);
3407
- return {
3408
- workingSet: sourceResult.workingSet.filter((item) => {
3409
- const index = excludeResult.workingSet.findIndex((i) => i.objectId === item.objectId);
3410
- return index === -1;
3411
- }),
3412
- trace: trace6
3413
- };
3414
- }
3415
- async _loadDocumentsAfterIndexQuery(indexHits) {
3416
- return Promise.all(indexHits.map(async (hit) => {
3417
- return this._loadFromIndexHit(hit);
3418
- }));
3419
- }
3420
- async _loadFromIndexHit(hit) {
3421
- const { objectId, documentId, spaceKey: spaceKeyInIndex } = import_protocols5.objectPointerCodec.decode(hit.id);
3422
- const handle = await this._automergeHost.loadDoc(import_context10.Context.default(void 0, {
3423
- F: __dxlog_file11,
3424
- L: 604
3425
- }), documentId);
3426
- const doc = handle.doc();
3427
- if (!doc) {
3428
- return null;
3429
- }
3430
- const spaceKey = spaceKeyInIndex ?? import_echo_protocol5.DatabaseDirectory.getSpaceKey(doc);
3431
- if (!spaceKey) {
3432
- return null;
3433
- }
3434
- const object = import_echo_protocol5.DatabaseDirectory.getInlineObject(doc, objectId);
3435
- if (!object) {
3436
- return null;
3437
- }
3438
- return {
3439
- objectId,
3440
- documentId,
3441
- spaceId: await (0, import_chunk_JXX6LF5U.createIdFromSpaceKey)(import_keys5.PublicKey.from(spaceKey)),
3442
- doc: object
3443
- };
3444
- }
3445
- async _loadFromDXN(dxn, { sourceSpaceId }) {
3446
- const echoDxn = dxn.asEchoDXN();
3447
- if (!echoDxn) {
3448
- import_log10.log.warn("unable to resolve DXN", {
3449
- dxn
3450
- }, {
3451
- F: __dxlog_file11,
3452
- L: 631,
3453
- S: this,
3454
- C: (f, a) => f(...a)
3455
- });
3456
- return null;
3457
- }
3458
- const spaceId = echoDxn.spaceId ?? sourceSpaceId;
3459
- const spaceRoot = this._spaceStateManager.getRootBySpaceId(spaceId);
3460
- if (!spaceRoot) {
3461
- import_log10.log.warn("no space state found for", {
3462
- spaceId
3463
- }, {
3464
- F: __dxlog_file11,
3465
- L: 639,
3466
- S: this,
3467
- C: (f, a) => f(...a)
3468
- });
3469
- return null;
3470
- }
3471
- const dbDirectory = spaceRoot.doc();
3472
- if (!dbDirectory) {
3473
- import_log10.log.warn("no space state found for", {
3474
- spaceId
3475
- }, {
3476
- F: __dxlog_file11,
3477
- L: 644,
3478
- S: this,
3479
- C: (f, a) => f(...a)
3480
- });
3481
- return null;
3482
- }
3483
- const inlineObject = import_echo_protocol5.DatabaseDirectory.getInlineObject(dbDirectory, echoDxn.echoId);
3484
- if (inlineObject) {
3485
- return {
3486
- objectId: echoDxn.echoId,
3487
- documentId: spaceRoot.documentId,
3488
- spaceId,
3489
- doc: inlineObject
3490
- };
3491
- }
3492
- const link = import_echo_protocol5.DatabaseDirectory.getLink(dbDirectory, echoDxn.echoId);
3493
- if (!link) {
3494
- return null;
3495
- }
3496
- const handle = await this._automergeHost.loadDoc(import_context10.Context.default(void 0, {
3497
- F: __dxlog_file11,
3498
- L: 663
3499
- }), link);
3500
- const doc = handle.doc();
3501
- if (!doc) {
3502
- return null;
3503
- }
3504
- const object = import_echo_protocol5.DatabaseDirectory.getInlineObject(doc, echoDxn.echoId);
3505
- if (!object) {
3506
- return null;
3507
- }
3508
- return {
3509
- objectId: echoDxn.echoId,
3510
- documentId: handle.documentId,
3511
- spaceId,
3512
- doc: object
3513
- };
3514
- }
3515
- };
3516
- function _ts_decorate5(decorators, target, key, desc) {
3517
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3518
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
3519
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
3520
- return c > 3 && r && Object.defineProperty(target, key, r), r;
3521
- }
3522
- var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/query-service.ts";
3523
- var QueryServiceImpl = class extends import_context9.Resource {
3524
- // TODO(burdon): OK for options, but not params. Pass separately and type readonly here.
3525
- constructor(_params) {
3526
- super(), this._params = _params, this._queries = /* @__PURE__ */ new Set();
3527
- import_tracing5.trace.diagnostic({
3528
- id: "active-queries",
3529
- name: "Active Queries",
3530
- fetch: () => {
3531
- return Array.from(this._queries).map((query) => {
3532
- return {
3533
- query: JSON.stringify(query.executor.query),
3534
- plan: JSON.stringify(query.executor.plan),
3535
- trace: JSON.stringify(query.executor.trace)
3536
- };
3537
- });
3538
- }
3539
- });
3540
- }
3541
- async _open() {
3542
- this._params.indexer.updated.on(this._ctx, () => this.invalidateQueries());
3543
- this._updateQueries = new import_async6.DeferredTask(this._ctx, this._executeQueries.bind(this));
3544
- }
3545
- async _close() {
3546
- await this._updateQueries.join();
3547
- await Promise.all(Array.from(this._queries).map((query) => query.close()));
3548
- }
3549
- async setConfig(config) {
3550
- await this._params.indexer.setConfig(config);
3551
- }
3552
- execQuery(request) {
3553
- return new import_stream2.Stream(({ next, close, ctx }) => {
3554
- const queryEntry = this._createQuery(ctx, request, next, close, close);
3555
- (0, import_async6.scheduleMicroTask)(ctx, async () => {
3556
- await queryEntry.executor.open();
3557
- queryEntry.open = true;
3558
- this._updateQueries.schedule();
3559
- });
3560
- return queryEntry.close;
3561
- });
3562
- }
3563
- /**
3564
- * Re-index all loaded documents.
3565
- */
3566
- async reindex() {
3567
- (0, import_log9.log)("Reindexing all documents...", void 0, {
3568
- F: __dxlog_file12,
3569
- L: 113,
3570
- S: this,
3571
- C: (f, a) => f(...a)
3572
- });
3573
- const iterator = createDocumentsIterator(this._params.automergeHost);
3574
- const ids = /* @__PURE__ */ new Map();
3575
- for await (const documents of iterator()) {
3576
- for (const { id, heads } of documents) {
3577
- ids.set(id, heads);
3578
- }
3579
- if (ids.size % 100 === 0) {
3580
- (0, import_log9.log)("Collected documents...", {
3581
- count: ids.size
3582
- }, {
3583
- F: __dxlog_file12,
3584
- L: 121,
3585
- S: this,
3586
- C: (f, a) => f(...a)
3587
- });
3588
- }
3589
- }
3590
- (0, import_log9.log)("Marking all documents as dirty...", {
3591
- count: ids.size
3592
- }, {
3593
- F: __dxlog_file12,
3594
- L: 125,
3595
- S: this,
3596
- C: (f, a) => f(...a)
3597
- });
3598
- await this._params.indexer.reindex(ids);
3599
- }
3600
- /**
3601
- * Schedule re-execution of all queries.
3602
- */
3603
- invalidateQueries() {
3604
- for (const query of this._queries) {
3605
- query.dirty = true;
3606
- }
3607
- this._updateQueries.schedule();
3608
- }
3609
- _createQuery(ctx, request, onResults, onError, onClose) {
3610
- const parsedQuery = import_echo_protocol4.QueryAST.Query.pipe(import_effect.Schema.decodeUnknownSync)(JSON.parse(request.query));
3611
- const queryEntry = {
3612
- executor: new QueryExecutor({
3613
- indexer: this._params.indexer,
3614
- automergeHost: this._params.automergeHost,
3615
- queryId: request.queryId ?? (0, import_debug2.raise)(new Error("query id required")),
3616
- query: parsedQuery,
3617
- reactivity: request.reactivity,
3618
- spaceStateManager: this._params.spaceStateManager
3619
- }),
3620
- dirty: true,
3621
- open: false,
3622
- firstResult: true,
3623
- sendResults: (results) => {
3624
- if (ctx.disposed) {
3625
- return;
3626
- }
3627
- onResults({
3628
- queryId: request.queryId,
3629
- results
3630
- });
3631
- },
3632
- onError,
3633
- close: async () => {
3634
- onClose();
3635
- await queryEntry.executor.close();
3636
- this._queries.delete(queryEntry);
3637
- }
3638
- };
3639
- this._queries.add(queryEntry);
3640
- return queryEntry;
3641
- }
3642
- async _executeQueries() {
3643
- const begin = performance.now();
3644
- let count = 0;
3645
- await Promise.all(Array.from(this._queries).map(async (query) => {
3646
- if (!query.dirty || !query.open) {
3647
- return;
3648
- }
3649
- count++;
3650
- try {
3651
- const { changed } = await query.executor.execQuery();
3652
- query.dirty = false;
3653
- if (changed || query.firstResult) {
3654
- query.firstResult = false;
3655
- query.sendResults(query.executor.getResults());
3656
- }
3657
- } catch (err) {
3658
- import_log9.log.catch(err, void 0, {
3659
- F: __dxlog_file12,
3660
- L: 196,
3661
- S: this,
3662
- C: (f, a) => f(...a)
3663
- });
3664
- }
3665
- }));
3666
- import_log9.log.verbose("executed queries", {
3667
- count,
3668
- duration: performance.now() - begin
3669
- }, {
3670
- F: __dxlog_file12,
3671
- L: 200,
3672
- S: this,
3673
- C: (f, a) => f(...a)
3674
- });
3675
- }
3676
- };
3677
- _ts_decorate5([
3678
- import_async6.synchronized
3679
- ], QueryServiceImpl.prototype, "_close", null);
3680
- _ts_decorate5([
3681
- import_tracing5.trace.span({
3682
- showInBrowserTimeline: true
3683
- })
3684
- ], QueryServiceImpl.prototype, "_executeQueries", null);
3685
- QueryServiceImpl = _ts_decorate5([
3686
- import_tracing5.trace.resource()
3687
- ], QueryServiceImpl);
3688
- var createDocumentsIterator = (automergeHost) => (
3689
- /**
3690
- * Recursively get all object data blobs from loaded documents from Automerge Repo.
3691
- */
3692
- // TODO(mykola): Unload automerge handles after usage.
3693
- async function* getAllDocuments() {
3694
- const visited = /* @__PURE__ */ new Set();
3695
- async function* getObjectsFromHandle(handle) {
3696
- if (visited.has(handle.documentId) || !handle.isReady()) {
3697
- return;
3698
- }
3699
- const doc = handle.doc();
3700
- const spaceKey = import_echo_protocol4.DatabaseDirectory.getSpaceKey(doc) ?? void 0;
3701
- if (doc.objects) {
3702
- yield Object.entries(doc.objects).map(([objectId, object]) => {
3703
- return {
3704
- id: import_protocols4.objectPointerCodec.encode({
3705
- documentId: handle.documentId,
3706
- objectId,
3707
- spaceKey
3708
- }),
3709
- object,
3710
- heads: (0, import_automerge4.getHeads)(doc)
3711
- };
3712
- });
3713
- }
3714
- if (doc.links) {
3715
- for (const id of Object.values(doc.links)) {
3716
- const urlString = id.toString();
3717
- if (visited.has(urlString)) {
3718
- continue;
3719
- }
3720
- const linkHandle = await automergeHost.loadDoc(import_context9.Context.default(void 0, {
3721
- F: __dxlog_file12,
3722
- L: 240
3723
- }), urlString);
3724
- for await (const result of getObjectsFromHandle(linkHandle)) {
3725
- yield result;
3726
- }
3727
- }
3728
- }
3729
- visited.add(handle.documentId);
3730
- }
3731
- for (const handle of Object.values(automergeHost.repo.handles)) {
3732
- if (visited.has(handle.documentId)) {
3733
- continue;
3734
- }
3735
- for await (const result of getObjectsFromHandle(handle)) {
3736
- yield result;
3737
- }
3738
- visited.add(handle.documentId);
3739
- }
3740
- }
3741
- );
3742
- var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/automerge-metrics.ts";
3743
- var measureDocMetrics = (doc) => {
3744
- const snapshot = A4.save(doc);
3745
- const start = Date.now();
3746
- const temp = A4.load(snapshot);
3747
- const end = Date.now();
3748
- A4.free(temp);
3749
- const getAllChangesStart = Date.now();
3750
- const mutationCount = A4.getAllChanges(doc).length;
3751
- const getAllChangesEnd = Date.now();
3752
- if (getAllChangesEnd - getAllChangesStart > 300) {
3753
- import_log11.log.warn("getAllChanges took too long", {
3754
- elapsed: getAllChangesEnd - getAllChangesStart
3755
- }, {
3756
- F: __dxlog_file13,
3757
- L: 31,
3758
- S: void 0,
3759
- C: (f, a) => f(...a)
3760
- });
3761
- }
3762
- return {
3763
- compressedByteSize: snapshot.byteLength,
3764
- loadTime: end - start,
3765
- mutationCount
3766
- };
3767
- };
3768
- var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/database-root.ts";
3769
- var DatabaseRoot = class {
3770
- static mapLinks(doc, mapping) {
3771
- doc.change((d) => {
3772
- if (!d.links) {
3773
- return;
3774
- }
3775
- for (const [key, value] of Object.entries(d.links)) {
3776
- const documentId = (0, import_automerge_repo5.interpretAsDocumentId)(value.toString());
3777
- if (mapping[documentId]) {
3778
- d.links[key] = `automerge:${mapping[documentId]}`;
3779
- }
3780
- }
3781
- });
3782
- }
3783
- constructor(_rootHandle) {
3784
- this._rootHandle = _rootHandle;
3785
- }
3786
- get documentId() {
3787
- return this._rootHandle.documentId;
3788
- }
3789
- get url() {
3790
- return this._rootHandle.url;
3791
- }
3792
- get isLoaded() {
3793
- return this._rootHandle.isReady();
3794
- }
3795
- get handle() {
3796
- return this._rootHandle;
3797
- }
3798
- doc() {
3799
- return this._rootHandle.isReady() ? this._rootHandle.doc() : null;
3800
- }
3801
- getVersion() {
3802
- const doc = this.doc();
3803
- if (!doc) {
3804
- return null;
3805
- }
3806
- return doc.version ?? import_echo_protocol6.SpaceDocVersion.LEGACY;
3807
- }
3808
- getSpaceKey() {
3809
- const doc = this.doc();
3810
- if (!doc) {
3811
- return null;
3812
- }
3813
- return import_echo_protocol6.DatabaseDirectory.getSpaceKey(doc);
3814
- }
3815
- getInlineObjectCount() {
3816
- const doc = this.doc();
3817
- if (!doc) {
3818
- return null;
3819
- }
3820
- return Object.keys(doc.objects ?? {}).length;
3821
- }
3822
- getLinkedObjectCount() {
3823
- const doc = this.doc();
3824
- if (!doc) {
3825
- return null;
3826
- }
3827
- return Object.keys(doc.links ?? {}).length;
3828
- }
3829
- getAllLinkedDocuments() {
3830
- const doc = this.doc();
3831
- (0, import_invariant13.invariant)(doc, void 0, {
3832
- F: __dxlog_file14,
3833
- L: 88,
3834
- S: this,
3835
- A: [
3836
- "doc",
3837
- ""
3838
- ]
3839
- });
3840
- return Object.values(doc.links ?? {}).map((s) => s.toString());
3841
- }
3842
- measureMetrics() {
3843
- const doc = this.doc();
3844
- if (!doc) {
3845
- return null;
3846
- }
3847
- return measureDocMetrics(doc);
3848
- }
3849
- };
3850
- var __dxlog_file15 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/space-state-manager.ts";
3851
- var SpaceStateManager = class extends import_context11.Resource {
3852
- constructor() {
3853
- super(...arguments);
3854
- this._roots = /* @__PURE__ */ new Map();
3855
- this._rootBySpace = /* @__PURE__ */ new Map();
3856
- this._perRootContext = /* @__PURE__ */ new Map();
3857
- this._lastSpaceDocumentList = /* @__PURE__ */ new Map();
3858
- this.spaceDocumentListUpdated = new import_async7.Event();
3859
- }
3860
- async _close(ctx) {
3861
- for (const [_, rootCtx] of this._perRootContext) {
3862
- await rootCtx.dispose();
3863
- }
3864
- this._roots.clear();
3865
- }
3866
- get roots() {
3867
- return this._roots;
3868
- }
3869
- getRootByDocumentId(documentId) {
3870
- return this._roots.get(documentId);
3871
- }
3872
- getSpaceRootDocumentId(spaceId) {
3873
- return this._rootBySpace.get(spaceId);
3874
- }
3875
- getRootBySpaceId(spaceId) {
3876
- (0, import_invariant12.invariant)(this._lifecycleState === import_context11.LifecycleState.OPEN, void 0, {
3877
- F: __dxlog_file15,
3878
- L: 44,
3879
- S: this,
3880
- A: [
3881
- "this._lifecycleState === LifecycleState.OPEN",
3882
- ""
3883
- ]
3884
- });
3885
- const documentId = this._rootBySpace.get(spaceId);
3886
- if (!documentId) {
3887
- return void 0;
3888
- }
3889
- return this._roots.get(documentId);
3890
- }
3891
- async assignRootToSpace(spaceId, handle) {
3892
- let root;
3893
- if (this._roots.has(handle.documentId)) {
3894
- root = this._roots.get(handle.documentId);
3895
- } else {
3896
- root = new DatabaseRoot(handle);
3897
- this._roots.set(handle.documentId, root);
3898
- }
3899
- if (this._rootBySpace.get(spaceId) === root.handle.documentId) {
3900
- return root;
3901
- }
3902
- const prevRootId = this._rootBySpace.get(spaceId);
3903
- if (prevRootId) {
3904
- void this._perRootContext.get(prevRootId)?.dispose();
3905
- this._perRootContext.delete(prevRootId);
3906
- }
3907
- this._rootBySpace.set(spaceId, root.handle.documentId);
3908
- const ctx = new import_context11.Context(void 0, {
3909
- F: __dxlog_file15,
3910
- L: 72
3911
- });
3912
- this._perRootContext.set(root.handle.documentId, ctx);
3913
- await root.handle.whenReady();
3914
- const documentListCheckScheduler = new import_async7.UpdateScheduler(ctx, async () => {
3915
- const documentIds = [
3916
- root.documentId,
3917
- ...root.getAllLinkedDocuments().map((url) => (0, import_automerge_repo4.interpretAsDocumentId)(url))
3918
- ];
3919
- if (!(0, import_lodash.default)(documentIds, this._lastSpaceDocumentList.get(spaceId))) {
3920
- this._lastSpaceDocumentList.set(spaceId, documentIds);
3921
- this.spaceDocumentListUpdated.emit(new SpaceDocumentListUpdatedEvent(spaceId, root.documentId, prevRootId, documentIds));
3922
- }
3923
- }, {
3924
- maxFrequency: 50
3925
- });
3926
- const triggerCheckOnChange = () => documentListCheckScheduler.trigger();
3927
- root.handle.addListener("change", triggerCheckOnChange);
3928
- ctx.onDispose(() => root.handle.removeListener("change", triggerCheckOnChange));
3929
- documentListCheckScheduler.trigger();
3930
- return root;
3931
- }
3932
- };
3933
- var SpaceDocumentListUpdatedEvent = class {
3934
- constructor(spaceId, spaceRootId, previousRootId, documentIds) {
3935
- this.spaceId = spaceId;
3936
- this.spaceRootId = spaceRootId;
3937
- this.previousRootId = previousRootId;
3938
- this.documentIds = documentIds;
3939
- }
3940
- };
3941
- var __dxlog_file16 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/echo-host.ts";
3942
- var DEFAULT_INDEXING_CONFIG = {
3943
- // TODO(dmaretskyi): Disabled by default since embedding generation is expensive.
3944
- fullText: false,
3945
- vector: false
3946
- };
3947
- var EchoHost = class extends import_context7.Resource {
3948
- constructor({ kv, indexing = {}, peerIdProvider, getSpaceKeyByRootDocumentId }) {
3949
- super();
3950
- this._spaceStateManager = new SpaceStateManager();
3951
- const indexingConfig = {
3952
- ...DEFAULT_INDEXING_CONFIG,
3953
- ...indexing
3954
- };
3955
- this._indexMetadataStore = new import_indexing2.IndexMetadataStore({
3956
- db: kv.sublevel("index-metadata")
3957
- });
3958
- this._echoDataMonitor = new EchoDataMonitor();
3959
- this._automergeHost = new AutomergeHost({
3960
- db: kv,
3961
- dataMonitor: this._echoDataMonitor,
3962
- indexMetadataStore: this._indexMetadataStore,
3963
- peerIdProvider,
3964
- getSpaceKeyByRootDocumentId
3965
- });
3966
- this._indexer = new import_indexing2.Indexer({
3967
- db: kv,
3968
- indexStore: new import_indexing2.IndexStore({
3969
- db: kv.sublevel("index-storage")
3970
- }),
3971
- metadataStore: this._indexMetadataStore,
3972
- loadDocuments: createSelectedDocumentsIterator(this._automergeHost),
3973
- indexCooldownTime: process.env.NODE_ENV === "test" ? 0 : void 0
3974
- });
3975
- void this._indexer.setConfig({
3976
- enabled: true,
3977
- indexes: [
3978
- //
3979
- {
3980
- kind: import_indexing3.IndexKind.Kind.SCHEMA_MATCH
3981
- },
3982
- {
3983
- kind: import_indexing3.IndexKind.Kind.GRAPH
3984
- },
3985
- ...indexingConfig.fullText ? [
3986
- {
3987
- kind: import_indexing3.IndexKind.Kind.FULL_TEXT
3988
- }
3989
- ] : [],
3990
- ...indexingConfig.vector ? [
3991
- {
3992
- kind: import_indexing3.IndexKind.Kind.VECTOR
3993
- }
3994
- ] : []
3995
- ]
3996
- });
3997
- this._queryService = new QueryServiceImpl({
3998
- automergeHost: this._automergeHost,
3999
- indexer: this._indexer,
4000
- spaceStateManager: this._spaceStateManager
4001
- });
4002
- this._dataService = new DataServiceImpl({
4003
- automergeHost: this._automergeHost,
4004
- spaceStateManager: this._spaceStateManager,
4005
- updateIndexes: async () => {
4006
- await this._indexer.updateIndexes();
4007
- }
4008
- });
4009
- import_tracing4.trace.diagnostic({
4010
- id: "echo-stats",
4011
- name: "Echo Stats",
4012
- fetch: async () => {
4013
- return {
4014
- dataStats: this._echoDataMonitor.computeStats(),
4015
- loadedDocsCount: this._automergeHost.loadedDocsCount
4016
- };
4017
- }
4018
- });
4019
- import_tracing4.trace.diagnostic({
4020
- id: "database-roots",
4021
- name: "Database Roots",
4022
- fetch: async () => {
4023
- return Array.from(this._spaceStateManager.roots.values()).map((root) => ({
4024
- url: root.url,
4025
- isLoaded: root.isLoaded,
4026
- spaceKey: root.getSpaceKey(),
4027
- inlineObjects: root.getInlineObjectCount(),
4028
- linkedObjects: root.getLinkedObjectCount()
4029
- }));
4030
- }
4031
- });
4032
- import_tracing4.trace.diagnostic({
4033
- id: "database-root-metrics",
4034
- name: "Database Roots (with metrics)",
4035
- fetch: async () => {
4036
- return Array.from(this._spaceStateManager.roots.values()).map((root) => ({
4037
- url: root.url,
4038
- isLoaded: root.isLoaded,
4039
- spaceKey: root.getSpaceKey(),
4040
- inlineObjects: root.getInlineObjectCount(),
4041
- linkedObjects: root.getLinkedObjectCount(),
4042
- ...root.measureMetrics() ?? {}
4043
- }));
4044
- }
4045
- });
4046
- }
4047
- get queryService() {
4048
- return this._queryService;
4049
- }
4050
- get dataService() {
4051
- return this._dataService;
4052
- }
4053
- /**
4054
- * @deprecated To be abstracted away.
4055
- */
4056
- get automergeRepo() {
4057
- return this._automergeHost.repo;
4058
- }
4059
- get roots() {
4060
- return this._spaceStateManager.roots;
4061
- }
4062
- async _open(ctx) {
4063
- await this._automergeHost.open();
4064
- await this._indexer.open(ctx);
4065
- await this._queryService.open(ctx);
4066
- await this._spaceStateManager.open(ctx);
4067
- this._spaceStateManager.spaceDocumentListUpdated.on(this._ctx, (e) => {
4068
- if (e.previousRootId) {
4069
- void this._automergeHost.clearLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.previousRootId));
4070
- }
4071
- void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId), e.documentIds);
4072
- void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.spaceRootId), e.documentIds);
4073
- });
4074
- this._automergeHost.documentsSaved.on(this._ctx, () => {
4075
- this._queryService.invalidateQueries();
4076
- });
4077
- }
4078
- async _close(ctx) {
4079
- await this._queryService.close(ctx);
4080
- await this._spaceStateManager.close(ctx);
4081
- await this._indexer.close(ctx);
4082
- await this._automergeHost.close();
4083
- }
4084
- /**
4085
- * Flush all pending writes to the underlying storage.
4086
- */
4087
- async flush() {
4088
- await this._automergeHost.repo.flush();
4089
- }
4090
- /**
4091
- * Perform any pending index updates.
4092
- */
4093
- async updateIndexes() {
4094
- await this._indexer.updateIndexes();
4095
- }
4096
- /**
4097
- * Loads the document handle from the repo and waits for it to be ready.
4098
- */
4099
- async loadDoc(ctx, documentId, opts) {
4100
- return await this._automergeHost.loadDoc(ctx, documentId, opts);
4101
- }
4102
- async exportDoc(ctx, id) {
4103
- return await this._automergeHost.exportDoc(ctx, id);
4104
- }
4105
- /**
4106
- * Create new persisted document.
4107
- */
4108
- createDoc(initialValue, opts) {
4109
- return this._automergeHost.createDoc(initialValue, opts);
4110
- }
4111
- /**
4112
- * Create new space root.
4113
- */
4114
- async createSpaceRoot(spaceKey) {
4115
- (0, import_invariant8.invariant)(this._lifecycleState === import_context7.LifecycleState.OPEN, void 0, {
4116
- F: __dxlog_file16,
4117
- L: 255,
4118
- S: this,
4119
- A: [
4120
- "this._lifecycleState === LifecycleState.OPEN",
4121
- ""
4122
- ]
4123
- });
4124
- const spaceId = await (0, import_echo_protocol2.createIdFromSpaceKey)(spaceKey);
4125
- const automergeRoot = this._automergeHost.createDoc({
4126
- version: import_echo_protocol2.SpaceDocVersion.CURRENT,
4127
- access: {
4128
- spaceKey: spaceKey.toHex()
4129
- },
4130
- // Better to initialize them right away to avoid merge conflicts and data loss that can occur if those maps get created on the fly.
4131
- objects: {},
4132
- links: {}
4133
- });
4134
- await this._automergeHost.flush({
4135
- documentIds: [
4136
- automergeRoot.documentId
4137
- ]
4138
- });
4139
- return await this.openSpaceRoot(spaceId, automergeRoot.url);
4140
- }
4141
- // TODO(dmaretskyi): Change to document id.
4142
- async openSpaceRoot(spaceId, automergeUrl) {
4143
- (0, import_invariant8.invariant)(this._lifecycleState === import_context7.LifecycleState.OPEN, void 0, {
4144
- F: __dxlog_file16,
4145
- L: 274,
4146
- S: this,
4147
- A: [
4148
- "this._lifecycleState === LifecycleState.OPEN",
4149
- ""
4150
- ]
4151
- });
4152
- const handle = await this._automergeHost.repo.find(automergeUrl, FIND_PARAMS);
4153
- await handle.whenReady();
4154
- return this._spaceStateManager.assignRootToSpace(spaceId, handle);
4155
- }
4156
- // TODO(dmaretskyi): Change to document id.
4157
- async closeSpaceRoot(automergeUrl) {
4158
- (0, import_debug.todo)();
4159
- }
4160
- /**
4161
- * Install data replicator.
4162
- */
4163
- async addReplicator(replicator) {
4164
- await this._automergeHost.addReplicator(replicator);
4165
- }
4166
- /**
4167
- * Remove data replicator.
4168
- */
4169
- async removeReplicator(replicator) {
4170
- await this._automergeHost.removeReplicator(replicator);
4171
- }
4172
- };
4173
- var __dxlog_file17 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/inflight-request-limiter.ts";
4174
- var InflightRequestLimiter = class extends import_context13.Resource {
4175
- constructor(_config) {
4176
- super(), this._config = _config, this._inflightRequestBalance = 0, this._requestBarrier = new import_async9.Trigger();
4177
- }
4178
- async _open() {
4179
- this._inflightRequestBalance = 0;
4180
- this._requestBarrier.reset();
4181
- this._requestBarrier.wake();
4182
- }
4183
- async _close() {
4184
- this._inflightRequestBalance = 0;
4185
- this._requestBarrier.throw(new Error("Rate limiter closed."));
4186
- clearTimeout(this._resetBalanceTimeout);
4187
- }
4188
- async rateLimit(message) {
4189
- if (message.type !== "sync") {
4190
- return;
4191
- }
4192
- while (this._inflightRequestBalance >= this._config.maxInflightRequests) {
4193
- await this._requestBarrier.wait();
4194
- }
4195
- this._inflightRequestBalance++;
4196
- if (this._inflightRequestBalance === this._config.maxInflightRequests) {
4197
- this._requestBarrier.reset();
4198
- this._resetBalanceTimeout = setTimeout(() => {
4199
- import_log13.log.warn("Request balance has not changed during specified timeout, resetting request limiter.", void 0, {
4200
- F: __dxlog_file17,
4201
- L: 52,
4202
- S: this,
4203
- C: (f, a) => f(...a)
4204
- });
4205
- this._inflightRequestBalance = 0;
4206
- this._requestBarrier.wake();
4207
- }, this._config.resetBalanceTimeoutMs);
4208
- }
4209
- }
4210
- handleResponse(message) {
4211
- if (message.type !== "sync") {
4212
- return;
4213
- }
4214
- this._inflightRequestBalance--;
4215
- if (this._inflightRequestBalance + 1 === this._config.maxInflightRequests) {
4216
- this._requestBarrier.wake();
4217
- clearInterval(this._resetBalanceTimeout);
4218
- }
4219
- }
4220
- };
4221
- function _ts_add_disposable_resource(env, value, async) {
4222
- if (value !== null && value !== void 0) {
4223
- if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4224
- var dispose, inner;
4225
- if (async) {
4226
- if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
4227
- dispose = value[Symbol.asyncDispose];
4228
- }
4229
- if (dispose === void 0) {
4230
- if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
4231
- dispose = value[Symbol.dispose];
4232
- if (async) inner = dispose;
4233
- }
4234
- if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
4235
- if (inner) dispose = function() {
4236
- try {
4237
- inner.call(this);
4238
- } catch (e) {
4239
- return Promise.reject(e);
4240
- }
4241
- };
4242
- env.stack.push({
4243
- value,
4244
- dispose,
4245
- async
4246
- });
4247
- } else if (async) {
4248
- env.stack.push({
4249
- async: true
4250
- });
4251
- }
4252
- return value;
4253
- }
4254
- function _ts_dispose_resources(env) {
4255
- var _SuppressedError = typeof SuppressedError === "function" ? SuppressedError : function(error, suppressed, message) {
4256
- var e = new Error(message);
4257
- return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
4258
- };
4259
- return (_ts_dispose_resources = function _ts_dispose_resources2(env2) {
4260
- function fail(e) {
4261
- env2.error = env2.hasError ? new _SuppressedError(e, env2.error, "An error was suppressed during disposal.") : e;
4262
- env2.hasError = true;
4263
- }
4264
- var r, s = 0;
4265
- function next() {
4266
- while (r = env2.stack.pop()) {
4267
- try {
4268
- if (!r.async && s === 1) return s = 0, env2.stack.push(r), Promise.resolve().then(next);
4269
- if (r.dispose) {
4270
- var result = r.dispose.call(r.value);
4271
- if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) {
4272
- fail(e);
4273
- return next();
4274
- });
4275
- } else s |= 1;
4276
- } catch (e) {
4277
- fail(e);
4278
- }
4279
- }
4280
- if (s === 1) return env2.hasError ? Promise.reject(env2.error) : Promise.resolve();
4281
- if (env2.hasError) throw env2.error;
4282
- }
4283
- return next();
4284
- })(env);
4285
- }
4286
- var __dxlog_file18 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts";
4287
- var INITIAL_RESTART_DELAY = 500;
4288
- var RESTART_DELAY_JITTER = 250;
4289
- var MAX_RESTART_DELAY = 5e3;
4290
- var EchoEdgeReplicator = class {
4291
- constructor({ edgeConnection, disableSharePolicy }) {
4292
- this._mutex = new import_async8.Mutex();
4293
- this._ctx = void 0;
4294
- this._context = null;
4295
- this._connectedSpaces = /* @__PURE__ */ new Set();
4296
- this._connections = /* @__PURE__ */ new Map();
4297
- this._sharePolicyEnabled = true;
4298
- this._edgeConnection = edgeConnection;
4299
- this._sharePolicyEnabled = !disableSharePolicy;
4300
- }
4301
- async connect(context) {
4302
- (0, import_log12.log)("connecting...", {
4303
- peerId: context.peerId,
4304
- connectedSpaces: this._connectedSpaces.size
4305
- }, {
4306
- F: __dxlog_file18,
4307
- L: 61,
4308
- S: this,
4309
- C: (f, a) => f(...a)
4310
- });
4311
- this._context = context;
4312
- this._ctx = import_context12.Context.default(void 0, {
4313
- F: __dxlog_file18,
4314
- L: 63
4315
- });
4316
- this._ctx.onDispose(this._edgeConnection.onReconnected(() => {
4317
- this._ctx && (0, import_async8.scheduleMicroTask)(this._ctx, () => this._handleReconnect());
4318
- }));
4319
- }
4320
- async _handleReconnect() {
4321
- const env = {
4322
- stack: [],
4323
- error: void 0,
4324
- hasError: false
4325
- };
4326
- try {
4327
- const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
4328
- const spaces = [
4329
- ...this._connectedSpaces
4330
- ];
4331
- for (const connection of this._connections.values()) {
4332
- await connection.close();
4333
- }
4334
- this._connections.clear();
4335
- if (this._context !== null) {
4336
- for (const spaceId of spaces) {
4337
- await this._openConnection(spaceId);
4338
- }
4339
- }
4340
- } catch (e) {
4341
- env.error = e;
4342
- env.hasError = true;
4343
- } finally {
4344
- _ts_dispose_resources(env);
4345
- }
4346
- }
4347
- async disconnect() {
4348
- const env = {
4349
- stack: [],
4350
- error: void 0,
4351
- hasError: false
4352
- };
4353
- try {
4354
- const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
4355
- await this._ctx?.dispose();
4356
- for (const connection of this._connections.values()) {
4357
- await connection.close();
4358
- }
4359
- this._connections.clear();
4360
- } catch (e) {
4361
- env.error = e;
4362
- env.hasError = true;
4363
- } finally {
4364
- _ts_dispose_resources(env);
4365
- }
4366
- }
4367
- async connectToSpace(spaceId) {
4368
- const env = {
4369
- stack: [],
4370
- error: void 0,
4371
- hasError: false
4372
- };
4373
- try {
4374
- const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
4375
- if (this._connectedSpaces.has(spaceId)) {
4376
- return;
4377
- }
4378
- this._connectedSpaces.add(spaceId);
4379
- if (this._context !== null) {
4380
- await this._openConnection(spaceId);
4381
- }
4382
- } catch (e) {
4383
- env.error = e;
4384
- env.hasError = true;
4385
- } finally {
4386
- _ts_dispose_resources(env);
4387
- }
4388
- }
4389
- async disconnectFromSpace(spaceId) {
4390
- const env = {
4391
- stack: [],
4392
- error: void 0,
4393
- hasError: false
4394
- };
4395
- try {
4396
- const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
4397
- this._connectedSpaces.delete(spaceId);
4398
- const connection = this._connections.get(spaceId);
4399
- if (connection) {
4400
- await connection.close();
4401
- this._connections.delete(spaceId);
4402
- }
4403
- } catch (e) {
4404
- env.error = e;
4405
- env.hasError = true;
4406
- } finally {
4407
- _ts_dispose_resources(env);
4408
- }
4409
- }
4410
- async _openConnection(spaceId, reconnects = 0) {
4411
- (0, import_invariant14.invariant)(this._context, void 0, {
4412
- F: __dxlog_file18,
4413
- L: 124,
4414
- S: this,
4415
- A: [
4416
- "this._context",
4417
- ""
4418
- ]
4419
- });
4420
- (0, import_invariant14.invariant)(!this._connections.has(spaceId), void 0, {
4421
- F: __dxlog_file18,
4422
- L: 125,
4423
- S: this,
4424
- A: [
4425
- "!this._connections.has(spaceId)",
4426
- ""
4427
- ]
4428
- });
4429
- let restartScheduled = false;
4430
- const connection = new EdgeReplicatorConnection({
4431
- edgeConnection: this._edgeConnection,
4432
- spaceId,
4433
- context: this._context,
4434
- sharedPolicyEnabled: this._sharePolicyEnabled,
4435
- onRemoteConnected: async () => {
4436
- this._context?.onConnectionOpen(connection);
4437
- },
4438
- onRemoteDisconnected: async () => {
4439
- this._context?.onConnectionClosed(connection);
4440
- },
4441
- onRestartRequested: async () => {
4442
- if (!this._ctx || restartScheduled) {
4443
- return;
4444
- }
4445
- const restartDelay = Math.min(MAX_RESTART_DELAY, INITIAL_RESTART_DELAY * reconnects) + Math.random() * RESTART_DELAY_JITTER;
4446
- (0, import_log12.log)("connection restart scheduled", {
4447
- spaceId,
4448
- reconnects,
4449
- restartDelay
4450
- }, {
4451
- F: __dxlog_file18,
4452
- L: 148,
4453
- S: this,
4454
- C: (f, a) => f(...a)
4455
- });
4456
- restartScheduled = true;
4457
- (0, import_async8.scheduleTask)(this._ctx, async () => {
4458
- const env = {
4459
- stack: [],
4460
- error: void 0,
4461
- hasError: false
4462
- };
4463
- try {
4464
- const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
4465
- if (this._connections.get(spaceId) !== connection) {
4466
- return;
4467
- }
4468
- const ctx = this._ctx;
4469
- await connection.close();
4470
- this._connections.delete(spaceId);
4471
- if (ctx?.disposed) {
4472
- return;
4473
- }
4474
- await this._openConnection(spaceId, reconnects + 1);
4475
- } catch (e) {
4476
- env.error = e;
4477
- env.hasError = true;
4478
- } finally {
4479
- _ts_dispose_resources(env);
4480
- }
4481
- }, restartDelay);
4482
- }
4483
- });
4484
- this._connections.set(spaceId, connection);
4485
- await connection.open();
4486
- }
4487
- };
4488
- var MAX_INFLIGHT_REQUESTS = 5;
4489
- var MAX_RATE_LIMIT_WAIT_TIME_MS = 3e3;
4490
- var EdgeReplicatorConnection = class extends import_context12.Resource {
4491
- constructor({ edgeConnection, spaceId, context, sharedPolicyEnabled, onRemoteConnected, onRemoteDisconnected, onRestartRequested }) {
4492
- super();
4493
- this._remotePeerId = null;
4494
- this._requestLimiter = new InflightRequestLimiter({
4495
- maxInflightRequests: MAX_INFLIGHT_REQUESTS,
4496
- resetBalanceTimeoutMs: MAX_RATE_LIMIT_WAIT_TIME_MS
4497
- });
4498
- this._edgeConnection = edgeConnection;
4499
- this._spaceId = spaceId;
4500
- this._context = context;
4501
- this._remotePeerId = `${import_protocols6.EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}-${(0, import_crypto.randomUUID)()}`;
4502
- this._targetServiceId = `${import_protocols6.EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}`;
4503
- this._sharedPolicyEnabled = sharedPolicyEnabled;
4504
- this._onRemoteConnected = onRemoteConnected;
4505
- this._onRemoteDisconnected = onRemoteDisconnected;
4506
- this._onRestartRequested = onRestartRequested;
4507
- this.readable = new ReadableStream({
4508
- start: (controller) => {
4509
- this._readableStreamController = controller;
4510
- }
4511
- });
4512
- this.writable = new WritableStream({
4513
- write: async (message, controller) => {
4514
- await this._requestLimiter.rateLimit(message);
4515
- await this._sendMessage(message);
4516
- }
4517
- });
4518
- }
4519
- async _open(ctx) {
4520
- (0, import_log12.log)("opening...", void 0, {
4521
- F: __dxlog_file18,
4522
- L: 251,
4523
- S: this,
4524
- C: (f, a) => f(...a)
4525
- });
4526
- await this._requestLimiter.open();
4527
- this._ctx.onDispose(this._edgeConnection.onMessage((msg) => {
4528
- this._onMessage(msg);
4529
- }));
4530
- await this._onRemoteConnected();
4531
- }
4532
- async _close() {
4533
- (0, import_log12.log)("closing...", void 0, {
4534
- F: __dxlog_file18,
4535
- L: 266,
4536
- S: this,
4537
- C: (f, a) => f(...a)
4538
- });
4539
- this._readableStreamController.close();
4540
- await this._requestLimiter.close();
4541
- await this._onRemoteDisconnected();
4542
- }
4543
- get peerId() {
4544
- (0, import_invariant14.invariant)(this._remotePeerId, "Not connected", {
4545
- F: __dxlog_file18,
4546
- L: 275,
4547
- S: this,
4548
- A: [
4549
- "this._remotePeerId",
4550
- "'Not connected'"
4551
- ]
4552
- });
4553
- return this._remotePeerId;
4554
- }
4555
- async shouldAdvertise(params) {
4556
- if (!this._sharedPolicyEnabled) {
4557
- return true;
4558
- }
4559
- const spaceId = await this._context.getContainingSpaceIdForDocument(params.documentId);
4560
- if (!spaceId) {
4561
- const remoteDocumentExists = await this._context.isDocumentInRemoteCollection({
4562
- documentId: params.documentId,
4563
- peerId: this._remotePeerId
4564
- });
4565
- import_log12.log.verbose("document not found locally for share policy check", {
4566
- documentId: params.documentId,
4567
- acceptDocument: remoteDocumentExists,
4568
- remoteId: this._remotePeerId
4569
- }, {
4570
- F: __dxlog_file18,
4571
- L: 290,
4572
- S: this,
4573
- C: (f, a) => f(...a)
4574
- });
4575
- return remoteDocumentExists;
4576
- }
4577
- return spaceId === this._spaceId;
4578
- }
4579
- shouldSyncCollection(params) {
4580
- if (!this._sharedPolicyEnabled) {
4581
- return true;
4582
- }
4583
- const spaceId = getSpaceIdFromCollectionId(params.collectionId);
4584
- return spaceId === this._spaceId && params.collectionId.split(":").length === 3;
4585
- }
4586
- _onMessage(message) {
4587
- if (message.serviceId !== this._targetServiceId) {
4588
- return;
4589
- }
4590
- const payload = import_automerge_repo6.cbor.decode(message.payload.value);
4591
- import_log12.log.verbose("received", {
4592
- type: payload.type,
4593
- documentId: payload.type === "sync" && payload.documentId,
4594
- remoteId: this._remotePeerId
4595
- }, {
4596
- F: __dxlog_file18,
4597
- L: 319,
4598
- S: this,
4599
- C: (f, a) => f(...a)
4600
- });
4601
- payload.senderId = this._remotePeerId;
4602
- this._processMessage(payload);
4603
- }
4604
- _processMessage(message) {
4605
- if (isForbiddenErrorMessage(message)) {
4606
- this._onRestartRequested();
4607
- return;
4608
- }
4609
- this._requestLimiter.handleResponse(message);
4610
- this._readableStreamController.enqueue(message);
4611
- }
4612
- async _sendMessage(message) {
4613
- message.targetId = this._targetServiceId;
4614
- import_log12.log.verbose("sending...", {
4615
- type: message.type,
4616
- documentId: message.type === "sync" && message.documentId,
4617
- remoteId: this._remotePeerId
4618
- }, {
4619
- F: __dxlog_file18,
4620
- L: 348,
4621
- S: this,
4622
- C: (f, a) => f(...a)
4623
- });
4624
- const encoded = import_automerge_repo6.cbor.encode(message);
4625
- await this._edgeConnection.send(import_buf.buf.create(import_messenger_pb.MessageSchema, {
4626
- serviceId: this._targetServiceId,
4627
- source: {
4628
- identityKey: this._edgeConnection.identityKey,
4629
- peerKey: this._edgeConnection.peerKey
4630
- },
4631
- payload: {
4632
- value: (0, import_util7.bufferToArray)(encoded)
4633
- }
4634
- }));
4635
- }
4636
- };
4637
- var isForbiddenErrorMessage = (message) => message.type === "error" && message.message === "Forbidden";
4638
- var findInlineObjectOfType = (spaceDoc, typename) => {
4639
- for (const id in spaceDoc.objects ?? {}) {
4640
- const obj = spaceDoc.objects[id];
4641
- const objType = import_echo_protocol7.ObjectStructure.getTypeReference(obj);
4642
- if (objType && (0, import_echo_protocol7.decodeReference)(objType).objectId === typename) {
4643
- return [
4644
- id,
4645
- obj
4646
- ];
4647
- }
4648
- }
4649
- return void 0;
4650
- };
4651
- // Annotate the CommonJS export names for ESM import in node:
4652
- 0 && (module.exports = {
4653
- AuthExtension,
4654
- AuthStatus,
4655
- AutomergeHost,
4656
- CredentialRetrieverExtension,
4657
- CredentialServerExtension,
4658
- DataServiceImpl,
4659
- DatabaseRoot,
4660
- DocumentsSynchronizer,
4661
- EchoDataMonitor,
4662
- EchoEdgeReplicator,
4663
- EchoHost,
4664
- ExecutionTrace,
4665
- FIND_PARAMS,
4666
- LevelDBStorageAdapter,
4667
- MOCK_AUTH_PROVIDER,
4668
- MOCK_AUTH_VERIFIER,
4669
- MeshEchoReplicator,
4670
- MetadataStore,
4671
- Pipeline,
4672
- QueryExecutor,
4673
- QueryPlan,
4674
- QueryPlanner,
4675
- QueryServiceImpl,
4676
- Space,
4677
- SpaceDocumentListUpdatedEvent,
4678
- SpaceManager,
4679
- SpaceProtocol,
4680
- SpaceProtocolSession,
4681
- SpaceStateManager,
4682
- TimeframeClock,
4683
- codec,
4684
- createIdFromSpaceKey,
4685
- createMappedFeedWriter,
4686
- deriveCollectionIdFromSpaceId,
4687
- diffCollectionState,
4688
- encodingOptions,
4689
- filterMatchObject,
4690
- filterMatchValue,
4691
- findInlineObjectOfType,
4692
- getSpaceIdFromCollectionId,
4693
- hasInvitationExpired,
4694
- mapFeedIndexesToTimeframe,
4695
- mapTimeframeToFeedIndexes,
4696
- startAfter,
4697
- valueEncoding
4698
- });
4699
- //# sourceMappingURL=index.cjs.map