@dxos/echo-pipeline 0.6.12-staging.e11e696 → 0.6.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/{node-esm/index.mjs → browser/chunk-MPWFDDQK.mjs} +309 -1614
- package/dist/lib/browser/chunk-MPWFDDQK.mjs.map +7 -0
- package/dist/lib/{node-esm/chunk-ASOERPR6.mjs → browser/chunk-UKXIJW43.mjs} +26 -52
- package/dist/lib/browser/chunk-UKXIJW43.mjs.map +7 -0
- package/dist/lib/browser/chunk-XPCF2V5U.mjs +31 -0
- package/dist/lib/browser/chunk-XPCF2V5U.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +17 -3401
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/light.mjs +32 -0
- package/dist/lib/browser/light.mjs.map +7 -0
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +4 -3
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/chunk-5DH4KR2S.cjs +2148 -0
- package/dist/lib/node/chunk-5DH4KR2S.cjs.map +7 -0
- package/dist/lib/node/{chunk-6SSC2M4R.cjs → chunk-7HHYCGUR.cjs} +37 -65
- package/dist/lib/node/chunk-7HHYCGUR.cjs.map +7 -0
- package/dist/lib/node/chunk-DZVH7HDD.cjs +43 -0
- package/dist/lib/node/chunk-DZVH7HDD.cjs.map +7 -0
- package/dist/lib/node/index.cjs +35 -3378
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/light.cjs +52 -0
- package/dist/lib/node/light.cjs.map +7 -0
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +12 -11
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/automerge/automerge-host.d.ts +1 -24
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts +0 -2
- package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/space-collection.d.ts +2 -3
- package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts +2 -3
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/index.d.ts +0 -5
- package/dist/types/src/db-host/index.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +0 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/light.d.ts +4 -0
- package/dist/types/src/light.d.ts.map +1 -0
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/package.json +50 -40
- package/src/automerge/automerge-host.test.ts +9 -8
- package/src/automerge/automerge-host.ts +6 -42
- package/src/automerge/automerge-repo.test.ts +16 -18
- package/src/automerge/collection-synchronizer.test.ts +5 -10
- package/src/automerge/collection-synchronizer.ts +6 -17
- package/src/automerge/echo-data-monitor.test.ts +3 -1
- package/src/automerge/echo-network-adapter.test.ts +3 -4
- package/src/automerge/mesh-echo-replicator.ts +1 -2
- package/src/automerge/space-collection.ts +2 -3
- package/src/automerge/storage-adapter.test.ts +3 -2
- package/src/db-host/data-service.ts +14 -29
- package/src/db-host/documents-synchronizer.test.ts +2 -2
- package/src/db-host/index.ts +1 -6
- package/src/index.ts +0 -1
- package/src/light.ts +7 -0
- package/src/pipeline/pipeline-stress.test.ts +47 -44
- package/src/pipeline/pipeline.test.ts +4 -3
- package/src/space/control-pipeline.test.ts +3 -2
- package/src/space/control-pipeline.ts +1 -10
- package/src/space/replication.browser.test.ts +8 -2
- package/src/space/space-manager.browser.test.ts +5 -6
- package/src/space/space-protocol.browser.test.ts +34 -29
- package/src/space/space-protocol.test.ts +27 -29
- package/src/space/space.test.ts +11 -28
- package/src/testing/test-agent-builder.ts +2 -2
- package/dist/lib/browser/chunk-YTSGJIY7.mjs +0 -2036
- package/dist/lib/browser/chunk-YTSGJIY7.mjs.map +0 -7
- package/dist/lib/node/chunk-6SSC2M4R.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-ASOERPR6.mjs.map +0 -7
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
- package/dist/lib/node-esm/testing/index.mjs +0 -551
- package/dist/lib/node-esm/testing/index.mjs.map +0 -7
- package/dist/types/src/db-host/automerge-metrics.d.ts +0 -11
- package/dist/types/src/db-host/automerge-metrics.d.ts.map +0 -1
- package/dist/types/src/db-host/database-root.d.ts +0 -20
- package/dist/types/src/db-host/database-root.d.ts.map +0 -1
- package/dist/types/src/db-host/documents-iterator.d.ts +0 -7
- package/dist/types/src/db-host/documents-iterator.d.ts.map +0 -1
- package/dist/types/src/db-host/echo-host.d.ts +0 -73
- package/dist/types/src/db-host/echo-host.d.ts.map +0 -1
- package/dist/types/src/db-host/migration.d.ts +0 -8
- package/dist/types/src/db-host/migration.d.ts.map +0 -1
- package/dist/types/src/db-host/query-service.d.ts +0 -25
- package/dist/types/src/db-host/query-service.d.ts.map +0 -1
- package/dist/types/src/db-host/query-state.d.ts +0 -41
- package/dist/types/src/db-host/query-state.d.ts.map +0 -1
- package/dist/types/src/db-host/space-state-manager.d.ts +0 -23
- package/dist/types/src/db-host/space-state-manager.d.ts.map +0 -1
- package/dist/types/src/edge/echo-edge-replicator.d.ts +0 -23
- package/dist/types/src/edge/echo-edge-replicator.d.ts.map +0 -1
- package/dist/types/src/edge/index.d.ts +0 -2
- package/dist/types/src/edge/index.d.ts.map +0 -1
- package/src/db-host/automerge-metrics.ts +0 -38
- package/src/db-host/database-root.ts +0 -86
- package/src/db-host/documents-iterator.ts +0 -73
- package/src/db-host/echo-host.ts +0 -257
- package/src/db-host/migration.ts +0 -57
- package/src/db-host/query-service.ts +0 -208
- package/src/db-host/query-state.ts +0 -200
- package/src/db-host/space-state-manager.ts +0 -90
- package/src/edge/echo-edge-replicator.ts +0 -286
- package/src/edge/index.ts +0 -5
|
@@ -1,34 +1,8 @@
|
|
|
1
|
-
import
|
|
1
|
+
import "@dxos/node-std/globals";
|
|
2
2
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
CredentialServerExtension,
|
|
7
|
-
MOCK_AUTH_PROVIDER,
|
|
8
|
-
MOCK_AUTH_VERIFIER,
|
|
9
|
-
MetadataStore,
|
|
10
|
-
Pipeline,
|
|
11
|
-
Space,
|
|
12
|
-
SpaceManager,
|
|
13
|
-
SpaceProtocol,
|
|
14
|
-
SpaceProtocolSession,
|
|
15
|
-
TimeframeClock,
|
|
16
|
-
codec,
|
|
17
|
-
createIdFromSpaceKey,
|
|
18
|
-
createMappedFeedWriter,
|
|
19
|
-
hasInvitationExpired,
|
|
20
|
-
mapFeedIndexesToTimeframe,
|
|
21
|
-
mapTimeframeToFeedIndexes,
|
|
22
|
-
startAfter,
|
|
23
|
-
valueEncoding
|
|
24
|
-
} from "./chunk-ASOERPR6.mjs";
|
|
25
|
-
|
|
26
|
-
// packages/core/echo/echo-pipeline/src/db-host/data-service.ts
|
|
27
|
-
import { UpdateScheduler as UpdateScheduler2 } from "@dxos/async";
|
|
28
|
-
import { Stream } from "@dxos/codec-protobuf/stream";
|
|
29
|
-
import { invariant as invariant7 } from "@dxos/invariant";
|
|
30
|
-
import { SpaceId as SpaceId2 } from "@dxos/keys";
|
|
31
|
-
import { log as log7 } from "@dxos/log";
|
|
3
|
+
Buffer,
|
|
4
|
+
createIdFromSpaceKey
|
|
5
|
+
} from "./chunk-XPCF2V5U.mjs";
|
|
32
6
|
|
|
33
7
|
// packages/core/echo/echo-pipeline/src/db-host/documents-synchronizer.ts
|
|
34
8
|
import { UpdateScheduler } from "@dxos/async";
|
|
@@ -196,24 +170,11 @@ var DocumentsSynchronizer = class extends Resource {
|
|
|
196
170
|
}
|
|
197
171
|
};
|
|
198
172
|
|
|
199
|
-
// packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
|
|
200
|
-
import { Event as Event2, asyncTimeout } from "@dxos/async";
|
|
201
|
-
import { getBackend, getHeads, isAutomerge, equals as headsEquals, save } from "@dxos/automerge/automerge";
|
|
202
|
-
import { Repo } from "@dxos/automerge/automerge-repo";
|
|
203
|
-
import { Context, Resource as Resource4, cancelWithContext } from "@dxos/context";
|
|
204
|
-
import { invariant as invariant3 } from "@dxos/invariant";
|
|
205
|
-
import { PublicKey } from "@dxos/keys";
|
|
206
|
-
import { log as log4 } from "@dxos/log";
|
|
207
|
-
import { objectPointerCodec } from "@dxos/protocols";
|
|
208
|
-
import { trace } from "@dxos/tracing";
|
|
209
|
-
|
|
210
173
|
// packages/core/echo/echo-pipeline/src/automerge/collection-synchronizer.ts
|
|
211
174
|
import { asyncReturn, Event, scheduleTask, scheduleTaskInterval } from "@dxos/async";
|
|
212
175
|
import { next as am } from "@dxos/automerge/automerge";
|
|
213
176
|
import { Resource as Resource2 } from "@dxos/context";
|
|
214
|
-
import { log as log2 } from "@dxos/log";
|
|
215
177
|
import { defaultMap } from "@dxos/util";
|
|
216
|
-
var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/collection-synchronizer.ts";
|
|
217
178
|
var MIN_QUERY_INTERVAL = 5e3;
|
|
218
179
|
var POLL_INTERVAL = 3e4;
|
|
219
180
|
var CollectionSynchronizer = class extends Resource2 {
|
|
@@ -246,15 +207,6 @@ var CollectionSynchronizer = class extends Resource2 {
|
|
|
246
207
|
return this._getPerCollectionState(collectionId).localState;
|
|
247
208
|
}
|
|
248
209
|
setLocalCollectionState(collectionId, state) {
|
|
249
|
-
log2("setLocalCollectionState", {
|
|
250
|
-
collectionId,
|
|
251
|
-
state
|
|
252
|
-
}, {
|
|
253
|
-
F: __dxlog_file2,
|
|
254
|
-
L: 68,
|
|
255
|
-
S: this,
|
|
256
|
-
C: (f, a) => f(...a)
|
|
257
|
-
});
|
|
258
210
|
this._getPerCollectionState(collectionId).localState = state;
|
|
259
211
|
queueMicrotask(async () => {
|
|
260
212
|
if (!this._ctx.disposed) {
|
|
@@ -324,16 +276,6 @@ var CollectionSynchronizer = class extends Resource2 {
|
|
|
324
276
|
* Callback when a peer sends the state of a collection.
|
|
325
277
|
*/
|
|
326
278
|
onRemoteStateReceived(collectionId, peerId, state) {
|
|
327
|
-
log2("onRemoteStateReceived", {
|
|
328
|
-
collectionId,
|
|
329
|
-
peerId,
|
|
330
|
-
state
|
|
331
|
-
}, {
|
|
332
|
-
F: __dxlog_file2,
|
|
333
|
-
L: 148,
|
|
334
|
-
S: this,
|
|
335
|
-
C: (f, a) => f(...a)
|
|
336
|
-
});
|
|
337
279
|
validateCollectionState(state);
|
|
338
280
|
const perCollectionState = this._getPerCollectionState(collectionId);
|
|
339
281
|
perCollectionState.remoteStates.set(peerId, state);
|
|
@@ -365,21 +307,13 @@ var diffCollectionState = (local, remote) => {
|
|
|
365
307
|
...Object.keys(local.documents),
|
|
366
308
|
...Object.keys(remote.documents)
|
|
367
309
|
]);
|
|
368
|
-
const missingOnRemote = [];
|
|
369
|
-
const missingOnLocal = [];
|
|
370
310
|
const different = [];
|
|
371
311
|
for (const documentId of allDocuments) {
|
|
372
|
-
if (!local.documents[documentId]) {
|
|
373
|
-
missingOnLocal.push(documentId);
|
|
374
|
-
} else if (!remote.documents[documentId]) {
|
|
375
|
-
missingOnRemote.push(documentId);
|
|
376
|
-
} else if (!am.equals(local.documents[documentId], remote.documents[documentId])) {
|
|
312
|
+
if (!local.documents[documentId] || !remote.documents[documentId] || !am.equals(local.documents[documentId], remote.documents[documentId])) {
|
|
377
313
|
different.push(documentId);
|
|
378
314
|
}
|
|
379
315
|
}
|
|
380
316
|
return {
|
|
381
|
-
missingOnRemote,
|
|
382
|
-
missingOnLocal,
|
|
383
317
|
different
|
|
384
318
|
};
|
|
385
319
|
};
|
|
@@ -397,12 +331,129 @@ var isValidDocumentId = (documentId) => {
|
|
|
397
331
|
return typeof documentId === "string" && !documentId.includes(":");
|
|
398
332
|
};
|
|
399
333
|
|
|
334
|
+
// packages/core/echo/echo-pipeline/src/automerge/leveldb-storage-adapter.ts
|
|
335
|
+
import { LifecycleState, Resource as Resource3 } from "@dxos/context";
|
|
336
|
+
var LevelDBStorageAdapter = class extends Resource3 {
|
|
337
|
+
constructor(_params) {
|
|
338
|
+
super();
|
|
339
|
+
this._params = _params;
|
|
340
|
+
}
|
|
341
|
+
async load(keyArray) {
|
|
342
|
+
try {
|
|
343
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
344
|
+
return void 0;
|
|
345
|
+
}
|
|
346
|
+
const startMs = Date.now();
|
|
347
|
+
const chunk = await this._params.db.get(keyArray, {
|
|
348
|
+
...encodingOptions
|
|
349
|
+
});
|
|
350
|
+
this._params.monitor?.recordBytesLoaded(chunk.byteLength);
|
|
351
|
+
this._params.monitor?.recordLoadDuration(Date.now() - startMs);
|
|
352
|
+
return chunk;
|
|
353
|
+
} catch (err) {
|
|
354
|
+
if (isLevelDbNotFoundError(err)) {
|
|
355
|
+
return void 0;
|
|
356
|
+
}
|
|
357
|
+
throw err;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
async save(keyArray, binary) {
|
|
361
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
362
|
+
return void 0;
|
|
363
|
+
}
|
|
364
|
+
const startMs = Date.now();
|
|
365
|
+
const batch = this._params.db.batch();
|
|
366
|
+
await this._params.callbacks?.beforeSave?.({
|
|
367
|
+
path: keyArray,
|
|
368
|
+
batch
|
|
369
|
+
});
|
|
370
|
+
batch.put(keyArray, Buffer.from(binary), {
|
|
371
|
+
...encodingOptions
|
|
372
|
+
});
|
|
373
|
+
await batch.write();
|
|
374
|
+
this._params.monitor?.recordBytesStored(binary.byteLength);
|
|
375
|
+
await this._params.callbacks?.afterSave?.(keyArray);
|
|
376
|
+
this._params.monitor?.recordStoreDuration(Date.now() - startMs);
|
|
377
|
+
}
|
|
378
|
+
async remove(keyArray) {
|
|
379
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
380
|
+
return void 0;
|
|
381
|
+
}
|
|
382
|
+
await this._params.db.del(keyArray, {
|
|
383
|
+
...encodingOptions
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
async loadRange(keyPrefix) {
|
|
387
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
388
|
+
return [];
|
|
389
|
+
}
|
|
390
|
+
const startMs = Date.now();
|
|
391
|
+
const result = [];
|
|
392
|
+
for await (const [key, value] of this._params.db.iterator({
|
|
393
|
+
gte: keyPrefix,
|
|
394
|
+
lte: [
|
|
395
|
+
...keyPrefix,
|
|
396
|
+
"\uFFFF"
|
|
397
|
+
],
|
|
398
|
+
...encodingOptions
|
|
399
|
+
})) {
|
|
400
|
+
result.push({
|
|
401
|
+
key,
|
|
402
|
+
data: value
|
|
403
|
+
});
|
|
404
|
+
this._params.monitor?.recordBytesLoaded(value.byteLength);
|
|
405
|
+
}
|
|
406
|
+
this._params.monitor?.recordLoadDuration(Date.now() - startMs);
|
|
407
|
+
return result;
|
|
408
|
+
}
|
|
409
|
+
async removeRange(keyPrefix) {
|
|
410
|
+
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
411
|
+
return void 0;
|
|
412
|
+
}
|
|
413
|
+
const batch = this._params.db.batch();
|
|
414
|
+
for await (const [key] of this._params.db.iterator({
|
|
415
|
+
gte: keyPrefix,
|
|
416
|
+
lte: [
|
|
417
|
+
...keyPrefix,
|
|
418
|
+
"\uFFFF"
|
|
419
|
+
],
|
|
420
|
+
...encodingOptions
|
|
421
|
+
})) {
|
|
422
|
+
batch.del(key, {
|
|
423
|
+
...encodingOptions
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
await batch.write();
|
|
427
|
+
}
|
|
428
|
+
};
|
|
429
|
+
var keyEncoder = {
|
|
430
|
+
encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
|
|
431
|
+
decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
|
|
432
|
+
format: "buffer"
|
|
433
|
+
};
|
|
434
|
+
var encodingOptions = {
|
|
435
|
+
keyEncoding: keyEncoder,
|
|
436
|
+
valueEncoding: "buffer"
|
|
437
|
+
};
|
|
438
|
+
var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
|
|
439
|
+
|
|
440
|
+
// packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
|
|
441
|
+
import { Event as Event2, asyncTimeout } from "@dxos/async";
|
|
442
|
+
import { getBackend, getHeads, isAutomerge, equals as headsEquals, save } from "@dxos/automerge/automerge";
|
|
443
|
+
import { Repo } from "@dxos/automerge/automerge-repo";
|
|
444
|
+
import { Context, Resource as Resource4, cancelWithContext } from "@dxos/context";
|
|
445
|
+
import { invariant as invariant3 } from "@dxos/invariant";
|
|
446
|
+
import { PublicKey } from "@dxos/keys";
|
|
447
|
+
import { log as log3 } from "@dxos/log";
|
|
448
|
+
import { objectPointerCodec } from "@dxos/protocols";
|
|
449
|
+
import { trace } from "@dxos/tracing";
|
|
450
|
+
|
|
400
451
|
// packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts
|
|
401
452
|
import { synchronized, Trigger } from "@dxos/async";
|
|
402
453
|
import { NetworkAdapter } from "@dxos/automerge/automerge-repo";
|
|
403
|
-
import { LifecycleState } from "@dxos/context";
|
|
454
|
+
import { LifecycleState as LifecycleState2 } from "@dxos/context";
|
|
404
455
|
import { invariant as invariant2 } from "@dxos/invariant";
|
|
405
|
-
import { log as
|
|
456
|
+
import { log as log2 } from "@dxos/log";
|
|
406
457
|
import { nonNullable } from "@dxos/util";
|
|
407
458
|
|
|
408
459
|
// packages/core/echo/echo-pipeline/src/automerge/network-protocol.ts
|
|
@@ -417,14 +468,14 @@ function _ts_decorate(decorators, target, key, desc) {
|
|
|
417
468
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
418
469
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
419
470
|
}
|
|
420
|
-
var
|
|
471
|
+
var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
|
|
421
472
|
var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
422
473
|
constructor(_params) {
|
|
423
474
|
super();
|
|
424
475
|
this._params = _params;
|
|
425
476
|
this._replicators = /* @__PURE__ */ new Set();
|
|
426
477
|
this._connections = /* @__PURE__ */ new Map();
|
|
427
|
-
this._lifecycleState =
|
|
478
|
+
this._lifecycleState = LifecycleState2.CLOSED;
|
|
428
479
|
this._connected = new Trigger();
|
|
429
480
|
}
|
|
430
481
|
connect(peerId, peerMetadata) {
|
|
@@ -438,12 +489,12 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
438
489
|
disconnect() {
|
|
439
490
|
}
|
|
440
491
|
async open() {
|
|
441
|
-
if (this._lifecycleState ===
|
|
492
|
+
if (this._lifecycleState === LifecycleState2.OPEN) {
|
|
442
493
|
return;
|
|
443
494
|
}
|
|
444
|
-
this._lifecycleState =
|
|
445
|
-
|
|
446
|
-
F:
|
|
495
|
+
this._lifecycleState = LifecycleState2.OPEN;
|
|
496
|
+
log2("emit ready", void 0, {
|
|
497
|
+
F: __dxlog_file2,
|
|
447
498
|
L: 81,
|
|
448
499
|
S: this,
|
|
449
500
|
C: (f, a) => f(...a)
|
|
@@ -453,14 +504,14 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
453
504
|
});
|
|
454
505
|
}
|
|
455
506
|
async close() {
|
|
456
|
-
if (this._lifecycleState ===
|
|
507
|
+
if (this._lifecycleState === LifecycleState2.CLOSED) {
|
|
457
508
|
return this;
|
|
458
509
|
}
|
|
459
510
|
for (const replicator of this._replicators) {
|
|
460
511
|
await replicator.disconnect();
|
|
461
512
|
}
|
|
462
513
|
this._replicators.clear();
|
|
463
|
-
this._lifecycleState =
|
|
514
|
+
this._lifecycleState = LifecycleState2.CLOSED;
|
|
464
515
|
}
|
|
465
516
|
async whenConnected() {
|
|
466
517
|
await this._connected.wait({
|
|
@@ -468,8 +519,8 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
468
519
|
});
|
|
469
520
|
}
|
|
470
521
|
async addReplicator(replicator) {
|
|
471
|
-
invariant2(this._lifecycleState ===
|
|
472
|
-
F:
|
|
522
|
+
invariant2(this._lifecycleState === LifecycleState2.OPEN, void 0, {
|
|
523
|
+
F: __dxlog_file2,
|
|
473
524
|
L: 107,
|
|
474
525
|
S: this,
|
|
475
526
|
A: [
|
|
@@ -478,7 +529,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
478
529
|
]
|
|
479
530
|
});
|
|
480
531
|
invariant2(this.peerId, void 0, {
|
|
481
|
-
F:
|
|
532
|
+
F: __dxlog_file2,
|
|
482
533
|
L: 108,
|
|
483
534
|
S: this,
|
|
484
535
|
A: [
|
|
@@ -487,7 +538,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
487
538
|
]
|
|
488
539
|
});
|
|
489
540
|
invariant2(!this._replicators.has(replicator), void 0, {
|
|
490
|
-
F:
|
|
541
|
+
F: __dxlog_file2,
|
|
491
542
|
L: 109,
|
|
492
543
|
S: this,
|
|
493
544
|
A: [
|
|
@@ -510,8 +561,8 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
510
561
|
});
|
|
511
562
|
}
|
|
512
563
|
async removeReplicator(replicator) {
|
|
513
|
-
invariant2(this._lifecycleState ===
|
|
514
|
-
F:
|
|
564
|
+
invariant2(this._lifecycleState === LifecycleState2.OPEN, void 0, {
|
|
565
|
+
F: __dxlog_file2,
|
|
515
566
|
L: 128,
|
|
516
567
|
S: this,
|
|
517
568
|
A: [
|
|
@@ -520,7 +571,7 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
520
571
|
]
|
|
521
572
|
});
|
|
522
573
|
invariant2(this._replicators.has(replicator), void 0, {
|
|
523
|
-
F:
|
|
574
|
+
F: __dxlog_file2,
|
|
524
575
|
L: 129,
|
|
525
576
|
S: this,
|
|
526
577
|
A: [
|
|
@@ -575,8 +626,8 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
575
626
|
this._params.monitor?.recordMessageSent(message, durationMs);
|
|
576
627
|
}).catch((err) => {
|
|
577
628
|
if (connectionEntry.isOpen) {
|
|
578
|
-
|
|
579
|
-
F:
|
|
629
|
+
log2.catch(err, void 0, {
|
|
630
|
+
F: __dxlog_file2,
|
|
580
631
|
L: 189,
|
|
581
632
|
S: this,
|
|
582
633
|
C: (f, a) => f(...a)
|
|
@@ -594,16 +645,16 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
594
645
|
}).filter(nonNullable);
|
|
595
646
|
}
|
|
596
647
|
_onConnectionOpen(connection) {
|
|
597
|
-
|
|
648
|
+
log2("Connection opened", {
|
|
598
649
|
peerId: connection.peerId
|
|
599
650
|
}, {
|
|
600
|
-
F:
|
|
651
|
+
F: __dxlog_file2,
|
|
601
652
|
L: 207,
|
|
602
653
|
S: this,
|
|
603
654
|
C: (f, a) => f(...a)
|
|
604
655
|
});
|
|
605
656
|
invariant2(!this._connections.has(connection.peerId), void 0, {
|
|
606
|
-
F:
|
|
657
|
+
F: __dxlog_file2,
|
|
607
658
|
L: 208,
|
|
608
659
|
S: this,
|
|
609
660
|
A: [
|
|
@@ -631,8 +682,8 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
631
682
|
}
|
|
632
683
|
} catch (err) {
|
|
633
684
|
if (connectionEntry.isOpen) {
|
|
634
|
-
|
|
635
|
-
F:
|
|
685
|
+
log2.catch(err, void 0, {
|
|
686
|
+
F: __dxlog_file2,
|
|
636
687
|
L: 227,
|
|
637
688
|
S: this,
|
|
638
689
|
C: (f, a) => f(...a)
|
|
@@ -640,10 +691,10 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
640
691
|
}
|
|
641
692
|
}
|
|
642
693
|
});
|
|
643
|
-
|
|
694
|
+
log2("emit peer-candidate", {
|
|
644
695
|
peerId: connection.peerId
|
|
645
696
|
}, {
|
|
646
|
-
F:
|
|
697
|
+
F: __dxlog_file2,
|
|
647
698
|
L: 232,
|
|
648
699
|
S: this,
|
|
649
700
|
C: (f, a) => f(...a)
|
|
@@ -666,17 +717,17 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
666
717
|
* TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
|
|
667
718
|
*/
|
|
668
719
|
_onConnectionAuthScopeChanged(connection) {
|
|
669
|
-
|
|
720
|
+
log2("Connection auth scope changed", {
|
|
670
721
|
peerId: connection.peerId
|
|
671
722
|
}, {
|
|
672
|
-
F:
|
|
723
|
+
F: __dxlog_file2,
|
|
673
724
|
L: 253,
|
|
674
725
|
S: this,
|
|
675
726
|
C: (f, a) => f(...a)
|
|
676
727
|
});
|
|
677
728
|
const entry = this._connections.get(connection.peerId);
|
|
678
729
|
invariant2(entry, void 0, {
|
|
679
|
-
F:
|
|
730
|
+
F: __dxlog_file2,
|
|
680
731
|
L: 255,
|
|
681
732
|
S: this,
|
|
682
733
|
A: [
|
|
@@ -690,17 +741,17 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
690
741
|
this._emitPeerCandidate(connection);
|
|
691
742
|
}
|
|
692
743
|
_onConnectionClosed(connection) {
|
|
693
|
-
|
|
744
|
+
log2("Connection closed", {
|
|
694
745
|
peerId: connection.peerId
|
|
695
746
|
}, {
|
|
696
|
-
F:
|
|
747
|
+
F: __dxlog_file2,
|
|
697
748
|
L: 261,
|
|
698
749
|
S: this,
|
|
699
750
|
C: (f, a) => f(...a)
|
|
700
751
|
});
|
|
701
752
|
const entry = this._connections.get(connection.peerId);
|
|
702
753
|
invariant2(entry, void 0, {
|
|
703
|
-
F:
|
|
754
|
+
F: __dxlog_file2,
|
|
704
755
|
L: 263,
|
|
705
756
|
S: this,
|
|
706
757
|
A: [
|
|
@@ -713,14 +764,14 @@ var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
|
713
764
|
peerId: connection.peerId
|
|
714
765
|
});
|
|
715
766
|
this._params.monitor?.recordPeerDisconnected(connection.peerId);
|
|
716
|
-
void entry.reader.cancel().catch((err) =>
|
|
717
|
-
F:
|
|
767
|
+
void entry.reader.cancel().catch((err) => log2.catch(err, void 0, {
|
|
768
|
+
F: __dxlog_file2,
|
|
718
769
|
L: 269,
|
|
719
770
|
S: this,
|
|
720
771
|
C: (f, a) => f(...a)
|
|
721
772
|
}));
|
|
722
|
-
void entry.writer.abort().catch((err) =>
|
|
723
|
-
F:
|
|
773
|
+
void entry.writer.abort().catch((err) => log2.catch(err, void 0, {
|
|
774
|
+
F: __dxlog_file2,
|
|
724
775
|
L: 270,
|
|
725
776
|
S: this,
|
|
726
777
|
C: (f, a) => f(...a)
|
|
@@ -774,149 +825,42 @@ var HeadsStore = class {
|
|
|
774
825
|
}
|
|
775
826
|
};
|
|
776
827
|
|
|
777
|
-
// packages/core/echo/echo-pipeline/src/automerge/
|
|
778
|
-
|
|
779
|
-
var
|
|
780
|
-
|
|
828
|
+
// packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
|
|
829
|
+
function _ts_decorate2(decorators, target, key, desc) {
|
|
830
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
831
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
832
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
833
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
834
|
+
}
|
|
835
|
+
var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
|
|
836
|
+
var AutomergeHost = class extends Resource4 {
|
|
837
|
+
constructor({ db, indexMetadataStore, dataMonitor }) {
|
|
781
838
|
super();
|
|
782
|
-
this.
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
787
|
-
return void 0;
|
|
788
|
-
}
|
|
789
|
-
const startMs = Date.now();
|
|
790
|
-
const chunk = await this._params.db.get(keyArray, {
|
|
791
|
-
...encodingOptions
|
|
792
|
-
});
|
|
793
|
-
this._params.monitor?.recordBytesLoaded(chunk.byteLength);
|
|
794
|
-
this._params.monitor?.recordLoadDuration(Date.now() - startMs);
|
|
795
|
-
return chunk;
|
|
796
|
-
} catch (err) {
|
|
797
|
-
if (isLevelDbNotFoundError(err)) {
|
|
798
|
-
return void 0;
|
|
799
|
-
}
|
|
800
|
-
throw err;
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
async save(keyArray, binary) {
|
|
804
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
805
|
-
return void 0;
|
|
806
|
-
}
|
|
807
|
-
const startMs = Date.now();
|
|
808
|
-
const batch = this._params.db.batch();
|
|
809
|
-
await this._params.callbacks?.beforeSave?.({
|
|
810
|
-
path: keyArray,
|
|
811
|
-
batch
|
|
839
|
+
this._collectionSynchronizer = new CollectionSynchronizer({
|
|
840
|
+
queryCollectionState: this._queryCollectionState.bind(this),
|
|
841
|
+
sendCollectionState: this._sendCollectionState.bind(this),
|
|
842
|
+
shouldSyncCollection: this._shouldSyncCollection.bind(this)
|
|
812
843
|
});
|
|
813
|
-
|
|
814
|
-
|
|
844
|
+
this._db = db;
|
|
845
|
+
this._storage = new LevelDBStorageAdapter({
|
|
846
|
+
db: db.sublevel("automerge"),
|
|
847
|
+
callbacks: {
|
|
848
|
+
beforeSave: async (params) => this._beforeSave(params),
|
|
849
|
+
afterSave: async (key) => this._afterSave(key)
|
|
850
|
+
},
|
|
851
|
+
monitor: dataMonitor
|
|
815
852
|
});
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
await this._params.db.del(keyArray, {
|
|
826
|
-
...encodingOptions
|
|
853
|
+
this._echoNetworkAdapter = new EchoNetworkAdapter({
|
|
854
|
+
getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
|
|
855
|
+
isDocumentInRemoteCollection: this._isDocumentInRemoteCollection.bind(this),
|
|
856
|
+
onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
|
|
857
|
+
onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
|
|
858
|
+
monitor: dataMonitor
|
|
859
|
+
});
|
|
860
|
+
this._headsStore = new HeadsStore({
|
|
861
|
+
db: db.sublevel("heads")
|
|
827
862
|
});
|
|
828
|
-
|
|
829
|
-
async loadRange(keyPrefix) {
|
|
830
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
831
|
-
return [];
|
|
832
|
-
}
|
|
833
|
-
const startMs = Date.now();
|
|
834
|
-
const result = [];
|
|
835
|
-
for await (const [key, value] of this._params.db.iterator({
|
|
836
|
-
gte: keyPrefix,
|
|
837
|
-
lte: [
|
|
838
|
-
...keyPrefix,
|
|
839
|
-
"\uFFFF"
|
|
840
|
-
],
|
|
841
|
-
...encodingOptions
|
|
842
|
-
})) {
|
|
843
|
-
result.push({
|
|
844
|
-
key,
|
|
845
|
-
data: value
|
|
846
|
-
});
|
|
847
|
-
this._params.monitor?.recordBytesLoaded(value.byteLength);
|
|
848
|
-
}
|
|
849
|
-
this._params.monitor?.recordLoadDuration(Date.now() - startMs);
|
|
850
|
-
return result;
|
|
851
|
-
}
|
|
852
|
-
async removeRange(keyPrefix) {
|
|
853
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
854
|
-
return void 0;
|
|
855
|
-
}
|
|
856
|
-
const batch = this._params.db.batch();
|
|
857
|
-
for await (const [key] of this._params.db.iterator({
|
|
858
|
-
gte: keyPrefix,
|
|
859
|
-
lte: [
|
|
860
|
-
...keyPrefix,
|
|
861
|
-
"\uFFFF"
|
|
862
|
-
],
|
|
863
|
-
...encodingOptions
|
|
864
|
-
})) {
|
|
865
|
-
batch.del(key, {
|
|
866
|
-
...encodingOptions
|
|
867
|
-
});
|
|
868
|
-
}
|
|
869
|
-
await batch.write();
|
|
870
|
-
}
|
|
871
|
-
};
|
|
872
|
-
var keyEncoder = {
|
|
873
|
-
encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
|
|
874
|
-
decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
|
|
875
|
-
format: "buffer"
|
|
876
|
-
};
|
|
877
|
-
var encodingOptions = {
|
|
878
|
-
keyEncoding: keyEncoder,
|
|
879
|
-
valueEncoding: "buffer"
|
|
880
|
-
};
|
|
881
|
-
var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
|
|
882
|
-
|
|
883
|
-
// packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
|
|
884
|
-
function _ts_decorate2(decorators, target, key, desc) {
|
|
885
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
886
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
887
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
888
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
889
|
-
}
|
|
890
|
-
var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
|
|
891
|
-
var AutomergeHost = class extends Resource4 {
|
|
892
|
-
constructor({ db, indexMetadataStore, dataMonitor }) {
|
|
893
|
-
super();
|
|
894
|
-
this._collectionSynchronizer = new CollectionSynchronizer({
|
|
895
|
-
queryCollectionState: this._queryCollectionState.bind(this),
|
|
896
|
-
sendCollectionState: this._sendCollectionState.bind(this),
|
|
897
|
-
shouldSyncCollection: this._shouldSyncCollection.bind(this)
|
|
898
|
-
});
|
|
899
|
-
this.collectionStateUpdated = new Event2();
|
|
900
|
-
this._db = db;
|
|
901
|
-
this._storage = new LevelDBStorageAdapter({
|
|
902
|
-
db: db.sublevel("automerge"),
|
|
903
|
-
callbacks: {
|
|
904
|
-
beforeSave: async (params) => this._beforeSave(params),
|
|
905
|
-
afterSave: async (key) => this._afterSave(key)
|
|
906
|
-
},
|
|
907
|
-
monitor: dataMonitor
|
|
908
|
-
});
|
|
909
|
-
this._echoNetworkAdapter = new EchoNetworkAdapter({
|
|
910
|
-
getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
|
|
911
|
-
isDocumentInRemoteCollection: this._isDocumentInRemoteCollection.bind(this),
|
|
912
|
-
onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
|
|
913
|
-
onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
|
|
914
|
-
monitor: dataMonitor
|
|
915
|
-
});
|
|
916
|
-
this._headsStore = new HeadsStore({
|
|
917
|
-
db: db.sublevel("heads")
|
|
918
|
-
});
|
|
919
|
-
this._indexMetadataStore = indexMetadataStore;
|
|
863
|
+
this._indexMetadataStore = indexMetadataStore;
|
|
920
864
|
}
|
|
921
865
|
async _open() {
|
|
922
866
|
this._peerId = `host-${PublicKey.random().toHex()}`;
|
|
@@ -934,9 +878,6 @@ var AutomergeHost = class extends Resource4 {
|
|
|
934
878
|
Event2.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => this._onPeerDisconnected(e.peerId));
|
|
935
879
|
this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId }) => {
|
|
936
880
|
this._onRemoteCollectionStateUpdated(collectionId, peerId);
|
|
937
|
-
this.collectionStateUpdated.emit({
|
|
938
|
-
collectionId
|
|
939
|
-
});
|
|
940
881
|
});
|
|
941
882
|
await this._echoNetworkAdapter.open();
|
|
942
883
|
await this._collectionSynchronizer.open();
|
|
@@ -1018,8 +959,8 @@ var AutomergeHost = class extends Resource4 {
|
|
|
1018
959
|
if (headsToWait.length > 0) {
|
|
1019
960
|
await Promise.all(headsToWait.map(async (entry, index) => {
|
|
1020
961
|
const handle = await this.loadDoc(Context.default(void 0, {
|
|
1021
|
-
F:
|
|
1022
|
-
L:
|
|
962
|
+
F: __dxlog_file3,
|
|
963
|
+
L: 227
|
|
1023
964
|
}), entry.documentId);
|
|
1024
965
|
await waitForHeads(handle, entry.heads);
|
|
1025
966
|
}));
|
|
@@ -1028,11 +969,11 @@ var AutomergeHost = class extends Resource4 {
|
|
|
1028
969
|
}
|
|
1029
970
|
async reIndexHeads(documentIds) {
|
|
1030
971
|
for (const documentId of documentIds) {
|
|
1031
|
-
|
|
972
|
+
log3.info("re-indexing heads for document", {
|
|
1032
973
|
documentId
|
|
1033
974
|
}, {
|
|
1034
|
-
F:
|
|
1035
|
-
L:
|
|
975
|
+
F: __dxlog_file3,
|
|
976
|
+
L: 239,
|
|
1036
977
|
S: this,
|
|
1037
978
|
C: (f, a) => f(...a)
|
|
1038
979
|
});
|
|
@@ -1044,11 +985,11 @@ var AutomergeHost = class extends Resource4 {
|
|
|
1044
985
|
if (handle.inState([
|
|
1045
986
|
"requesting"
|
|
1046
987
|
])) {
|
|
1047
|
-
|
|
988
|
+
log3.warn("document is not available locally, skipping", {
|
|
1048
989
|
documentId
|
|
1049
990
|
}, {
|
|
1050
|
-
F:
|
|
1051
|
-
L:
|
|
991
|
+
F: __dxlog_file3,
|
|
992
|
+
L: 243,
|
|
1052
993
|
S: this,
|
|
1053
994
|
C: (f, a) => f(...a)
|
|
1054
995
|
});
|
|
@@ -1056,8 +997,8 @@ var AutomergeHost = class extends Resource4 {
|
|
|
1056
997
|
}
|
|
1057
998
|
const doc = handle.docSync();
|
|
1058
999
|
invariant3(doc, void 0, {
|
|
1059
|
-
F:
|
|
1060
|
-
L:
|
|
1000
|
+
F: __dxlog_file3,
|
|
1001
|
+
L: 248,
|
|
1061
1002
|
S: this,
|
|
1062
1003
|
A: [
|
|
1063
1004
|
"doc",
|
|
@@ -1069,9 +1010,9 @@ var AutomergeHost = class extends Resource4 {
|
|
|
1069
1010
|
this._headsStore.setHeads(documentId, heads, batch);
|
|
1070
1011
|
await batch.write();
|
|
1071
1012
|
}
|
|
1072
|
-
|
|
1073
|
-
F:
|
|
1074
|
-
L:
|
|
1013
|
+
log3.info("done re-indexing heads", void 0, {
|
|
1014
|
+
F: __dxlog_file3,
|
|
1015
|
+
L: 255,
|
|
1075
1016
|
S: this,
|
|
1076
1017
|
C: (f, a) => f(...a)
|
|
1077
1018
|
});
|
|
@@ -1217,11 +1158,7 @@ var AutomergeHost = class extends Resource4 {
|
|
|
1217
1158
|
const diff = diffCollectionState(localState, state);
|
|
1218
1159
|
result.peers.push({
|
|
1219
1160
|
peerId,
|
|
1220
|
-
|
|
1221
|
-
missingOnLocal: diff.missingOnLocal.length,
|
|
1222
|
-
differentDocuments: diff.different.length,
|
|
1223
|
-
localDocumentCount: Object.keys(localState.documents).length,
|
|
1224
|
-
remoteDocumentCount: Object.keys(state.documents).length
|
|
1161
|
+
differentDocuments: diff.different.length
|
|
1225
1162
|
});
|
|
1226
1163
|
}
|
|
1227
1164
|
return result;
|
|
@@ -1263,43 +1200,31 @@ var AutomergeHost = class extends Resource4 {
|
|
|
1263
1200
|
if (!localState || !remoteState) {
|
|
1264
1201
|
return;
|
|
1265
1202
|
}
|
|
1266
|
-
const { different
|
|
1267
|
-
|
|
1268
|
-
...missingOnLocal,
|
|
1269
|
-
...missingOnRemote,
|
|
1270
|
-
...different
|
|
1271
|
-
];
|
|
1272
|
-
if (toReplicate.length === 0) {
|
|
1203
|
+
const { different } = diffCollectionState(localState, remoteState);
|
|
1204
|
+
if (different.length === 0) {
|
|
1273
1205
|
return;
|
|
1274
1206
|
}
|
|
1275
|
-
|
|
1276
|
-
count:
|
|
1207
|
+
log3.info("replication documents after collection sync", {
|
|
1208
|
+
count: different.length
|
|
1277
1209
|
}, {
|
|
1278
|
-
F:
|
|
1279
|
-
L:
|
|
1210
|
+
F: __dxlog_file3,
|
|
1211
|
+
L: 475,
|
|
1280
1212
|
S: this,
|
|
1281
1213
|
C: (f, a) => f(...a)
|
|
1282
1214
|
});
|
|
1283
|
-
for (const documentId of
|
|
1215
|
+
for (const documentId of different) {
|
|
1284
1216
|
this._repo.find(documentId);
|
|
1285
1217
|
}
|
|
1286
1218
|
}
|
|
1287
1219
|
_onHeadsChanged(documentId, heads) {
|
|
1288
|
-
const collectionsChanged = /* @__PURE__ */ new Set();
|
|
1289
1220
|
for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
|
|
1290
1221
|
const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
|
|
1291
1222
|
if (state?.documents[documentId]) {
|
|
1292
1223
|
const newState = structuredClone(state);
|
|
1293
1224
|
newState.documents[documentId] = heads;
|
|
1294
1225
|
this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
|
|
1295
|
-
collectionsChanged.add(collectionId);
|
|
1296
1226
|
}
|
|
1297
1227
|
}
|
|
1298
|
-
for (const collectionId of collectionsChanged) {
|
|
1299
|
-
this.collectionStateUpdated.emit({
|
|
1300
|
-
collectionId
|
|
1301
|
-
});
|
|
1302
|
-
}
|
|
1303
1228
|
}
|
|
1304
1229
|
};
|
|
1305
1230
|
_ts_decorate2([
|
|
@@ -1342,8 +1267,8 @@ var changeIsPresentInDoc = (doc, changeHash) => {
|
|
|
1342
1267
|
};
|
|
1343
1268
|
var decodeCollectionState = (state) => {
|
|
1344
1269
|
invariant3(typeof state === "object" && state !== null, "Invalid state", {
|
|
1345
|
-
F:
|
|
1346
|
-
L:
|
|
1270
|
+
F: __dxlog_file3,
|
|
1271
|
+
L: 528,
|
|
1347
1272
|
S: void 0,
|
|
1348
1273
|
A: [
|
|
1349
1274
|
"typeof state === 'object' && state !== null",
|
|
@@ -1356,18 +1281,37 @@ var encodeCollectionState = (state) => {
|
|
|
1356
1281
|
return state;
|
|
1357
1282
|
};
|
|
1358
1283
|
|
|
1284
|
+
// packages/core/echo/echo-pipeline/src/automerge/space-collection.ts
|
|
1285
|
+
import { invariant as invariant4 } from "@dxos/invariant";
|
|
1286
|
+
import { SpaceId } from "@dxos/keys";
|
|
1287
|
+
var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
|
|
1288
|
+
var deriveCollectionIdFromSpaceId = (spaceId) => `space:${spaceId}`;
|
|
1289
|
+
var getSpaceIdFromCollectionId = (collectionId) => {
|
|
1290
|
+
const spaceId = collectionId.replace(/^space:/, "");
|
|
1291
|
+
invariant4(SpaceId.isValid(spaceId), void 0, {
|
|
1292
|
+
F: __dxlog_file4,
|
|
1293
|
+
L: 12,
|
|
1294
|
+
S: void 0,
|
|
1295
|
+
A: [
|
|
1296
|
+
"SpaceId.isValid(spaceId)",
|
|
1297
|
+
""
|
|
1298
|
+
]
|
|
1299
|
+
});
|
|
1300
|
+
return spaceId;
|
|
1301
|
+
};
|
|
1302
|
+
|
|
1359
1303
|
// packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts
|
|
1360
1304
|
import { invariant as invariant6 } from "@dxos/invariant";
|
|
1361
1305
|
import { PublicKey as PublicKey2 } from "@dxos/keys";
|
|
1362
|
-
import { log as
|
|
1306
|
+
import { log as log5 } from "@dxos/log";
|
|
1363
1307
|
import { ComplexSet, defaultMap as defaultMap2 } from "@dxos/util";
|
|
1364
1308
|
|
|
1365
1309
|
// packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts
|
|
1366
1310
|
import * as A2 from "@dxos/automerge/automerge";
|
|
1367
1311
|
import { cbor } from "@dxos/automerge/automerge-repo";
|
|
1368
1312
|
import { Resource as Resource5 } from "@dxos/context";
|
|
1369
|
-
import { invariant as
|
|
1370
|
-
import { log as
|
|
1313
|
+
import { invariant as invariant5 } from "@dxos/invariant";
|
|
1314
|
+
import { log as log4 } from "@dxos/log";
|
|
1371
1315
|
import { AutomergeReplicator } from "@dxos/teleport-extension-automerge-replicator";
|
|
1372
1316
|
var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
|
|
1373
1317
|
var DEFAULT_FACTORY = (params) => new AutomergeReplicator(...params);
|
|
@@ -1387,7 +1331,7 @@ var MeshReplicatorConnection = class extends Resource5 {
|
|
|
1387
1331
|
});
|
|
1388
1332
|
this.writable = new WritableStream({
|
|
1389
1333
|
write: async (message, controller) => {
|
|
1390
|
-
|
|
1334
|
+
invariant5(this._isEnabled, "Writing to a disabled connection", {
|
|
1391
1335
|
F: __dxlog_file5,
|
|
1392
1336
|
L: 49,
|
|
1393
1337
|
S: this,
|
|
@@ -1416,7 +1360,7 @@ var MeshReplicatorConnection = class extends Resource5 {
|
|
|
1416
1360
|
onStartReplication: async (info, remotePeerId) => {
|
|
1417
1361
|
this.remoteDeviceKey = remotePeerId;
|
|
1418
1362
|
this._remotePeerId = info.id;
|
|
1419
|
-
|
|
1363
|
+
log4("onStartReplication", {
|
|
1420
1364
|
id: info.id,
|
|
1421
1365
|
thisPeerId: this.peerId,
|
|
1422
1366
|
remotePeerId: remotePeerId.toHex()
|
|
@@ -1447,7 +1391,7 @@ var MeshReplicatorConnection = class extends Resource5 {
|
|
|
1447
1391
|
}
|
|
1448
1392
|
}
|
|
1449
1393
|
get peerId() {
|
|
1450
|
-
|
|
1394
|
+
invariant5(this._remotePeerId != null, "Remote peer has not connected yet.", {
|
|
1451
1395
|
F: __dxlog_file5,
|
|
1452
1396
|
L: 110,
|
|
1453
1397
|
S: this,
|
|
@@ -1469,7 +1413,7 @@ var MeshReplicatorConnection = class extends Resource5 {
|
|
|
1469
1413
|
* Call after the remote peer has connected.
|
|
1470
1414
|
*/
|
|
1471
1415
|
enable() {
|
|
1472
|
-
|
|
1416
|
+
invariant5(this._remotePeerId != null, "Remote peer has not connected yet.", {
|
|
1473
1417
|
F: __dxlog_file5,
|
|
1474
1418
|
L: 127,
|
|
1475
1419
|
S: this,
|
|
@@ -1488,7 +1432,7 @@ var MeshReplicatorConnection = class extends Resource5 {
|
|
|
1488
1432
|
}
|
|
1489
1433
|
};
|
|
1490
1434
|
var logSendSync = (message) => {
|
|
1491
|
-
|
|
1435
|
+
log4("sendSyncMessage", () => {
|
|
1492
1436
|
const decodedSyncMessage = message.type === "sync" && message.data ? A2.decodeSyncMessage(message.data) : void 0;
|
|
1493
1437
|
return {
|
|
1494
1438
|
sync: decodedSyncMessage && {
|
|
@@ -1508,27 +1452,8 @@ var logSendSync = (message) => {
|
|
|
1508
1452
|
});
|
|
1509
1453
|
};
|
|
1510
1454
|
|
|
1511
|
-
// packages/core/echo/echo-pipeline/src/automerge/space-collection.ts
|
|
1512
|
-
import { invariant as invariant5 } from "@dxos/invariant";
|
|
1513
|
-
import { SpaceId } from "@dxos/keys";
|
|
1514
|
-
var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
|
|
1515
|
-
var deriveCollectionIdFromSpaceId = (spaceId) => `space:${spaceId}`;
|
|
1516
|
-
var getSpaceIdFromCollectionId = (collectionId) => {
|
|
1517
|
-
const spaceId = collectionId.replace(/^space:/, "");
|
|
1518
|
-
invariant5(SpaceId.isValid(spaceId), void 0, {
|
|
1519
|
-
F: __dxlog_file6,
|
|
1520
|
-
L: 13,
|
|
1521
|
-
S: void 0,
|
|
1522
|
-
A: [
|
|
1523
|
-
"SpaceId.isValid(spaceId)",
|
|
1524
|
-
""
|
|
1525
|
-
]
|
|
1526
|
-
});
|
|
1527
|
-
return spaceId;
|
|
1528
|
-
};
|
|
1529
|
-
|
|
1530
1455
|
// packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts
|
|
1531
|
-
var
|
|
1456
|
+
var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts";
|
|
1532
1457
|
var MeshEchoReplicator = class {
|
|
1533
1458
|
constructor() {
|
|
1534
1459
|
this._connections = /* @__PURE__ */ new Set();
|
|
@@ -1558,8 +1483,8 @@ var MeshEchoReplicator = class {
|
|
|
1558
1483
|
}
|
|
1559
1484
|
createExtension(extensionFactory) {
|
|
1560
1485
|
invariant6(this._context, void 0, {
|
|
1561
|
-
F:
|
|
1562
|
-
L:
|
|
1486
|
+
F: __dxlog_file6,
|
|
1487
|
+
L: 56,
|
|
1563
1488
|
S: this,
|
|
1564
1489
|
A: [
|
|
1565
1490
|
"this._context",
|
|
@@ -1570,17 +1495,17 @@ var MeshEchoReplicator = class {
|
|
|
1570
1495
|
ownPeerId: this._context.peerId,
|
|
1571
1496
|
replicatorFactory: extensionFactory,
|
|
1572
1497
|
onRemoteConnected: async () => {
|
|
1573
|
-
|
|
1498
|
+
log5("onRemoteConnected", {
|
|
1574
1499
|
peerId: connection.peerId
|
|
1575
1500
|
}, {
|
|
1576
|
-
F:
|
|
1577
|
-
L:
|
|
1501
|
+
F: __dxlog_file6,
|
|
1502
|
+
L: 62,
|
|
1578
1503
|
S: this,
|
|
1579
1504
|
C: (f, a) => f(...a)
|
|
1580
1505
|
});
|
|
1581
1506
|
invariant6(this._context, void 0, {
|
|
1582
|
-
F:
|
|
1583
|
-
L:
|
|
1507
|
+
F: __dxlog_file6,
|
|
1508
|
+
L: 63,
|
|
1584
1509
|
S: this,
|
|
1585
1510
|
A: [
|
|
1586
1511
|
"this._context",
|
|
@@ -1596,11 +1521,11 @@ var MeshEchoReplicator = class {
|
|
|
1596
1521
|
}
|
|
1597
1522
|
},
|
|
1598
1523
|
onRemoteDisconnected: async () => {
|
|
1599
|
-
|
|
1524
|
+
log5("onRemoteDisconnected", {
|
|
1600
1525
|
peerId: connection.peerId
|
|
1601
1526
|
}, {
|
|
1602
|
-
F:
|
|
1603
|
-
L:
|
|
1527
|
+
F: __dxlog_file6,
|
|
1528
|
+
L: 74,
|
|
1604
1529
|
S: this,
|
|
1605
1530
|
C: (f, a) => f(...a)
|
|
1606
1531
|
});
|
|
@@ -1610,18 +1535,18 @@ var MeshEchoReplicator = class {
|
|
|
1610
1535
|
this._connections.delete(connection);
|
|
1611
1536
|
},
|
|
1612
1537
|
shouldAdvertise: async (params) => {
|
|
1613
|
-
|
|
1538
|
+
log5("shouldAdvertise", {
|
|
1614
1539
|
peerId: connection.peerId,
|
|
1615
1540
|
documentId: params.documentId
|
|
1616
1541
|
}, {
|
|
1617
|
-
F:
|
|
1618
|
-
L:
|
|
1542
|
+
F: __dxlog_file6,
|
|
1543
|
+
L: 81,
|
|
1619
1544
|
S: this,
|
|
1620
1545
|
C: (f, a) => f(...a)
|
|
1621
1546
|
});
|
|
1622
1547
|
invariant6(this._context, void 0, {
|
|
1623
|
-
F:
|
|
1624
|
-
L:
|
|
1548
|
+
F: __dxlog_file6,
|
|
1549
|
+
L: 82,
|
|
1625
1550
|
S: this,
|
|
1626
1551
|
A: [
|
|
1627
1552
|
"this._context",
|
|
@@ -1635,13 +1560,13 @@ var MeshEchoReplicator = class {
|
|
|
1635
1560
|
documentId: params.documentId,
|
|
1636
1561
|
peerId: connection.peerId
|
|
1637
1562
|
});
|
|
1638
|
-
|
|
1563
|
+
log5("document not found locally for share policy check, accepting the remote document", {
|
|
1639
1564
|
peerId: connection.peerId,
|
|
1640
1565
|
documentId: params.documentId,
|
|
1641
1566
|
remoteDocumentExists
|
|
1642
1567
|
}, {
|
|
1643
|
-
F:
|
|
1644
|
-
L:
|
|
1568
|
+
F: __dxlog_file6,
|
|
1569
|
+
L: 90,
|
|
1645
1570
|
S: this,
|
|
1646
1571
|
C: (f, a) => f(...a)
|
|
1647
1572
|
});
|
|
@@ -1650,19 +1575,19 @@ var MeshEchoReplicator = class {
|
|
|
1650
1575
|
const spaceId = await createIdFromSpaceKey(spaceKey);
|
|
1651
1576
|
const authorizedDevices = this._authorizedDevices.get(spaceId);
|
|
1652
1577
|
if (!connection.remoteDeviceKey) {
|
|
1653
|
-
|
|
1578
|
+
log5("device key not found for share policy check", {
|
|
1654
1579
|
peerId: connection.peerId,
|
|
1655
1580
|
documentId: params.documentId
|
|
1656
1581
|
}, {
|
|
1657
|
-
F:
|
|
1658
|
-
L:
|
|
1582
|
+
F: __dxlog_file6,
|
|
1583
|
+
L: 106,
|
|
1659
1584
|
S: this,
|
|
1660
1585
|
C: (f, a) => f(...a)
|
|
1661
1586
|
});
|
|
1662
1587
|
return false;
|
|
1663
1588
|
}
|
|
1664
1589
|
const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
|
|
1665
|
-
|
|
1590
|
+
log5("share policy check", {
|
|
1666
1591
|
localPeer: this._context.peerId,
|
|
1667
1592
|
remotePeer: connection.peerId,
|
|
1668
1593
|
documentId: params.documentId,
|
|
@@ -1670,16 +1595,16 @@ var MeshEchoReplicator = class {
|
|
|
1670
1595
|
spaceKey,
|
|
1671
1596
|
isAuthorized
|
|
1672
1597
|
}, {
|
|
1673
|
-
F:
|
|
1674
|
-
L:
|
|
1598
|
+
F: __dxlog_file6,
|
|
1599
|
+
L: 114,
|
|
1675
1600
|
S: this,
|
|
1676
1601
|
C: (f, a) => f(...a)
|
|
1677
1602
|
});
|
|
1678
1603
|
return isAuthorized;
|
|
1679
1604
|
} catch (err) {
|
|
1680
|
-
|
|
1681
|
-
F:
|
|
1682
|
-
L:
|
|
1605
|
+
log5.catch(err, void 0, {
|
|
1606
|
+
F: __dxlog_file6,
|
|
1607
|
+
L: 124,
|
|
1683
1608
|
S: this,
|
|
1684
1609
|
C: (f, a) => f(...a)
|
|
1685
1610
|
});
|
|
@@ -1690,12 +1615,12 @@ var MeshEchoReplicator = class {
|
|
|
1690
1615
|
const spaceId = getSpaceIdFromCollectionId(collectionId);
|
|
1691
1616
|
const authorizedDevices = this._authorizedDevices.get(spaceId);
|
|
1692
1617
|
if (!connection.remoteDeviceKey) {
|
|
1693
|
-
|
|
1618
|
+
log5("device key not found for collection sync check", {
|
|
1694
1619
|
peerId: connection.peerId,
|
|
1695
1620
|
collectionId
|
|
1696
1621
|
}, {
|
|
1697
|
-
F:
|
|
1698
|
-
L:
|
|
1622
|
+
F: __dxlog_file6,
|
|
1623
|
+
L: 134,
|
|
1699
1624
|
S: this,
|
|
1700
1625
|
C: (f, a) => f(...a)
|
|
1701
1626
|
});
|
|
@@ -1709,12 +1634,12 @@ var MeshEchoReplicator = class {
|
|
|
1709
1634
|
return connection.replicatorExtension;
|
|
1710
1635
|
}
|
|
1711
1636
|
async authorizeDevice(spaceKey, deviceKey) {
|
|
1712
|
-
|
|
1637
|
+
log5("authorizeDevice", {
|
|
1713
1638
|
spaceKey,
|
|
1714
1639
|
deviceKey
|
|
1715
1640
|
}, {
|
|
1716
|
-
F:
|
|
1717
|
-
L:
|
|
1641
|
+
F: __dxlog_file6,
|
|
1642
|
+
L: 151,
|
|
1718
1643
|
S: this,
|
|
1719
1644
|
C: (f, a) => f(...a)
|
|
1720
1645
|
});
|
|
@@ -2092,7 +2017,11 @@ var getByteCount = (message) => {
|
|
|
2092
2017
|
};
|
|
2093
2018
|
|
|
2094
2019
|
// packages/core/echo/echo-pipeline/src/db-host/data-service.ts
|
|
2095
|
-
|
|
2020
|
+
import { Stream } from "@dxos/codec-protobuf";
|
|
2021
|
+
import { invariant as invariant7 } from "@dxos/invariant";
|
|
2022
|
+
import { SpaceId as SpaceId2 } from "@dxos/keys";
|
|
2023
|
+
import { log as log6 } from "@dxos/log";
|
|
2024
|
+
var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
|
|
2096
2025
|
var DataServiceImpl = class {
|
|
2097
2026
|
constructor(params) {
|
|
2098
2027
|
/**
|
|
@@ -2112,9 +2041,9 @@ var DataServiceImpl = class {
|
|
|
2112
2041
|
synchronizer.open().then(() => {
|
|
2113
2042
|
this._subscriptions.set(request.subscriptionId, synchronizer);
|
|
2114
2043
|
ready();
|
|
2115
|
-
}).catch((err) =>
|
|
2116
|
-
F:
|
|
2117
|
-
L:
|
|
2044
|
+
}).catch((err) => log6.catch(err, void 0, {
|
|
2045
|
+
F: __dxlog_file7,
|
|
2046
|
+
L: 64,
|
|
2118
2047
|
S: this,
|
|
2119
2048
|
C: (f, a) => f(...a)
|
|
2120
2049
|
}));
|
|
@@ -2124,8 +2053,8 @@ var DataServiceImpl = class {
|
|
|
2124
2053
|
async updateSubscription(request) {
|
|
2125
2054
|
const synchronizer = this._subscriptions.get(request.subscriptionId);
|
|
2126
2055
|
invariant7(synchronizer, "Subscription not found", {
|
|
2127
|
-
F:
|
|
2128
|
-
L:
|
|
2056
|
+
F: __dxlog_file7,
|
|
2057
|
+
L: 71,
|
|
2129
2058
|
S: this,
|
|
2130
2059
|
A: [
|
|
2131
2060
|
"synchronizer",
|
|
@@ -2145,8 +2074,8 @@ var DataServiceImpl = class {
|
|
|
2145
2074
|
}
|
|
2146
2075
|
const synchronizer = this._subscriptions.get(request.subscriptionId);
|
|
2147
2076
|
invariant7(synchronizer, "Subscription not found", {
|
|
2148
|
-
F:
|
|
2149
|
-
L:
|
|
2077
|
+
F: __dxlog_file7,
|
|
2078
|
+
L: 86,
|
|
2150
2079
|
S: this,
|
|
2151
2080
|
A: [
|
|
2152
2081
|
"synchronizer",
|
|
@@ -2186,1272 +2115,38 @@ var DataServiceImpl = class {
|
|
|
2186
2115
|
async updateIndexes() {
|
|
2187
2116
|
await this._updateIndexes();
|
|
2188
2117
|
}
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
]
|
|
2199
|
-
});
|
|
2200
|
-
const collectionId = deriveCollectionIdFromSpaceId(request.spaceId);
|
|
2201
|
-
const scheduler = new UpdateScheduler2(ctx, async () => {
|
|
2202
|
-
const state = await this._automergeHost.getCollectionSyncState(collectionId);
|
|
2203
|
-
next({
|
|
2204
|
-
peers: state.peers.map((peer) => ({
|
|
2205
|
-
peerId: peer.peerId,
|
|
2206
|
-
missingOnRemote: peer.missingOnRemote,
|
|
2207
|
-
missingOnLocal: peer.missingOnLocal,
|
|
2208
|
-
differentDocuments: peer.differentDocuments,
|
|
2209
|
-
localDocumentCount: peer.localDocumentCount,
|
|
2210
|
-
remoteDocumentCount: peer.remoteDocumentCount
|
|
2211
|
-
}))
|
|
2212
|
-
});
|
|
2213
|
-
});
|
|
2214
|
-
this._automergeHost.collectionStateUpdated.on(ctx, (e) => {
|
|
2215
|
-
if (e.collectionId === collectionId) {
|
|
2216
|
-
scheduler.trigger();
|
|
2217
|
-
}
|
|
2218
|
-
});
|
|
2219
|
-
scheduler.trigger();
|
|
2118
|
+
async getSpaceSyncState(request, options) {
|
|
2119
|
+
invariant7(SpaceId2.isValid(request.spaceId), void 0, {
|
|
2120
|
+
F: __dxlog_file7,
|
|
2121
|
+
L: 127,
|
|
2122
|
+
S: this,
|
|
2123
|
+
A: [
|
|
2124
|
+
"SpaceId.isValid(request.spaceId)",
|
|
2125
|
+
""
|
|
2126
|
+
]
|
|
2220
2127
|
});
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
// packages/core/echo/echo-pipeline/src/db-host/echo-host.ts
|
|
2225
|
-
import { LifecycleState as LifecycleState4, Resource as Resource9 } from "@dxos/context";
|
|
2226
|
-
import { todo } from "@dxos/debug";
|
|
2227
|
-
import { createIdFromSpaceKey as createIdFromSpaceKey3, SpaceDocVersion as SpaceDocVersion3 } from "@dxos/echo-protocol";
|
|
2228
|
-
import { IndexMetadataStore, IndexStore, Indexer } from "@dxos/indexing";
|
|
2229
|
-
import { invariant as invariant11 } from "@dxos/invariant";
|
|
2230
|
-
import { IndexKind } from "@dxos/protocols/proto/dxos/echo/indexing";
|
|
2231
|
-
import { trace as trace5 } from "@dxos/tracing";
|
|
2232
|
-
|
|
2233
|
-
// packages/core/echo/echo-pipeline/src/db-host/documents-iterator.ts
|
|
2234
|
-
import * as A3 from "@dxos/automerge/automerge";
|
|
2235
|
-
import { Context as Context2 } from "@dxos/context";
|
|
2236
|
-
import { SpaceDocVersion } from "@dxos/echo-protocol";
|
|
2237
|
-
import { invariant as invariant8 } from "@dxos/invariant";
|
|
2238
|
-
import { log as log8 } from "@dxos/log";
|
|
2239
|
-
import { ObjectPointerVersion, objectPointerCodec as objectPointerCodec2 } from "@dxos/protocols";
|
|
2240
|
-
var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/documents-iterator.ts";
|
|
2241
|
-
var LOG_VIEW_OPERATION_THRESHOLD = 300;
|
|
2242
|
-
var createSelectedDocumentsIterator = (automergeHost) => (
|
|
2243
|
-
/**
|
|
2244
|
-
* Get object data blobs from Automerge Repo by ids.
|
|
2245
|
-
*/
|
|
2246
|
-
// TODO(mykola): Unload automerge handles after usage.
|
|
2247
|
-
async function* loadDocuments(objects) {
|
|
2248
|
-
for (const [id, heads] of objects.entries()) {
|
|
2249
|
-
try {
|
|
2250
|
-
const { documentId, objectId } = objectPointerCodec2.decode(id);
|
|
2251
|
-
const handle = await automergeHost.loadDoc(Context2.default(void 0, {
|
|
2252
|
-
F: __dxlog_file9,
|
|
2253
|
-
L: 30
|
|
2254
|
-
}), documentId);
|
|
2255
|
-
let doc = handle.docSync();
|
|
2256
|
-
invariant8(doc, void 0, {
|
|
2257
|
-
F: __dxlog_file9,
|
|
2258
|
-
L: 33,
|
|
2259
|
-
S: this,
|
|
2260
|
-
A: [
|
|
2261
|
-
"doc",
|
|
2262
|
-
""
|
|
2263
|
-
]
|
|
2264
|
-
});
|
|
2265
|
-
const currentHeads = A3.getHeads(doc);
|
|
2266
|
-
if (!A3.equals(currentHeads, heads)) {
|
|
2267
|
-
const begin = Date.now();
|
|
2268
|
-
doc = A3.view(doc, heads);
|
|
2269
|
-
const end = Date.now();
|
|
2270
|
-
if (end - begin > LOG_VIEW_OPERATION_THRESHOLD) {
|
|
2271
|
-
log8.info("Checking out document version is taking too long", {
|
|
2272
|
-
duration: end - begin,
|
|
2273
|
-
requestedHeads: heads,
|
|
2274
|
-
originalHeads: currentHeads
|
|
2275
|
-
}, {
|
|
2276
|
-
F: __dxlog_file9,
|
|
2277
|
-
L: 44,
|
|
2278
|
-
S: this,
|
|
2279
|
-
C: (f, a) => f(...a)
|
|
2280
|
-
});
|
|
2281
|
-
}
|
|
2282
|
-
}
|
|
2283
|
-
if (doc.version !== SpaceDocVersion.CURRENT) {
|
|
2284
|
-
continue;
|
|
2285
|
-
}
|
|
2286
|
-
if (!doc.objects?.[objectId]) {
|
|
2287
|
-
continue;
|
|
2288
|
-
}
|
|
2289
|
-
let newId = id;
|
|
2290
|
-
if (objectPointerCodec2.getVersion(id) === ObjectPointerVersion.V0) {
|
|
2291
|
-
const spaceKey = getSpaceKeyFromDoc(doc) ?? void 0;
|
|
2292
|
-
newId = objectPointerCodec2.encode({
|
|
2293
|
-
documentId,
|
|
2294
|
-
objectId,
|
|
2295
|
-
spaceKey
|
|
2296
|
-
});
|
|
2297
|
-
}
|
|
2298
|
-
yield [
|
|
2299
|
-
{
|
|
2300
|
-
id: newId,
|
|
2301
|
-
object: doc.objects[objectId],
|
|
2302
|
-
heads
|
|
2303
|
-
}
|
|
2304
|
-
];
|
|
2305
|
-
} catch (error) {
|
|
2306
|
-
log8.error("Error loading document", {
|
|
2307
|
-
heads,
|
|
2308
|
-
id,
|
|
2309
|
-
error
|
|
2310
|
-
}, {
|
|
2311
|
-
F: __dxlog_file9,
|
|
2312
|
-
L: 70,
|
|
2313
|
-
S: this,
|
|
2314
|
-
C: (f, a) => f(...a)
|
|
2315
|
-
});
|
|
2316
|
-
}
|
|
2317
|
-
}
|
|
2318
|
-
}
|
|
2319
|
-
);
|
|
2320
|
-
|
|
2321
|
-
// packages/core/echo/echo-pipeline/src/db-host/query-service.ts
|
|
2322
|
-
import { DeferredTask } from "@dxos/async";
|
|
2323
|
-
import { getHeads as getHeads3 } from "@dxos/automerge/automerge";
|
|
2324
|
-
import { Stream as Stream2 } from "@dxos/codec-protobuf";
|
|
2325
|
-
import { Context as Context4, Resource as Resource7 } from "@dxos/context";
|
|
2326
|
-
import { log as log9 } from "@dxos/log";
|
|
2327
|
-
import { objectPointerCodec as objectPointerCodec4 } from "@dxos/protocols";
|
|
2328
|
-
import { trace as trace4 } from "@dxos/tracing";
|
|
2329
|
-
|
|
2330
|
-
// packages/core/echo/echo-pipeline/src/db-host/query-state.ts
|
|
2331
|
-
import { Context as Context3, LifecycleState as LifecycleState3, Resource as Resource6 } from "@dxos/context";
|
|
2332
|
-
import { createIdFromSpaceKey as createIdFromSpaceKey2 } from "@dxos/echo-protocol";
|
|
2333
|
-
import { invariant as invariant9 } from "@dxos/invariant";
|
|
2334
|
-
import { PublicKey as PublicKey3 } from "@dxos/keys";
|
|
2335
|
-
import { objectPointerCodec as objectPointerCodec3 } from "@dxos/protocols";
|
|
2336
|
-
import { trace as trace3 } from "@dxos/tracing";
|
|
2337
|
-
import { nonNullable as nonNullable2 } from "@dxos/util";
|
|
2338
|
-
function _ts_decorate4(decorators, target, key, desc) {
|
|
2339
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
2340
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
2341
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
2342
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
2343
|
-
}
|
|
2344
|
-
var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/query-state.ts";
|
|
2345
|
-
var QueryState = class extends Resource6 {
|
|
2346
|
-
get active() {
|
|
2347
|
-
return this._lifecycleState === LifecycleState3.OPEN;
|
|
2348
|
-
}
|
|
2349
|
-
constructor(_params) {
|
|
2350
|
-
super();
|
|
2351
|
-
this._params = _params;
|
|
2352
|
-
this._results = [];
|
|
2353
|
-
this._firstRun = true;
|
|
2354
|
-
this.metrics = {
|
|
2355
|
-
objectsReturned: 0,
|
|
2356
|
-
objectsReturnedFromIndex: 0,
|
|
2357
|
-
documentsLoaded: 0,
|
|
2358
|
-
executionTime: 0,
|
|
2359
|
-
indexQueryTime: 0,
|
|
2360
|
-
documentLoadTime: 0
|
|
2361
|
-
};
|
|
2362
|
-
this.filter = _params.request.filter;
|
|
2363
|
-
}
|
|
2364
|
-
getResults() {
|
|
2365
|
-
return this._results;
|
|
2366
|
-
}
|
|
2367
|
-
// https://github.com/open-telemetry/semantic-conventions/blob/main/docs/attributes-registry/db.md#generic-database-attributes
|
|
2368
|
-
async execQuery() {
|
|
2369
|
-
const filter = this._params.request.filter;
|
|
2370
|
-
const beginQuery = performance.now();
|
|
2371
|
-
const hits = filter.objectIds && filter.objectIds?.length > 0 ? [] : await this._params.indexer.execQuery(filterToIndexQuery(filter));
|
|
2372
|
-
if (this._firstRun) {
|
|
2373
|
-
this.metrics.indexQueryTime = performance.now() - beginQuery;
|
|
2374
|
-
}
|
|
2375
|
-
const beginFilter = performance.now();
|
|
2376
|
-
const results = (await Promise.all(hits.map(async (result) => {
|
|
2377
|
-
if (this._firstRun) {
|
|
2378
|
-
this.metrics.objectsReturnedFromIndex++;
|
|
2379
|
-
}
|
|
2380
|
-
const { objectId, documentId, spaceKey: spaceKeyInIndex } = objectPointerCodec3.decode(result.id);
|
|
2381
|
-
let spaceKey;
|
|
2382
|
-
if (spaceKeyInIndex !== void 0) {
|
|
2383
|
-
spaceKey = spaceKeyInIndex;
|
|
2384
|
-
} else {
|
|
2385
|
-
if (this._firstRun) {
|
|
2386
|
-
this.metrics.documentsLoaded++;
|
|
2387
|
-
}
|
|
2388
|
-
const handle = await this._params.automergeHost.loadDoc(Context3.default(void 0, {
|
|
2389
|
-
F: __dxlog_file10,
|
|
2390
|
-
L: 116
|
|
2391
|
-
}), documentId);
|
|
2392
|
-
if (this._ctx.disposed) {
|
|
2393
|
-
return;
|
|
2394
|
-
}
|
|
2395
|
-
spaceKey = getSpaceKeyFromDoc(handle.docSync());
|
|
2396
|
-
}
|
|
2397
|
-
if (!spaceKey) {
|
|
2398
|
-
return;
|
|
2399
|
-
}
|
|
2400
|
-
if (this._params.request.filter.options?.spaces?.length && !this._params.request.filter.options.spaces.some((key) => key.equals(spaceKey))) {
|
|
2401
|
-
return;
|
|
2402
|
-
}
|
|
2403
|
-
if (this._firstRun) {
|
|
2404
|
-
this.metrics.objectsReturned++;
|
|
2405
|
-
}
|
|
2406
|
-
return {
|
|
2407
|
-
id: objectId,
|
|
2408
|
-
documentId,
|
|
2409
|
-
spaceId: await createIdFromSpaceKey2(PublicKey3.from(spaceKey)),
|
|
2410
|
-
spaceKey: PublicKey3.from(spaceKey),
|
|
2411
|
-
rank: result.rank
|
|
2412
|
-
};
|
|
2413
|
-
}))).filter(nonNullable2);
|
|
2414
|
-
if (this._firstRun) {
|
|
2415
|
-
this.metrics.documentLoadTime = performance.now() - beginFilter;
|
|
2416
|
-
}
|
|
2417
|
-
if (this._ctx.disposed) {
|
|
2418
|
-
return {
|
|
2419
|
-
changed: false
|
|
2420
|
-
};
|
|
2421
|
-
}
|
|
2422
|
-
const areResultsUnchanged = !this._firstRun && this._results.length === results.length && this._results.every((oldResult) => results.some((result) => result.id === oldResult.id)) && results.every((result) => this._results.some((oldResult) => oldResult.id === result.id));
|
|
2423
|
-
if (this._firstRun) {
|
|
2424
|
-
this.metrics.executionTime = performance.now() - beginQuery;
|
|
2425
|
-
}
|
|
2426
|
-
this._firstRun = false;
|
|
2427
|
-
if (areResultsUnchanged) {
|
|
2428
|
-
return {
|
|
2429
|
-
changed: false
|
|
2430
|
-
};
|
|
2431
|
-
}
|
|
2432
|
-
this._results = results;
|
|
2433
|
-
return {
|
|
2434
|
-
changed: true
|
|
2435
|
-
};
|
|
2436
|
-
}
|
|
2437
|
-
};
|
|
2438
|
-
_ts_decorate4([
|
|
2439
|
-
trace3.info({
|
|
2440
|
-
depth: null
|
|
2441
|
-
})
|
|
2442
|
-
], QueryState.prototype, "filter", void 0);
|
|
2443
|
-
_ts_decorate4([
|
|
2444
|
-
trace3.info()
|
|
2445
|
-
], QueryState.prototype, "metrics", void 0);
|
|
2446
|
-
_ts_decorate4([
|
|
2447
|
-
trace3.info()
|
|
2448
|
-
], QueryState.prototype, "active", null);
|
|
2449
|
-
_ts_decorate4([
|
|
2450
|
-
trace3.span({
|
|
2451
|
-
showInBrowserTimeline: true,
|
|
2452
|
-
op: "db.query",
|
|
2453
|
-
attributes: {
|
|
2454
|
-
"db.system": "echo"
|
|
2455
|
-
}
|
|
2456
|
-
})
|
|
2457
|
-
], QueryState.prototype, "execQuery", null);
|
|
2458
|
-
QueryState = _ts_decorate4([
|
|
2459
|
-
trace3.resource()
|
|
2460
|
-
], QueryState);
|
|
2461
|
-
var filterToIndexQuery = (filter) => {
|
|
2462
|
-
invariant9(!(filter.type && (filter.or ?? []).length > 0), "Cannot mix type and or filters.", {
|
|
2463
|
-
F: __dxlog_file10,
|
|
2464
|
-
L: 181,
|
|
2465
|
-
S: void 0,
|
|
2466
|
-
A: [
|
|
2467
|
-
"!(filter.type && (filter.or ?? []).length > 0)",
|
|
2468
|
-
"'Cannot mix type and or filters.'"
|
|
2469
|
-
]
|
|
2470
|
-
});
|
|
2471
|
-
invariant9((filter.or ?? []).every((subFilter) => !(subFilter.type && (subFilter.or ?? []).length > 0)), "Cannot mix type and or filters.", {
|
|
2472
|
-
F: __dxlog_file10,
|
|
2473
|
-
L: 182,
|
|
2474
|
-
S: void 0,
|
|
2475
|
-
A: [
|
|
2476
|
-
"(filter.or ?? []).every((subFilter) => !(subFilter.type && (subFilter.or ?? []).length > 0))",
|
|
2477
|
-
"'Cannot mix type and or filters.'"
|
|
2478
|
-
]
|
|
2479
|
-
});
|
|
2480
|
-
if (filter.type || (filter.or ?? []).length > 0 && (filter.or ?? []).every((subFilter) => !subFilter.not && subFilter.type)) {
|
|
2481
|
-
return {
|
|
2482
|
-
typenames: filter.type?.objectId ? [
|
|
2483
|
-
filter.type.objectId
|
|
2484
|
-
] : (filter.or ?? []).map((f) => f.type?.objectId).filter(nonNullable2),
|
|
2485
|
-
inverted: filter.not
|
|
2486
|
-
};
|
|
2487
|
-
} else {
|
|
2128
|
+
const collectionId = deriveCollectionIdFromSpaceId(request.spaceId);
|
|
2129
|
+
const state = await this._automergeHost.getCollectionSyncState(collectionId);
|
|
2488
2130
|
return {
|
|
2489
|
-
|
|
2131
|
+
peers: state.peers.map((peer) => ({
|
|
2132
|
+
peerId: peer.peerId,
|
|
2133
|
+
documentsToReconcile: peer.differentDocuments
|
|
2134
|
+
}))
|
|
2490
2135
|
};
|
|
2491
2136
|
}
|
|
2492
2137
|
};
|
|
2493
2138
|
|
|
2494
|
-
// packages/core/echo/echo-pipeline/src/db-host/query-service.ts
|
|
2495
|
-
var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/query-service.ts";
|
|
2496
|
-
var QueryServiceImpl = class extends Resource7 {
|
|
2497
|
-
// TODO(burdon): OK for options, but not params. Pass separately and type readonly here.
|
|
2498
|
-
constructor(_params) {
|
|
2499
|
-
super();
|
|
2500
|
-
this._params = _params;
|
|
2501
|
-
this._queries = /* @__PURE__ */ new Set();
|
|
2502
|
-
this._updateQueries = new DeferredTask(this._ctx, async () => {
|
|
2503
|
-
await Promise.all(Array.from(this._queries).map(async (query) => {
|
|
2504
|
-
try {
|
|
2505
|
-
const { changed } = await query.state.execQuery();
|
|
2506
|
-
if (changed) {
|
|
2507
|
-
query.sendResults(query.state.getResults());
|
|
2508
|
-
}
|
|
2509
|
-
} catch (err) {
|
|
2510
|
-
log9.catch(err, void 0, {
|
|
2511
|
-
F: __dxlog_file11,
|
|
2512
|
-
L: 52,
|
|
2513
|
-
S: this,
|
|
2514
|
-
C: (f, a) => f(...a)
|
|
2515
|
-
});
|
|
2516
|
-
}
|
|
2517
|
-
}));
|
|
2518
|
-
});
|
|
2519
|
-
trace4.diagnostic({
|
|
2520
|
-
id: "active-queries",
|
|
2521
|
-
name: "Active Queries",
|
|
2522
|
-
fetch: () => {
|
|
2523
|
-
return Array.from(this._queries).map((query) => {
|
|
2524
|
-
return {
|
|
2525
|
-
filter: JSON.stringify(query.state.filter),
|
|
2526
|
-
metrics: query.state.metrics
|
|
2527
|
-
};
|
|
2528
|
-
});
|
|
2529
|
-
}
|
|
2530
|
-
});
|
|
2531
|
-
}
|
|
2532
|
-
async _open() {
|
|
2533
|
-
this._params.indexer.updated.on(this._ctx, () => this._updateQueries.schedule());
|
|
2534
|
-
}
|
|
2535
|
-
async _close() {
|
|
2536
|
-
await Promise.all(Array.from(this._queries).map((query) => query.close()));
|
|
2537
|
-
}
|
|
2538
|
-
async setConfig(config) {
|
|
2539
|
-
if (this._params.indexer.initialized) {
|
|
2540
|
-
log9.warn("Indexer already initialized.", void 0, {
|
|
2541
|
-
F: __dxlog_file11,
|
|
2542
|
-
L: 86,
|
|
2543
|
-
S: this,
|
|
2544
|
-
C: (f, a) => f(...a)
|
|
2545
|
-
});
|
|
2546
|
-
return;
|
|
2547
|
-
}
|
|
2548
|
-
this._params.indexer.setConfig(config);
|
|
2549
|
-
}
|
|
2550
|
-
execQuery(request) {
|
|
2551
|
-
return new Stream2(({ next, close, ctx }) => {
|
|
2552
|
-
const query = {
|
|
2553
|
-
state: new QueryState({
|
|
2554
|
-
indexer: this._params.indexer,
|
|
2555
|
-
automergeHost: this._params.automergeHost,
|
|
2556
|
-
request
|
|
2557
|
-
}),
|
|
2558
|
-
sendResults: (results) => {
|
|
2559
|
-
if (ctx.disposed) {
|
|
2560
|
-
return;
|
|
2561
|
-
}
|
|
2562
|
-
next({
|
|
2563
|
-
queryId: request.queryId,
|
|
2564
|
-
results
|
|
2565
|
-
});
|
|
2566
|
-
},
|
|
2567
|
-
close: async () => {
|
|
2568
|
-
close();
|
|
2569
|
-
await query.state.close();
|
|
2570
|
-
this._queries.delete(query);
|
|
2571
|
-
}
|
|
2572
|
-
};
|
|
2573
|
-
this._queries.add(query);
|
|
2574
|
-
queueMicrotask(async () => {
|
|
2575
|
-
await query.state.open();
|
|
2576
|
-
try {
|
|
2577
|
-
const { changed } = await query.state.execQuery();
|
|
2578
|
-
if (changed) {
|
|
2579
|
-
query.sendResults(query.state.getResults());
|
|
2580
|
-
}
|
|
2581
|
-
} catch (error) {
|
|
2582
|
-
log9.catch(error, void 0, {
|
|
2583
|
-
F: __dxlog_file11,
|
|
2584
|
-
L: 123,
|
|
2585
|
-
S: this,
|
|
2586
|
-
C: (f, a) => f(...a)
|
|
2587
|
-
});
|
|
2588
|
-
}
|
|
2589
|
-
});
|
|
2590
|
-
return query.close;
|
|
2591
|
-
});
|
|
2592
|
-
}
|
|
2593
|
-
/**
|
|
2594
|
-
* Re-index all loaded documents.
|
|
2595
|
-
*/
|
|
2596
|
-
async reindex() {
|
|
2597
|
-
log9.info("Reindexing all documents...", void 0, {
|
|
2598
|
-
F: __dxlog_file11,
|
|
2599
|
-
L: 135,
|
|
2600
|
-
S: this,
|
|
2601
|
-
C: (f, a) => f(...a)
|
|
2602
|
-
});
|
|
2603
|
-
const iterator = createDocumentsIterator(this._params.automergeHost);
|
|
2604
|
-
const ids = /* @__PURE__ */ new Map();
|
|
2605
|
-
for await (const documents of iterator()) {
|
|
2606
|
-
for (const { id, heads } of documents) {
|
|
2607
|
-
ids.set(id, heads);
|
|
2608
|
-
}
|
|
2609
|
-
if (ids.size % 100 === 0) {
|
|
2610
|
-
log9.info("Collected documents...", {
|
|
2611
|
-
count: ids.size
|
|
2612
|
-
}, {
|
|
2613
|
-
F: __dxlog_file11,
|
|
2614
|
-
L: 143,
|
|
2615
|
-
S: this,
|
|
2616
|
-
C: (f, a) => f(...a)
|
|
2617
|
-
});
|
|
2618
|
-
}
|
|
2619
|
-
}
|
|
2620
|
-
log9.info("Marking all documents as dirty...", {
|
|
2621
|
-
count: ids.size
|
|
2622
|
-
}, {
|
|
2623
|
-
F: __dxlog_file11,
|
|
2624
|
-
L: 147,
|
|
2625
|
-
S: this,
|
|
2626
|
-
C: (f, a) => f(...a)
|
|
2627
|
-
});
|
|
2628
|
-
await this._params.indexer.reindex(ids);
|
|
2629
|
-
}
|
|
2630
|
-
};
|
|
2631
|
-
var createDocumentsIterator = (automergeHost) => (
|
|
2632
|
-
/**
|
|
2633
|
-
* Recursively get all object data blobs from loaded documents from Automerge Repo.
|
|
2634
|
-
*/
|
|
2635
|
-
// TODO(mykola): Unload automerge handles after usage.
|
|
2636
|
-
async function* getAllDocuments() {
|
|
2637
|
-
const visited = /* @__PURE__ */ new Set();
|
|
2638
|
-
async function* getObjectsFromHandle(handle) {
|
|
2639
|
-
if (visited.has(handle.documentId)) {
|
|
2640
|
-
return;
|
|
2641
|
-
}
|
|
2642
|
-
const doc = handle.docSync();
|
|
2643
|
-
const spaceKey = getSpaceKeyFromDoc(doc) ?? void 0;
|
|
2644
|
-
if (doc.objects) {
|
|
2645
|
-
yield Object.entries(doc.objects).map(([objectId, object]) => {
|
|
2646
|
-
return {
|
|
2647
|
-
id: objectPointerCodec4.encode({
|
|
2648
|
-
documentId: handle.documentId,
|
|
2649
|
-
objectId,
|
|
2650
|
-
spaceKey
|
|
2651
|
-
}),
|
|
2652
|
-
object,
|
|
2653
|
-
heads: getHeads3(doc)
|
|
2654
|
-
};
|
|
2655
|
-
});
|
|
2656
|
-
}
|
|
2657
|
-
if (doc.links) {
|
|
2658
|
-
for (const id of Object.values(doc.links)) {
|
|
2659
|
-
if (visited.has(id)) {
|
|
2660
|
-
continue;
|
|
2661
|
-
}
|
|
2662
|
-
const linkHandle = await automergeHost.loadDoc(Context4.default(void 0, {
|
|
2663
|
-
F: __dxlog_file11,
|
|
2664
|
-
L: 188
|
|
2665
|
-
}), id);
|
|
2666
|
-
for await (const result of getObjectsFromHandle(linkHandle)) {
|
|
2667
|
-
yield result;
|
|
2668
|
-
}
|
|
2669
|
-
}
|
|
2670
|
-
}
|
|
2671
|
-
visited.add(handle.documentId);
|
|
2672
|
-
}
|
|
2673
|
-
for (const handle of Object.values(automergeHost.repo.handles)) {
|
|
2674
|
-
if (visited.has(handle.documentId)) {
|
|
2675
|
-
continue;
|
|
2676
|
-
}
|
|
2677
|
-
for await (const result of getObjectsFromHandle(handle)) {
|
|
2678
|
-
yield result;
|
|
2679
|
-
}
|
|
2680
|
-
visited.add(handle.documentId);
|
|
2681
|
-
}
|
|
2682
|
-
}
|
|
2683
|
-
);
|
|
2684
|
-
|
|
2685
|
-
// packages/core/echo/echo-pipeline/src/db-host/space-state-manager.ts
|
|
2686
|
-
import isEqual from "lodash.isequal";
|
|
2687
|
-
import { Event as Event3, UpdateScheduler as UpdateScheduler3 } from "@dxos/async";
|
|
2688
|
-
import { interpretAsDocumentId } from "@dxos/automerge/automerge-repo";
|
|
2689
|
-
import { Resource as Resource8, Context as Context5 } from "@dxos/context";
|
|
2690
|
-
|
|
2691
|
-
// packages/core/echo/echo-pipeline/src/db-host/database-root.ts
|
|
2692
|
-
import { SpaceDocVersion as SpaceDocVersion2 } from "@dxos/echo-protocol";
|
|
2693
|
-
import { invariant as invariant10 } from "@dxos/invariant";
|
|
2694
|
-
|
|
2695
|
-
// packages/core/echo/echo-pipeline/src/db-host/automerge-metrics.ts
|
|
2696
|
-
import * as A4 from "@dxos/automerge/automerge";
|
|
2697
|
-
import { log as log10 } from "@dxos/log";
|
|
2698
|
-
var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/automerge-metrics.ts";
|
|
2699
|
-
var measureDocMetrics = (doc) => {
|
|
2700
|
-
const snapshot = A4.save(doc);
|
|
2701
|
-
const start = Date.now();
|
|
2702
|
-
const temp = A4.load(snapshot);
|
|
2703
|
-
const end = Date.now();
|
|
2704
|
-
A4.free(temp);
|
|
2705
|
-
const getAllChangesStart = Date.now();
|
|
2706
|
-
const mutationCount = A4.getAllChanges(doc).length;
|
|
2707
|
-
const getAllChangesEnd = Date.now();
|
|
2708
|
-
if (getAllChangesEnd - getAllChangesStart > 300) {
|
|
2709
|
-
log10.warn("getAllChanges took too long", {
|
|
2710
|
-
elapsed: getAllChangesEnd - getAllChangesStart
|
|
2711
|
-
}, {
|
|
2712
|
-
F: __dxlog_file12,
|
|
2713
|
-
L: 30,
|
|
2714
|
-
S: void 0,
|
|
2715
|
-
C: (f, a) => f(...a)
|
|
2716
|
-
});
|
|
2717
|
-
}
|
|
2718
|
-
return {
|
|
2719
|
-
compressedByteSize: snapshot.byteLength,
|
|
2720
|
-
loadTime: end - start,
|
|
2721
|
-
mutationCount
|
|
2722
|
-
};
|
|
2723
|
-
};
|
|
2724
|
-
|
|
2725
|
-
// packages/core/echo/echo-pipeline/src/db-host/database-root.ts
|
|
2726
|
-
var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/database-root.ts";
|
|
2727
|
-
var DatabaseRoot = class {
|
|
2728
|
-
constructor(_rootHandle) {
|
|
2729
|
-
this._rootHandle = _rootHandle;
|
|
2730
|
-
}
|
|
2731
|
-
get documentId() {
|
|
2732
|
-
return this._rootHandle.documentId;
|
|
2733
|
-
}
|
|
2734
|
-
get url() {
|
|
2735
|
-
return this._rootHandle.url;
|
|
2736
|
-
}
|
|
2737
|
-
get isLoaded() {
|
|
2738
|
-
return !!this._rootHandle.docSync();
|
|
2739
|
-
}
|
|
2740
|
-
get handle() {
|
|
2741
|
-
return this._rootHandle;
|
|
2742
|
-
}
|
|
2743
|
-
docSync() {
|
|
2744
|
-
return this._rootHandle.docSync();
|
|
2745
|
-
}
|
|
2746
|
-
getVersion() {
|
|
2747
|
-
const doc = this.docSync();
|
|
2748
|
-
if (!doc) {
|
|
2749
|
-
return null;
|
|
2750
|
-
}
|
|
2751
|
-
return doc.version ?? SpaceDocVersion2.LEGACY;
|
|
2752
|
-
}
|
|
2753
|
-
getSpaceKey() {
|
|
2754
|
-
const doc = this.docSync();
|
|
2755
|
-
if (!doc) {
|
|
2756
|
-
return null;
|
|
2757
|
-
}
|
|
2758
|
-
return getSpaceKeyFromDoc(doc);
|
|
2759
|
-
}
|
|
2760
|
-
getInlineObjectCount() {
|
|
2761
|
-
const doc = this.docSync();
|
|
2762
|
-
if (!doc) {
|
|
2763
|
-
return null;
|
|
2764
|
-
}
|
|
2765
|
-
return Object.keys(doc.objects ?? {}).length;
|
|
2766
|
-
}
|
|
2767
|
-
getLinkedObjectCount() {
|
|
2768
|
-
const doc = this.docSync();
|
|
2769
|
-
if (!doc) {
|
|
2770
|
-
return null;
|
|
2771
|
-
}
|
|
2772
|
-
return Object.keys(doc.links ?? {}).length;
|
|
2773
|
-
}
|
|
2774
|
-
getAllLinkedDocuments() {
|
|
2775
|
-
const doc = this.docSync();
|
|
2776
|
-
invariant10(doc, void 0, {
|
|
2777
|
-
F: __dxlog_file13,
|
|
2778
|
-
L: 74,
|
|
2779
|
-
S: this,
|
|
2780
|
-
A: [
|
|
2781
|
-
"doc",
|
|
2782
|
-
""
|
|
2783
|
-
]
|
|
2784
|
-
});
|
|
2785
|
-
return Object.values(doc.links ?? {});
|
|
2786
|
-
}
|
|
2787
|
-
measureMetrics() {
|
|
2788
|
-
const doc = this.docSync();
|
|
2789
|
-
if (!doc) {
|
|
2790
|
-
return null;
|
|
2791
|
-
}
|
|
2792
|
-
return measureDocMetrics(doc);
|
|
2793
|
-
}
|
|
2794
|
-
};
|
|
2795
|
-
|
|
2796
|
-
// packages/core/echo/echo-pipeline/src/db-host/space-state-manager.ts
|
|
2797
|
-
var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/space-state-manager.ts";
|
|
2798
|
-
var SpaceStateManager = class extends Resource8 {
|
|
2799
|
-
constructor() {
|
|
2800
|
-
super(...arguments);
|
|
2801
|
-
this._roots = /* @__PURE__ */ new Map();
|
|
2802
|
-
this._rootBySpace = /* @__PURE__ */ new Map();
|
|
2803
|
-
this._perRootContext = /* @__PURE__ */ new Map();
|
|
2804
|
-
this._lastSpaceDocumentList = /* @__PURE__ */ new Map();
|
|
2805
|
-
this.spaceDocumentListUpdated = new Event3();
|
|
2806
|
-
}
|
|
2807
|
-
async _close(ctx) {
|
|
2808
|
-
for (const [_, rootCtx] of this._perRootContext) {
|
|
2809
|
-
await rootCtx.dispose();
|
|
2810
|
-
}
|
|
2811
|
-
this._roots.clear();
|
|
2812
|
-
}
|
|
2813
|
-
get roots() {
|
|
2814
|
-
return this._roots;
|
|
2815
|
-
}
|
|
2816
|
-
getRootByDocumentId(documentId) {
|
|
2817
|
-
return this._roots.get(documentId);
|
|
2818
|
-
}
|
|
2819
|
-
async assignRootToSpace(spaceId, handle) {
|
|
2820
|
-
let root;
|
|
2821
|
-
if (this._roots.has(handle.documentId)) {
|
|
2822
|
-
root = this._roots.get(handle.documentId);
|
|
2823
|
-
} else {
|
|
2824
|
-
root = new DatabaseRoot(handle);
|
|
2825
|
-
this._roots.set(handle.documentId, root);
|
|
2826
|
-
}
|
|
2827
|
-
if (this._rootBySpace.get(spaceId) === root.handle.documentId) {
|
|
2828
|
-
return root;
|
|
2829
|
-
}
|
|
2830
|
-
const prevRootId = this._rootBySpace.get(spaceId);
|
|
2831
|
-
if (prevRootId) {
|
|
2832
|
-
void this._perRootContext.get(prevRootId)?.dispose();
|
|
2833
|
-
this._perRootContext.delete(prevRootId);
|
|
2834
|
-
}
|
|
2835
|
-
this._rootBySpace.set(spaceId, root.handle.documentId);
|
|
2836
|
-
const ctx = new Context5(void 0, {
|
|
2837
|
-
F: __dxlog_file14,
|
|
2838
|
-
L: 58
|
|
2839
|
-
});
|
|
2840
|
-
this._perRootContext.set(root.handle.documentId, ctx);
|
|
2841
|
-
await root.handle.whenReady();
|
|
2842
|
-
const documentListCheckScheduler = new UpdateScheduler3(ctx, async () => {
|
|
2843
|
-
const documentIds = [
|
|
2844
|
-
root.documentId,
|
|
2845
|
-
...root.getAllLinkedDocuments().map((url) => interpretAsDocumentId(url))
|
|
2846
|
-
];
|
|
2847
|
-
if (!isEqual(documentIds, this._lastSpaceDocumentList.get(spaceId))) {
|
|
2848
|
-
this._lastSpaceDocumentList.set(spaceId, documentIds);
|
|
2849
|
-
this.spaceDocumentListUpdated.emit(new SpaceDocumentListUpdatedEvent(spaceId, documentIds));
|
|
2850
|
-
}
|
|
2851
|
-
}, {
|
|
2852
|
-
maxFrequency: 50
|
|
2853
|
-
});
|
|
2854
|
-
const triggerCheckOnChange = () => documentListCheckScheduler.trigger();
|
|
2855
|
-
root.handle.addListener("change", triggerCheckOnChange);
|
|
2856
|
-
ctx.onDispose(() => root.handle.removeListener("change", triggerCheckOnChange));
|
|
2857
|
-
documentListCheckScheduler.trigger();
|
|
2858
|
-
return root;
|
|
2859
|
-
}
|
|
2860
|
-
};
|
|
2861
|
-
var SpaceDocumentListUpdatedEvent = class {
|
|
2862
|
-
constructor(spaceId, documentIds) {
|
|
2863
|
-
this.spaceId = spaceId;
|
|
2864
|
-
this.documentIds = documentIds;
|
|
2865
|
-
}
|
|
2866
|
-
};
|
|
2867
|
-
|
|
2868
|
-
// packages/core/echo/echo-pipeline/src/db-host/echo-host.ts
|
|
2869
|
-
var __dxlog_file15 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/echo-host.ts";
|
|
2870
|
-
var INDEXER_CONFIG = {
|
|
2871
|
-
enabled: true,
|
|
2872
|
-
indexes: [
|
|
2873
|
-
{
|
|
2874
|
-
kind: IndexKind.Kind.SCHEMA_MATCH
|
|
2875
|
-
}
|
|
2876
|
-
]
|
|
2877
|
-
};
|
|
2878
|
-
var EchoHost = class extends Resource9 {
|
|
2879
|
-
constructor({ kv }) {
|
|
2880
|
-
super();
|
|
2881
|
-
this._spaceStateManager = new SpaceStateManager();
|
|
2882
|
-
this._indexMetadataStore = new IndexMetadataStore({
|
|
2883
|
-
db: kv.sublevel("index-metadata")
|
|
2884
|
-
});
|
|
2885
|
-
this._echoDataMonitor = new EchoDataMonitor();
|
|
2886
|
-
this._automergeHost = new AutomergeHost({
|
|
2887
|
-
db: kv,
|
|
2888
|
-
dataMonitor: this._echoDataMonitor,
|
|
2889
|
-
indexMetadataStore: this._indexMetadataStore
|
|
2890
|
-
});
|
|
2891
|
-
this._indexer = new Indexer({
|
|
2892
|
-
db: kv,
|
|
2893
|
-
indexStore: new IndexStore({
|
|
2894
|
-
db: kv.sublevel("index-storage")
|
|
2895
|
-
}),
|
|
2896
|
-
metadataStore: this._indexMetadataStore,
|
|
2897
|
-
loadDocuments: createSelectedDocumentsIterator(this._automergeHost),
|
|
2898
|
-
indexCooldownTime: process.env.NODE_ENV === "test" ? 0 : void 0
|
|
2899
|
-
});
|
|
2900
|
-
this._indexer.setConfig(INDEXER_CONFIG);
|
|
2901
|
-
this._queryService = new QueryServiceImpl({
|
|
2902
|
-
automergeHost: this._automergeHost,
|
|
2903
|
-
indexer: this._indexer
|
|
2904
|
-
});
|
|
2905
|
-
this._dataService = new DataServiceImpl({
|
|
2906
|
-
automergeHost: this._automergeHost,
|
|
2907
|
-
updateIndexes: async () => {
|
|
2908
|
-
await this._indexer.updateIndexes();
|
|
2909
|
-
}
|
|
2910
|
-
});
|
|
2911
|
-
trace5.diagnostic({
|
|
2912
|
-
id: "echo-stats",
|
|
2913
|
-
name: "Echo Stats",
|
|
2914
|
-
fetch: async () => {
|
|
2915
|
-
return {
|
|
2916
|
-
dataStats: this._echoDataMonitor.computeStats(),
|
|
2917
|
-
loadedDocsCount: this._automergeHost.loadedDocsCount
|
|
2918
|
-
};
|
|
2919
|
-
}
|
|
2920
|
-
});
|
|
2921
|
-
trace5.diagnostic({
|
|
2922
|
-
id: "database-roots",
|
|
2923
|
-
name: "Database Roots",
|
|
2924
|
-
fetch: async () => {
|
|
2925
|
-
return Array.from(this._spaceStateManager.roots.values()).map((root) => ({
|
|
2926
|
-
url: root.url,
|
|
2927
|
-
isLoaded: root.isLoaded,
|
|
2928
|
-
spaceKey: root.getSpaceKey(),
|
|
2929
|
-
inlineObjects: root.getInlineObjectCount(),
|
|
2930
|
-
linkedObjects: root.getLinkedObjectCount()
|
|
2931
|
-
}));
|
|
2932
|
-
}
|
|
2933
|
-
});
|
|
2934
|
-
trace5.diagnostic({
|
|
2935
|
-
id: "database-root-metrics",
|
|
2936
|
-
name: "Database Roots (with metrics)",
|
|
2937
|
-
fetch: async () => {
|
|
2938
|
-
return Array.from(this._spaceStateManager.roots.values()).map((root) => ({
|
|
2939
|
-
url: root.url,
|
|
2940
|
-
isLoaded: root.isLoaded,
|
|
2941
|
-
spaceKey: root.getSpaceKey(),
|
|
2942
|
-
inlineObjects: root.getInlineObjectCount(),
|
|
2943
|
-
linkedObjects: root.getLinkedObjectCount(),
|
|
2944
|
-
...root.measureMetrics() ?? {}
|
|
2945
|
-
}));
|
|
2946
|
-
}
|
|
2947
|
-
});
|
|
2948
|
-
}
|
|
2949
|
-
get queryService() {
|
|
2950
|
-
return this._queryService;
|
|
2951
|
-
}
|
|
2952
|
-
get dataService() {
|
|
2953
|
-
return this._dataService;
|
|
2954
|
-
}
|
|
2955
|
-
/**
|
|
2956
|
-
* @deprecated To be abstracted away.
|
|
2957
|
-
*/
|
|
2958
|
-
get automergeRepo() {
|
|
2959
|
-
return this._automergeHost.repo;
|
|
2960
|
-
}
|
|
2961
|
-
get roots() {
|
|
2962
|
-
return this._spaceStateManager.roots;
|
|
2963
|
-
}
|
|
2964
|
-
async _open(ctx) {
|
|
2965
|
-
await this._automergeHost.open();
|
|
2966
|
-
await this._indexer.open(ctx);
|
|
2967
|
-
await this._queryService.open(ctx);
|
|
2968
|
-
await this._spaceStateManager.open(ctx);
|
|
2969
|
-
this._spaceStateManager.spaceDocumentListUpdated.on(this._ctx, (e) => {
|
|
2970
|
-
void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId), e.documentIds);
|
|
2971
|
-
});
|
|
2972
|
-
}
|
|
2973
|
-
async _close(ctx) {
|
|
2974
|
-
await this._spaceStateManager.close();
|
|
2975
|
-
await this._queryService.close(ctx);
|
|
2976
|
-
await this._indexer.close(ctx);
|
|
2977
|
-
await this._automergeHost.close();
|
|
2978
|
-
}
|
|
2979
|
-
/**
|
|
2980
|
-
* Flush all pending writes to the underlying storage.
|
|
2981
|
-
*/
|
|
2982
|
-
async flush() {
|
|
2983
|
-
await this._automergeHost.repo.flush();
|
|
2984
|
-
}
|
|
2985
|
-
/**
|
|
2986
|
-
* Perform any pending index updates.
|
|
2987
|
-
*/
|
|
2988
|
-
async updateIndexes() {
|
|
2989
|
-
await this._indexer.updateIndexes();
|
|
2990
|
-
}
|
|
2991
|
-
/**
|
|
2992
|
-
* Loads the document handle from the repo and waits for it to be ready.
|
|
2993
|
-
*/
|
|
2994
|
-
async loadDoc(ctx, documentId, opts) {
|
|
2995
|
-
return await this._automergeHost.loadDoc(ctx, documentId, opts);
|
|
2996
|
-
}
|
|
2997
|
-
/**
|
|
2998
|
-
* Create new persisted document.
|
|
2999
|
-
*/
|
|
3000
|
-
createDoc(initialValue, opts) {
|
|
3001
|
-
return this._automergeHost.createDoc(initialValue, opts);
|
|
3002
|
-
}
|
|
3003
|
-
/**
|
|
3004
|
-
* Create new space root.
|
|
3005
|
-
*/
|
|
3006
|
-
async createSpaceRoot(spaceKey) {
|
|
3007
|
-
invariant11(this._lifecycleState === LifecycleState4.OPEN, void 0, {
|
|
3008
|
-
F: __dxlog_file15,
|
|
3009
|
-
L: 206,
|
|
3010
|
-
S: this,
|
|
3011
|
-
A: [
|
|
3012
|
-
"this._lifecycleState === LifecycleState.OPEN",
|
|
3013
|
-
""
|
|
3014
|
-
]
|
|
3015
|
-
});
|
|
3016
|
-
const spaceId = await createIdFromSpaceKey3(spaceKey);
|
|
3017
|
-
const automergeRoot = this._automergeHost.createDoc({
|
|
3018
|
-
version: SpaceDocVersion3.CURRENT,
|
|
3019
|
-
access: {
|
|
3020
|
-
spaceKey: spaceKey.toHex()
|
|
3021
|
-
}
|
|
3022
|
-
});
|
|
3023
|
-
await this._automergeHost.flush({
|
|
3024
|
-
documentIds: [
|
|
3025
|
-
automergeRoot.documentId
|
|
3026
|
-
]
|
|
3027
|
-
});
|
|
3028
|
-
return await this.openSpaceRoot(spaceId, automergeRoot.url);
|
|
3029
|
-
}
|
|
3030
|
-
// TODO(dmaretskyi): Change to document id.
|
|
3031
|
-
async openSpaceRoot(spaceId, automergeUrl) {
|
|
3032
|
-
invariant11(this._lifecycleState === LifecycleState4.OPEN, void 0, {
|
|
3033
|
-
F: __dxlog_file15,
|
|
3034
|
-
L: 221,
|
|
3035
|
-
S: this,
|
|
3036
|
-
A: [
|
|
3037
|
-
"this._lifecycleState === LifecycleState.OPEN",
|
|
3038
|
-
""
|
|
3039
|
-
]
|
|
3040
|
-
});
|
|
3041
|
-
const handle = this._automergeHost.repo.find(automergeUrl);
|
|
3042
|
-
return this._spaceStateManager.assignRootToSpace(spaceId, handle);
|
|
3043
|
-
}
|
|
3044
|
-
// TODO(dmaretskyi): Change to document id.
|
|
3045
|
-
async closeSpaceRoot(automergeUrl) {
|
|
3046
|
-
todo();
|
|
3047
|
-
}
|
|
3048
|
-
/**
|
|
3049
|
-
* Install data replicator.
|
|
3050
|
-
*/
|
|
3051
|
-
async addReplicator(replicator) {
|
|
3052
|
-
await this._automergeHost.addReplicator(replicator);
|
|
3053
|
-
}
|
|
3054
|
-
/**
|
|
3055
|
-
* Remove data replicator.
|
|
3056
|
-
*/
|
|
3057
|
-
async removeReplicator(replicator) {
|
|
3058
|
-
await this._automergeHost.removeReplicator(replicator);
|
|
3059
|
-
}
|
|
3060
|
-
async getSpaceSyncState(spaceId) {
|
|
3061
|
-
const collectionId = deriveCollectionIdFromSpaceId(spaceId);
|
|
3062
|
-
return this._automergeHost.getCollectionSyncState(collectionId);
|
|
3063
|
-
}
|
|
3064
|
-
};
|
|
3065
|
-
|
|
3066
|
-
// packages/core/echo/echo-pipeline/src/db-host/migration.ts
|
|
3067
|
-
import { convertLegacyReference } from "@dxos/echo-protocol";
|
|
3068
|
-
import { decodeReference, encodeReference, isLegacyReference, LEGACY_TYPE_PROPERTIES, Reference, SpaceDocVersion as SpaceDocVersion4 } from "@dxos/echo-protocol";
|
|
3069
|
-
import { TYPE_PROPERTIES } from "@dxos/echo-schema";
|
|
3070
|
-
import { deepMapValuesAsync } from "@dxos/util";
|
|
3071
|
-
var convertLegacyReferences = async (doc) => {
|
|
3072
|
-
const newDoc = await deepMapValuesAsync(doc, async (value, recurse) => {
|
|
3073
|
-
if (isLegacyReference(value)) {
|
|
3074
|
-
return convertLegacyReference(value);
|
|
3075
|
-
}
|
|
3076
|
-
return recurse(value);
|
|
3077
|
-
});
|
|
3078
|
-
newDoc.version = SpaceDocVersion4.CURRENT;
|
|
3079
|
-
return newDoc;
|
|
3080
|
-
};
|
|
3081
|
-
var convertLegacySpaceRootDoc = async (root) => {
|
|
3082
|
-
const newDoc = await convertLegacyReferences(root);
|
|
3083
|
-
const properties = findInlineObjectOfType(newDoc, LEGACY_TYPE_PROPERTIES);
|
|
3084
|
-
if (properties) {
|
|
3085
|
-
const [_, obj] = properties;
|
|
3086
|
-
obj.system.type = encodeReference(Reference.fromLegacyTypename(TYPE_PROPERTIES));
|
|
3087
|
-
}
|
|
3088
|
-
return newDoc;
|
|
3089
|
-
};
|
|
3090
|
-
var findInlineObjectOfType = (spaceDoc, typename) => {
|
|
3091
|
-
for (const id in spaceDoc.objects ?? {}) {
|
|
3092
|
-
const obj = spaceDoc.objects[id];
|
|
3093
|
-
if (obj.system.type && decodeReference(obj.system.type).objectId === typename) {
|
|
3094
|
-
return [
|
|
3095
|
-
id,
|
|
3096
|
-
obj
|
|
3097
|
-
];
|
|
3098
|
-
}
|
|
3099
|
-
}
|
|
3100
|
-
return void 0;
|
|
3101
|
-
};
|
|
3102
|
-
|
|
3103
|
-
// packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts
|
|
3104
|
-
import { Mutex, Trigger as Trigger2 } from "@dxos/async";
|
|
3105
|
-
import * as A5 from "@dxos/automerge/automerge";
|
|
3106
|
-
import { cbor as cbor2 } from "@dxos/automerge/automerge-repo";
|
|
3107
|
-
import { Context as Context6, Resource as Resource10 } from "@dxos/context";
|
|
3108
|
-
import { randomUUID } from "@dxos/crypto";
|
|
3109
|
-
import { invariant as invariant12 } from "@dxos/invariant";
|
|
3110
|
-
import { log as log11 } from "@dxos/log";
|
|
3111
|
-
import { EdgeService } from "@dxos/protocols";
|
|
3112
|
-
import { buf } from "@dxos/protocols/buf";
|
|
3113
|
-
import { MessageSchema as RouterMessageSchema } from "@dxos/protocols/buf/dxos/edge/messenger_pb";
|
|
3114
|
-
import { bufferToArray } from "@dxos/util";
|
|
3115
|
-
function _using_ctx() {
|
|
3116
|
-
var _disposeSuppressedError = typeof SuppressedError === "function" ? SuppressedError : function(error, suppressed) {
|
|
3117
|
-
var err = new Error();
|
|
3118
|
-
err.name = "SuppressedError";
|
|
3119
|
-
err.suppressed = suppressed;
|
|
3120
|
-
err.error = error;
|
|
3121
|
-
return err;
|
|
3122
|
-
}, empty = {}, stack = [];
|
|
3123
|
-
function using(isAwait, value) {
|
|
3124
|
-
if (value != null) {
|
|
3125
|
-
if (Object(value) !== value) {
|
|
3126
|
-
throw new TypeError("using declarations can only be used with objects, functions, null, or undefined.");
|
|
3127
|
-
}
|
|
3128
|
-
if (isAwait) {
|
|
3129
|
-
var dispose = value[Symbol.asyncDispose || Symbol.for("Symbol.asyncDispose")];
|
|
3130
|
-
}
|
|
3131
|
-
if (dispose == null) {
|
|
3132
|
-
dispose = value[Symbol.dispose || Symbol.for("Symbol.dispose")];
|
|
3133
|
-
}
|
|
3134
|
-
if (typeof dispose !== "function") {
|
|
3135
|
-
throw new TypeError(`Property [Symbol.dispose] is not a function.`);
|
|
3136
|
-
}
|
|
3137
|
-
stack.push({
|
|
3138
|
-
v: value,
|
|
3139
|
-
d: dispose,
|
|
3140
|
-
a: isAwait
|
|
3141
|
-
});
|
|
3142
|
-
} else if (isAwait) {
|
|
3143
|
-
stack.push({
|
|
3144
|
-
d: value,
|
|
3145
|
-
a: isAwait
|
|
3146
|
-
});
|
|
3147
|
-
}
|
|
3148
|
-
return value;
|
|
3149
|
-
}
|
|
3150
|
-
return {
|
|
3151
|
-
e: empty,
|
|
3152
|
-
u: using.bind(null, false),
|
|
3153
|
-
a: using.bind(null, true),
|
|
3154
|
-
d: function() {
|
|
3155
|
-
var error = this.e;
|
|
3156
|
-
function next() {
|
|
3157
|
-
while (resource = stack.pop()) {
|
|
3158
|
-
try {
|
|
3159
|
-
var resource, disposalResult = resource.d && resource.d.call(resource.v);
|
|
3160
|
-
if (resource.a) {
|
|
3161
|
-
return Promise.resolve(disposalResult).then(next, err);
|
|
3162
|
-
}
|
|
3163
|
-
} catch (e) {
|
|
3164
|
-
return err(e);
|
|
3165
|
-
}
|
|
3166
|
-
}
|
|
3167
|
-
if (error !== empty) throw error;
|
|
3168
|
-
}
|
|
3169
|
-
function err(e) {
|
|
3170
|
-
error = error !== empty ? new _disposeSuppressedError(error, e) : e;
|
|
3171
|
-
return next();
|
|
3172
|
-
}
|
|
3173
|
-
return next();
|
|
3174
|
-
}
|
|
3175
|
-
};
|
|
3176
|
-
}
|
|
3177
|
-
var __dxlog_file16 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts";
|
|
3178
|
-
var EchoEdgeReplicator = class {
|
|
3179
|
-
constructor({ edgeConnection, disableSharePolicy }) {
|
|
3180
|
-
this._mutex = new Mutex();
|
|
3181
|
-
this._ctx = void 0;
|
|
3182
|
-
this._context = null;
|
|
3183
|
-
this._connectedSpaces = /* @__PURE__ */ new Set();
|
|
3184
|
-
this._connections = /* @__PURE__ */ new Map();
|
|
3185
|
-
this._sharePolicyEnabled = true;
|
|
3186
|
-
this._edgeConnection = edgeConnection;
|
|
3187
|
-
this._sharePolicyEnabled = !disableSharePolicy;
|
|
3188
|
-
}
|
|
3189
|
-
async connect(context) {
|
|
3190
|
-
log11.info("connect", {
|
|
3191
|
-
peerId: context.peerId
|
|
3192
|
-
}, {
|
|
3193
|
-
F: __dxlog_file16,
|
|
3194
|
-
L: 53,
|
|
3195
|
-
S: this,
|
|
3196
|
-
C: (f, a) => f(...a)
|
|
3197
|
-
});
|
|
3198
|
-
this._context = context;
|
|
3199
|
-
this._ctx = Context6.default(void 0, {
|
|
3200
|
-
F: __dxlog_file16,
|
|
3201
|
-
L: 56
|
|
3202
|
-
});
|
|
3203
|
-
this._edgeConnection.reconnect.on(this._ctx, async () => {
|
|
3204
|
-
try {
|
|
3205
|
-
var _usingCtx = _using_ctx();
|
|
3206
|
-
const _guard = _usingCtx.u(await this._mutex.acquire());
|
|
3207
|
-
const spaces = [
|
|
3208
|
-
...this._connectedSpaces
|
|
3209
|
-
];
|
|
3210
|
-
for (const connection of this._connections.values()) {
|
|
3211
|
-
await connection.close();
|
|
3212
|
-
}
|
|
3213
|
-
this._connections.clear();
|
|
3214
|
-
if (this._context !== null) {
|
|
3215
|
-
for (const spaceId of spaces) {
|
|
3216
|
-
await this._openConnection(spaceId);
|
|
3217
|
-
}
|
|
3218
|
-
}
|
|
3219
|
-
} catch (_) {
|
|
3220
|
-
_usingCtx.e = _;
|
|
3221
|
-
} finally {
|
|
3222
|
-
_usingCtx.d();
|
|
3223
|
-
}
|
|
3224
|
-
});
|
|
3225
|
-
for (const spaceId of this._connectedSpaces) {
|
|
3226
|
-
await this._openConnection(spaceId);
|
|
3227
|
-
}
|
|
3228
|
-
}
|
|
3229
|
-
async disconnect() {
|
|
3230
|
-
try {
|
|
3231
|
-
var _usingCtx = _using_ctx();
|
|
3232
|
-
const _guard = _usingCtx.u(await this._mutex.acquire());
|
|
3233
|
-
for (const connection of this._connections.values()) {
|
|
3234
|
-
await connection.close();
|
|
3235
|
-
}
|
|
3236
|
-
this._connections.clear();
|
|
3237
|
-
} catch (_) {
|
|
3238
|
-
_usingCtx.e = _;
|
|
3239
|
-
} finally {
|
|
3240
|
-
_usingCtx.d();
|
|
3241
|
-
}
|
|
3242
|
-
}
|
|
3243
|
-
async connectToSpace(spaceId) {
|
|
3244
|
-
try {
|
|
3245
|
-
var _usingCtx = _using_ctx();
|
|
3246
|
-
const _guard = _usingCtx.u(await this._mutex.acquire());
|
|
3247
|
-
this._connectedSpaces.add(spaceId);
|
|
3248
|
-
if (this._context !== null) {
|
|
3249
|
-
await this._openConnection(spaceId);
|
|
3250
|
-
}
|
|
3251
|
-
} catch (_) {
|
|
3252
|
-
_usingCtx.e = _;
|
|
3253
|
-
} finally {
|
|
3254
|
-
_usingCtx.d();
|
|
3255
|
-
}
|
|
3256
|
-
}
|
|
3257
|
-
async disconnectFromSpace(spaceId) {
|
|
3258
|
-
try {
|
|
3259
|
-
var _usingCtx = _using_ctx();
|
|
3260
|
-
const _guard = _usingCtx.u(await this._mutex.acquire());
|
|
3261
|
-
this._connectedSpaces.delete(spaceId);
|
|
3262
|
-
const connection = this._connections.get(spaceId);
|
|
3263
|
-
if (connection) {
|
|
3264
|
-
await connection.close();
|
|
3265
|
-
this._connections.delete(spaceId);
|
|
3266
|
-
}
|
|
3267
|
-
} catch (_) {
|
|
3268
|
-
_usingCtx.e = _;
|
|
3269
|
-
} finally {
|
|
3270
|
-
_usingCtx.d();
|
|
3271
|
-
}
|
|
3272
|
-
}
|
|
3273
|
-
async _openConnection(spaceId) {
|
|
3274
|
-
invariant12(this._context, void 0, {
|
|
3275
|
-
F: __dxlog_file16,
|
|
3276
|
-
L: 111,
|
|
3277
|
-
S: this,
|
|
3278
|
-
A: [
|
|
3279
|
-
"this._context",
|
|
3280
|
-
""
|
|
3281
|
-
]
|
|
3282
|
-
});
|
|
3283
|
-
const connection = new EdgeReplicatorConnection({
|
|
3284
|
-
edgeConnection: this._edgeConnection,
|
|
3285
|
-
ownPeerId: this._context.peerId,
|
|
3286
|
-
spaceId,
|
|
3287
|
-
context: this._context,
|
|
3288
|
-
sharedPolicyEnabled: this._sharePolicyEnabled,
|
|
3289
|
-
onRemoteConnected: async () => {
|
|
3290
|
-
this._context?.onConnectionOpen(connection);
|
|
3291
|
-
},
|
|
3292
|
-
onRemoteDisconnected: async () => {
|
|
3293
|
-
this._context?.onConnectionClosed(connection);
|
|
3294
|
-
}
|
|
3295
|
-
});
|
|
3296
|
-
this._connections.set(spaceId, connection);
|
|
3297
|
-
await connection.open();
|
|
3298
|
-
}
|
|
3299
|
-
};
|
|
3300
|
-
var EdgeReplicatorConnection = class extends Resource10 {
|
|
3301
|
-
constructor({ edgeConnection, ownPeerId, spaceId, context, sharedPolicyEnabled, onRemoteConnected, onRemoteDisconnected }) {
|
|
3302
|
-
super();
|
|
3303
|
-
this._remotePeerId = null;
|
|
3304
|
-
this._streamStarted = new Trigger2();
|
|
3305
|
-
this._edgeConnection = edgeConnection;
|
|
3306
|
-
this._ownPeerId = ownPeerId;
|
|
3307
|
-
this._spaceId = spaceId;
|
|
3308
|
-
this._context = context;
|
|
3309
|
-
this._remotePeerId = `${EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}-${randomUUID()}`;
|
|
3310
|
-
this._targetServiceId = `${EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}`;
|
|
3311
|
-
this._sharedPolicyEnabled = sharedPolicyEnabled;
|
|
3312
|
-
this._onRemoteConnected = onRemoteConnected;
|
|
3313
|
-
this._onRemoteDisconnected = onRemoteDisconnected;
|
|
3314
|
-
this.readable = new ReadableStream({
|
|
3315
|
-
start: (controller) => {
|
|
3316
|
-
this._readableStreamController = controller;
|
|
3317
|
-
this._ctx.onDispose(() => controller.close());
|
|
3318
|
-
this._streamStarted.wake();
|
|
3319
|
-
}
|
|
3320
|
-
});
|
|
3321
|
-
this.writable = new WritableStream({
|
|
3322
|
-
write: async (message, controller) => {
|
|
3323
|
-
await this._sendMessage(message);
|
|
3324
|
-
}
|
|
3325
|
-
});
|
|
3326
|
-
}
|
|
3327
|
-
async _open(ctx) {
|
|
3328
|
-
this._ctx.onDispose(this._edgeConnection.addListener((msg) => {
|
|
3329
|
-
this._onMessage(msg);
|
|
3330
|
-
}));
|
|
3331
|
-
await this._onRemoteConnected();
|
|
3332
|
-
}
|
|
3333
|
-
async _close() {
|
|
3334
|
-
await this._onRemoteDisconnected();
|
|
3335
|
-
}
|
|
3336
|
-
get peerId() {
|
|
3337
|
-
invariant12(this._remotePeerId, "Not connected", {
|
|
3338
|
-
F: __dxlog_file16,
|
|
3339
|
-
L: 214,
|
|
3340
|
-
S: this,
|
|
3341
|
-
A: [
|
|
3342
|
-
"this._remotePeerId",
|
|
3343
|
-
"'Not connected'"
|
|
3344
|
-
]
|
|
3345
|
-
});
|
|
3346
|
-
return this._remotePeerId;
|
|
3347
|
-
}
|
|
3348
|
-
async shouldAdvertise(params) {
|
|
3349
|
-
if (!this._sharedPolicyEnabled) {
|
|
3350
|
-
return true;
|
|
3351
|
-
}
|
|
3352
|
-
const spaceId = await this._context.getContainingSpaceIdForDocument(params.documentId);
|
|
3353
|
-
if (!spaceId) {
|
|
3354
|
-
return true;
|
|
3355
|
-
}
|
|
3356
|
-
return spaceId === this._spaceId;
|
|
3357
|
-
}
|
|
3358
|
-
shouldSyncCollection(params) {
|
|
3359
|
-
if (!this._sharedPolicyEnabled) {
|
|
3360
|
-
return true;
|
|
3361
|
-
}
|
|
3362
|
-
const spaceId = getSpaceIdFromCollectionId(params.collectionId);
|
|
3363
|
-
return spaceId === this._spaceId;
|
|
3364
|
-
}
|
|
3365
|
-
_onMessage(message) {
|
|
3366
|
-
if (message.serviceId !== this._targetServiceId) {
|
|
3367
|
-
return;
|
|
3368
|
-
}
|
|
3369
|
-
const payload = cbor2.decode(message.payload.value);
|
|
3370
|
-
log11("recv", () => {
|
|
3371
|
-
const decodedData = payload.type === "sync" && payload.data ? A5.decodeSyncMessage(payload.data) : payload.type === "collection-state" ? payload.state : payload;
|
|
3372
|
-
return {
|
|
3373
|
-
from: message.serviceId,
|
|
3374
|
-
type: payload.type,
|
|
3375
|
-
decodedData
|
|
3376
|
-
};
|
|
3377
|
-
}, {
|
|
3378
|
-
F: __dxlog_file16,
|
|
3379
|
-
L: 245,
|
|
3380
|
-
S: this,
|
|
3381
|
-
C: (f, a) => f(...a)
|
|
3382
|
-
});
|
|
3383
|
-
payload.senderId = this._remotePeerId;
|
|
3384
|
-
this._processMessage(payload);
|
|
3385
|
-
}
|
|
3386
|
-
_processMessage(message) {
|
|
3387
|
-
this._readableStreamController.enqueue(message);
|
|
3388
|
-
}
|
|
3389
|
-
async _sendMessage(message) {
|
|
3390
|
-
message.targetId = this._targetServiceId;
|
|
3391
|
-
log11("send", {
|
|
3392
|
-
type: message.type,
|
|
3393
|
-
senderId: message.senderId,
|
|
3394
|
-
targetId: message.targetId,
|
|
3395
|
-
documentId: message.documentId
|
|
3396
|
-
}, {
|
|
3397
|
-
F: __dxlog_file16,
|
|
3398
|
-
L: 267,
|
|
3399
|
-
S: this,
|
|
3400
|
-
C: (f, a) => f(...a)
|
|
3401
|
-
});
|
|
3402
|
-
const encoded = cbor2.encode(message);
|
|
3403
|
-
await this._edgeConnection.send(buf.create(RouterMessageSchema, {
|
|
3404
|
-
serviceId: this._targetServiceId,
|
|
3405
|
-
source: {
|
|
3406
|
-
identityKey: this._edgeConnection.identityKey,
|
|
3407
|
-
peerKey: this._edgeConnection.peerKey
|
|
3408
|
-
},
|
|
3409
|
-
payload: {
|
|
3410
|
-
value: bufferToArray(encoded)
|
|
3411
|
-
}
|
|
3412
|
-
}));
|
|
3413
|
-
}
|
|
3414
|
-
};
|
|
3415
2139
|
export {
|
|
3416
|
-
AuthExtension,
|
|
3417
|
-
AuthStatus,
|
|
3418
|
-
AutomergeHost,
|
|
3419
|
-
CredentialRetrieverExtension,
|
|
3420
|
-
CredentialServerExtension,
|
|
3421
|
-
DataServiceImpl,
|
|
3422
|
-
DatabaseRoot,
|
|
3423
2140
|
DocumentsSynchronizer,
|
|
3424
|
-
EchoDataMonitor,
|
|
3425
|
-
EchoEdgeReplicator,
|
|
3426
|
-
EchoHost,
|
|
3427
|
-
LevelDBStorageAdapter,
|
|
3428
|
-
MOCK_AUTH_PROVIDER,
|
|
3429
|
-
MOCK_AUTH_VERIFIER,
|
|
3430
|
-
MeshEchoReplicator,
|
|
3431
|
-
MetadataStore,
|
|
3432
|
-
Pipeline,
|
|
3433
|
-
QueryServiceImpl,
|
|
3434
|
-
QueryState,
|
|
3435
|
-
Space,
|
|
3436
|
-
SpaceManager,
|
|
3437
|
-
SpaceProtocol,
|
|
3438
|
-
SpaceProtocolSession,
|
|
3439
|
-
TimeframeClock,
|
|
3440
|
-
codec,
|
|
3441
|
-
convertLegacyReferences,
|
|
3442
|
-
convertLegacySpaceRootDoc,
|
|
3443
|
-
createIdFromSpaceKey,
|
|
3444
|
-
createMappedFeedWriter,
|
|
3445
|
-
deriveCollectionIdFromSpaceId,
|
|
3446
2141
|
diffCollectionState,
|
|
2142
|
+
LevelDBStorageAdapter,
|
|
3447
2143
|
encodingOptions,
|
|
3448
|
-
|
|
3449
|
-
getSpaceIdFromCollectionId,
|
|
2144
|
+
AutomergeHost,
|
|
3450
2145
|
getSpaceKeyFromDoc,
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
|
|
3455
|
-
|
|
2146
|
+
deriveCollectionIdFromSpaceId,
|
|
2147
|
+
getSpaceIdFromCollectionId,
|
|
2148
|
+
MeshEchoReplicator,
|
|
2149
|
+
EchoDataMonitor,
|
|
2150
|
+
DataServiceImpl
|
|
3456
2151
|
};
|
|
3457
|
-
//# sourceMappingURL=
|
|
2152
|
+
//# sourceMappingURL=chunk-MPWFDDQK.mjs.map
|