@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1c7ec43d41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/{browser/chunk-35I6ERLG.mjs → neutral/chunk-VIHYDCCC.mjs} +313 -709
- package/dist/lib/neutral/chunk-VIHYDCCC.mjs.map +7 -0
- package/dist/lib/neutral/chunk-YBC6ONBZ.mjs +247 -0
- package/dist/lib/neutral/chunk-YBC6ONBZ.mjs.map +7 -0
- package/dist/lib/{browser → neutral}/filter/index.mjs +3 -2
- package/dist/lib/neutral/index.mjs +7065 -0
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/{browser → neutral}/testing/index.mjs +90 -311
- package/dist/lib/neutral/testing/index.mjs.map +7 -0
- package/dist/types/src/automerge/automerge-host-subduction.test.d.ts +2 -0
- package/dist/types/src/automerge/automerge-host-subduction.test.d.ts.map +1 -0
- package/dist/types/src/automerge/automerge-host.d.ts +99 -54
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts +2 -0
- package/dist/types/src/automerge/automerge-repo-subduction.test.d.ts.map +1 -0
- package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
- package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
- package/dist/types/src/automerge/automerge-subduction.test.d.ts +2 -0
- package/dist/types/src/automerge/automerge-subduction.test.d.ts.map +1 -0
- package/dist/types/src/automerge/collection-synchronizer.d.ts +25 -4
- package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-data-monitor.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts +23 -10
- package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-replicator.d.ts +38 -15
- package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/handle-state.d.ts +52 -0
- package/dist/types/src/automerge/handle-state.d.ts.map +1 -0
- package/dist/types/src/automerge/heads-store.d.ts +9 -2
- package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
- package/dist/types/src/automerge/index.d.ts +2 -1
- package/dist/types/src/automerge/index.d.ts.map +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +9 -5
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +9 -8
- package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts +6 -5
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/network-protocol.d.ts.map +1 -1
- package/dist/types/src/automerge/space-collection.d.ts.map +1 -1
- package/dist/types/src/common/codec.d.ts +1 -1
- package/dist/types/src/common/codec.d.ts.map +1 -1
- package/dist/types/src/common/feeds.d.ts.map +1 -1
- package/dist/types/src/common/space-id.d.ts.map +1 -1
- package/dist/types/src/db-host/automerge-data-source.d.ts +35 -0
- package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
- package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
- package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
- package/dist/types/src/db-host/automerge-metrics.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts +8 -7
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/database-root.d.ts +9 -3
- package/dist/types/src/db-host/database-root.d.ts.map +1 -1
- package/dist/types/src/db-host/documents-synchronizer.d.ts +9 -8
- package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
- package/dist/types/src/db-host/echo-host.d.ts +62 -27
- package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
- package/dist/types/src/db-host/index.d.ts +2 -0
- package/dist/types/src/db-host/index.d.ts.map +1 -1
- package/dist/types/src/db-host/invalidation-hint.d.ts +19 -0
- package/dist/types/src/db-host/invalidation-hint.d.ts.map +1 -0
- package/dist/types/src/db-host/local-queue-service.d.ts +19 -0
- package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
- package/dist/types/src/db-host/query-invalidation.test.d.ts +2 -0
- package/dist/types/src/db-host/query-invalidation.test.d.ts.map +1 -0
- package/dist/types/src/db-host/query-service.d.ts +19 -11
- package/dist/types/src/db-host/query-service.d.ts.map +1 -1
- package/dist/types/src/db-host/queue-data-source.d.ts +38 -0
- package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
- package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
- package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
- package/dist/types/src/db-host/space-state-manager.d.ts +4 -3
- package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
- package/dist/types/src/db-host/stub.d.ts +11 -0
- package/dist/types/src/db-host/stub.d.ts.map +1 -0
- package/dist/types/src/edge/echo-edge-replicator.d.ts +10 -7
- package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
- package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts +39 -0
- package/dist/types/src/edge/echo-edge-subduction-replicator.d.ts.map +1 -0
- package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts +2 -0
- package/dist/types/src/edge/echo-edge-subduction-replicator.test.d.ts.map +1 -0
- package/dist/types/src/edge/index.d.ts +1 -0
- package/dist/types/src/edge/index.d.ts.map +1 -1
- package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -1
- package/dist/types/src/filter/filter-match.d.ts +4 -1
- package/dist/types/src/filter/filter-match.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +3 -3
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/pipeline/timeframe-clock.d.ts.map +1 -1
- package/dist/types/src/query/errors.d.ts +22 -10
- package/dist/types/src/query/errors.d.ts.map +1 -1
- package/dist/types/src/query/plan.d.ts +66 -8
- package/dist/types/src/query/plan.d.ts.map +1 -1
- package/dist/types/src/query/query-executor.d.ts +66 -11
- package/dist/types/src/query/query-executor.d.ts.map +1 -1
- package/dist/types/src/query/query-planner.d.ts +12 -1
- package/dist/types/src/query/query-planner.d.ts.map +1 -1
- package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
- package/dist/types/src/space/auth.d.ts +3 -3
- package/dist/types/src/space/auth.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +5 -4
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts +8 -8
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts +10 -9
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +6 -6
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/change-metadata.d.ts.map +1 -1
- package/dist/types/src/testing/index.d.ts +0 -2
- package/dist/types/src/testing/index.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/testing/test-feed-builder.d.ts.map +1 -1
- package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
- package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
- package/dist/types/src/testing/test-replicator.d.ts +14 -13
- package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
- package/dist/types/src/util.d.ts +1 -1
- package/dist/types/src/util.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +56 -54
- package/src/automerge/automerge-host-subduction.test.ts +247 -0
- package/src/automerge/automerge-host.test.ts +65 -39
- package/src/automerge/automerge-host.ts +589 -206
- package/src/automerge/automerge-repo-subduction.test.ts +1049 -0
- package/src/automerge/automerge-repo.test.ts +349 -91
- package/src/automerge/automerge-subduction.test.ts +330 -0
- package/src/automerge/collection-synchronizer.test.ts +53 -17
- package/src/automerge/collection-synchronizer.ts +119 -36
- package/src/automerge/echo-data-monitor.ts +1 -1
- package/src/automerge/echo-network-adapter.test.ts +5 -4
- package/src/automerge/echo-network-adapter.ts +74 -30
- package/src/automerge/echo-replicator.ts +41 -15
- package/src/automerge/handle-state.ts +62 -0
- package/src/automerge/heads-store.ts +42 -3
- package/src/automerge/index.ts +2 -1
- package/src/automerge/leveldb-storage-adapter.ts +40 -11
- package/src/automerge/mesh-echo-replicator-connection.ts +12 -8
- package/src/automerge/mesh-echo-replicator.ts +15 -11
- package/src/automerge/storage-adapter.test.ts +1 -1
- package/src/common/space-id.ts +1 -1
- package/src/db-host/automerge-data-source.test.ts +299 -0
- package/src/db-host/automerge-data-source.ts +187 -0
- package/src/db-host/data-service.ts +25 -26
- package/src/db-host/database-root.ts +20 -8
- package/src/db-host/documents-synchronizer.test.ts +143 -14
- package/src/db-host/documents-synchronizer.ts +72 -55
- package/src/db-host/echo-host.ts +348 -96
- package/src/db-host/index.ts +2 -0
- package/src/db-host/invalidation-hint.ts +50 -0
- package/src/db-host/local-queue-service.ts +127 -0
- package/src/db-host/query-invalidation.test.ts +270 -0
- package/src/db-host/query-service.ts +111 -106
- package/src/db-host/queue-data-source.ts +160 -0
- package/src/db-host/queue-service.test.ts +145 -0
- package/src/db-host/space-state-manager.ts +12 -10
- package/src/db-host/stub.ts +33 -0
- package/src/edge/echo-edge-replicator.test.ts +12 -10
- package/src/edge/echo-edge-replicator.ts +144 -58
- package/src/edge/echo-edge-subduction-replicator.test.ts +158 -0
- package/src/edge/echo-edge-subduction-replicator.ts +377 -0
- package/src/edge/index.ts +1 -0
- package/src/filter/filter-match.test.ts +68 -32
- package/src/filter/filter-match.ts +184 -20
- package/src/metadata/metadata-store.ts +11 -8
- package/src/pipeline/pipeline-stress.test.ts +1 -1
- package/src/pipeline/pipeline.test.ts +2 -2
- package/src/pipeline/pipeline.ts +9 -7
- package/src/query/errors.ts +3 -1
- package/src/query/plan.ts +82 -8
- package/src/query/query-executor.ts +1199 -164
- package/src/query/query-planner.test.ts +993 -81
- package/src/query/query-planner.ts +467 -27
- package/src/space/admission-discovery-extension.ts +3 -3
- package/src/space/auth.ts +6 -6
- package/src/space/control-pipeline.test.ts +5 -4
- package/src/space/control-pipeline.ts +16 -13
- package/src/space/space-manager.browser.test.ts +1 -1
- package/src/space/space-manager.ts +16 -20
- package/src/space/space-protocol.browser.test.ts +15 -14
- package/src/space/space-protocol.test.ts +15 -14
- package/src/space/space-protocol.ts +22 -18
- package/src/space/space.test.ts +3 -3
- package/src/space/space.ts +11 -11
- package/src/testing/index.ts +0 -2
- package/src/testing/test-agent-builder.ts +3 -3
- package/src/testing/test-network-adapter.ts +24 -3
- package/src/testing/test-replicator.ts +19 -20
- package/src/util.ts +7 -3
- package/dist/lib/browser/chunk-35I6ERLG.mjs.map +0 -7
- package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
- package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
- package/dist/lib/browser/chunk-TQJTKNMS.mjs +0 -126
- package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +0 -7
- package/dist/lib/browser/index.mjs +0 -4743
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/browser/testing/index.mjs.map +0 -7
- package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
- package/dist/lib/node/chunk-HOPOFWAL.cjs.map +0 -7
- package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
- package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
- package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
- package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
- package/dist/lib/node/filter/index.cjs +0 -32
- package/dist/lib/node/filter/index.cjs.map +0 -7
- package/dist/lib/node/index.cjs +0 -4699
- package/dist/lib/node/index.cjs.map +0 -7
- package/dist/lib/node/meta.json +0 -1
- package/dist/lib/node/testing/index.cjs +0 -753
- package/dist/lib/node/testing/index.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-5BHLPT24.mjs +0 -2067
- package/dist/lib/node-esm/chunk-5BHLPT24.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-RVK35BS7.mjs +0 -126
- package/dist/lib/node-esm/chunk-RVK35BS7.mjs.map +0 -7
- package/dist/lib/node-esm/filter/index.mjs +0 -11
- package/dist/lib/node-esm/filter/index.mjs.map +0 -7
- package/dist/lib/node-esm/index.mjs +0 -4743
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
- package/dist/lib/node-esm/testing/index.mjs +0 -742
- package/dist/lib/node-esm/testing/index.mjs.map +0 -7
- package/dist/types/src/db-host/documents-iterator.d.ts +0 -7
- package/dist/types/src/db-host/documents-iterator.d.ts.map +0 -1
- package/dist/types/src/testing/test-data.d.ts +0 -18
- package/dist/types/src/testing/test-data.d.ts.map +0 -1
- package/dist/types/src/testing/test-schema.d.ts +0 -37
- package/dist/types/src/testing/test-schema.d.ts.map +0 -1
- package/src/db-host/documents-iterator.ts +0 -74
- package/src/testing/test-data.ts +0 -127
- package/src/testing/test-schema.ts +0 -53
- /package/dist/lib/{browser → neutral}/filter/index.mjs.map +0 -0
|
@@ -1,4743 +0,0 @@
|
|
|
1
|
-
import "@dxos/node-std/globals";
|
|
2
|
-
import {
|
|
3
|
-
filterMatchObject,
|
|
4
|
-
filterMatchValue
|
|
5
|
-
} from "./chunk-TQJTKNMS.mjs";
|
|
6
|
-
import {
|
|
7
|
-
AuthExtension,
|
|
8
|
-
AuthStatus,
|
|
9
|
-
CredentialRetrieverExtension,
|
|
10
|
-
CredentialServerExtension,
|
|
11
|
-
MOCK_AUTH_PROVIDER,
|
|
12
|
-
MOCK_AUTH_VERIFIER,
|
|
13
|
-
MetadataStore,
|
|
14
|
-
Pipeline,
|
|
15
|
-
Space,
|
|
16
|
-
SpaceManager,
|
|
17
|
-
SpaceProtocol,
|
|
18
|
-
SpaceProtocolSession,
|
|
19
|
-
TimeframeClock,
|
|
20
|
-
codec,
|
|
21
|
-
createIdFromSpaceKey,
|
|
22
|
-
createMappedFeedWriter,
|
|
23
|
-
hasInvitationExpired,
|
|
24
|
-
mapFeedIndexesToTimeframe,
|
|
25
|
-
mapTimeframeToFeedIndexes,
|
|
26
|
-
startAfter,
|
|
27
|
-
valueEncoding
|
|
28
|
-
} from "./chunk-35I6ERLG.mjs";
|
|
29
|
-
import "./chunk-CGS2ULMK.mjs";
|
|
30
|
-
|
|
31
|
-
// packages/core/echo/echo-pipeline/src/db-host/data-service.ts
|
|
32
|
-
import { UpdateScheduler as UpdateScheduler2 } from "@dxos/async";
|
|
33
|
-
import { Stream } from "@dxos/codec-protobuf/stream";
|
|
34
|
-
import { invariant as invariant7 } from "@dxos/invariant";
|
|
35
|
-
import { SpaceId as SpaceId2 } from "@dxos/keys";
|
|
36
|
-
import { log as log7 } from "@dxos/log";
|
|
37
|
-
|
|
38
|
-
// packages/core/echo/echo-pipeline/src/db-host/documents-synchronizer.ts
|
|
39
|
-
import { next as A2 } from "@automerge/automerge";
|
|
40
|
-
import { UpdateScheduler } from "@dxos/async";
|
|
41
|
-
import { Resource as Resource5 } from "@dxos/context";
|
|
42
|
-
import { invariant as invariant6 } from "@dxos/invariant";
|
|
43
|
-
import { log as log6 } from "@dxos/log";
|
|
44
|
-
|
|
45
|
-
// packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
|
|
46
|
-
import { getBackend, getHeads, isAutomerge, equals as headsEquals, save } from "@automerge/automerge";
|
|
47
|
-
import { Repo, interpretAsDocumentId } from "@automerge/automerge-repo";
|
|
48
|
-
import { Event as Event2, asyncTimeout } from "@dxos/async";
|
|
49
|
-
import { Context, Resource as Resource3, cancelWithContext } from "@dxos/context";
|
|
50
|
-
import { DatabaseDirectory } from "@dxos/echo-protocol";
|
|
51
|
-
import { invariant as invariant2 } from "@dxos/invariant";
|
|
52
|
-
import { PublicKey } from "@dxos/keys";
|
|
53
|
-
import { log as log3 } from "@dxos/log";
|
|
54
|
-
import { objectPointerCodec } from "@dxos/protocols";
|
|
55
|
-
import { trace as trace2 } from "@dxos/tracing";
|
|
56
|
-
import { bufferToArray } from "@dxos/util";
|
|
57
|
-
|
|
58
|
-
// packages/core/echo/echo-pipeline/src/automerge/collection-synchronizer.ts
|
|
59
|
-
import { next as am } from "@automerge/automerge";
|
|
60
|
-
import { asyncReturn, Event, scheduleTask, scheduleTaskInterval } from "@dxos/async";
|
|
61
|
-
import { Resource } from "@dxos/context";
|
|
62
|
-
import { log } from "@dxos/log";
|
|
63
|
-
import { trace } from "@dxos/tracing";
|
|
64
|
-
import { defaultMap } from "@dxos/util";
|
|
65
|
-
function _ts_decorate(decorators, target, key, desc) {
|
|
66
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
67
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
68
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
69
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
70
|
-
}
|
|
71
|
-
var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/collection-synchronizer.ts";
|
|
72
|
-
var MIN_QUERY_INTERVAL = 5e3;
|
|
73
|
-
var POLL_INTERVAL = 3e4;
|
|
74
|
-
var CollectionSynchronizer = class extends Resource {
|
|
75
|
-
constructor(params) {
|
|
76
|
-
super();
|
|
77
|
-
/**
|
|
78
|
-
* CollectionId -> State.
|
|
79
|
-
*/
|
|
80
|
-
this._perCollectionStates = /* @__PURE__ */ new Map();
|
|
81
|
-
this._activeCollections = /* @__PURE__ */ new Set();
|
|
82
|
-
this._connectedPeers = /* @__PURE__ */ new Set();
|
|
83
|
-
this.remoteStateUpdated = new Event();
|
|
84
|
-
this._sendCollectionState = params.sendCollectionState;
|
|
85
|
-
this._queryCollectionState = params.queryCollectionState;
|
|
86
|
-
this._shouldSyncCollection = params.shouldSyncCollection;
|
|
87
|
-
}
|
|
88
|
-
async _open(ctx) {
|
|
89
|
-
scheduleTaskInterval(this._ctx, async () => {
|
|
90
|
-
for (const collectionId of this._perCollectionStates.keys()) {
|
|
91
|
-
if (this._activeCollections.has(collectionId)) {
|
|
92
|
-
this.refreshCollection(collectionId);
|
|
93
|
-
await asyncReturn();
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}, POLL_INTERVAL);
|
|
97
|
-
}
|
|
98
|
-
getRegisteredCollectionIds() {
|
|
99
|
-
return [
|
|
100
|
-
...this._activeCollections
|
|
101
|
-
];
|
|
102
|
-
}
|
|
103
|
-
getLocalCollectionState(collectionId) {
|
|
104
|
-
return this._perCollectionStates.get(collectionId)?.localState;
|
|
105
|
-
}
|
|
106
|
-
setLocalCollectionState(collectionId, state) {
|
|
107
|
-
this._activeCollections.add(collectionId);
|
|
108
|
-
log("setLocalCollectionState", {
|
|
109
|
-
collectionId,
|
|
110
|
-
state
|
|
111
|
-
}, {
|
|
112
|
-
F: __dxlog_file,
|
|
113
|
-
L: 76,
|
|
114
|
-
S: this,
|
|
115
|
-
C: (f, a) => f(...a)
|
|
116
|
-
});
|
|
117
|
-
this._getOrCreatePerCollectionState(collectionId).localState = state;
|
|
118
|
-
queueMicrotask(async () => {
|
|
119
|
-
if (!this._ctx.disposed && this._activeCollections.has(collectionId)) {
|
|
120
|
-
this._refreshInterestedPeers(collectionId);
|
|
121
|
-
this.refreshCollection(collectionId);
|
|
122
|
-
}
|
|
123
|
-
});
|
|
124
|
-
}
|
|
125
|
-
clearLocalCollectionState(collectionId) {
|
|
126
|
-
this._activeCollections.delete(collectionId);
|
|
127
|
-
this._perCollectionStates.delete(collectionId);
|
|
128
|
-
log("clearLocalCollectionState", {
|
|
129
|
-
collectionId
|
|
130
|
-
}, {
|
|
131
|
-
F: __dxlog_file,
|
|
132
|
-
L: 90,
|
|
133
|
-
S: this,
|
|
134
|
-
C: (f, a) => f(...a)
|
|
135
|
-
});
|
|
136
|
-
}
|
|
137
|
-
getRemoteCollectionStates(collectionId) {
|
|
138
|
-
return this._getOrCreatePerCollectionState(collectionId).remoteStates;
|
|
139
|
-
}
|
|
140
|
-
refreshCollection(collectionId) {
|
|
141
|
-
let scheduleAnotherRefresh = false;
|
|
142
|
-
const state = this._getOrCreatePerCollectionState(collectionId);
|
|
143
|
-
for (const peerId of this._connectedPeers) {
|
|
144
|
-
if (state.interestedPeers.has(peerId)) {
|
|
145
|
-
const lastQueried = state.lastQueried.get(peerId) ?? 0;
|
|
146
|
-
if (Date.now() - lastQueried > MIN_QUERY_INTERVAL) {
|
|
147
|
-
state.lastQueried.set(peerId, Date.now());
|
|
148
|
-
this._queryCollectionState(collectionId, peerId);
|
|
149
|
-
} else {
|
|
150
|
-
scheduleAnotherRefresh = true;
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
if (scheduleAnotherRefresh) {
|
|
155
|
-
scheduleTask(this._ctx, () => this.refreshCollection(collectionId), MIN_QUERY_INTERVAL);
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
/**
|
|
159
|
-
* Callback when a connection to a peer is established.
|
|
160
|
-
*/
|
|
161
|
-
onConnectionOpen(peerId) {
|
|
162
|
-
const spanId = getSpanName(peerId);
|
|
163
|
-
trace.spanStart({
|
|
164
|
-
id: spanId,
|
|
165
|
-
methodName: spanId,
|
|
166
|
-
instance: this,
|
|
167
|
-
parentCtx: this._ctx,
|
|
168
|
-
showInBrowserTimeline: true,
|
|
169
|
-
attributes: {
|
|
170
|
-
peerId
|
|
171
|
-
}
|
|
172
|
-
});
|
|
173
|
-
this._connectedPeers.add(peerId);
|
|
174
|
-
queueMicrotask(async () => {
|
|
175
|
-
if (this._ctx.disposed) {
|
|
176
|
-
return;
|
|
177
|
-
}
|
|
178
|
-
for (const [collectionId, state] of this._perCollectionStates.entries()) {
|
|
179
|
-
if (this._activeCollections.has(collectionId) && this._shouldSyncCollection(collectionId, peerId)) {
|
|
180
|
-
state.interestedPeers.add(peerId);
|
|
181
|
-
state.lastQueried.set(peerId, Date.now());
|
|
182
|
-
this._queryCollectionState(collectionId, peerId);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
});
|
|
186
|
-
}
|
|
187
|
-
/**
|
|
188
|
-
* Callback when a connection to a peer is closed.
|
|
189
|
-
*/
|
|
190
|
-
onConnectionClosed(peerId) {
|
|
191
|
-
this._connectedPeers.delete(peerId);
|
|
192
|
-
for (const perCollectionState of this._perCollectionStates.values()) {
|
|
193
|
-
perCollectionState.remoteStates.delete(peerId);
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
/**
|
|
197
|
-
* Callback when a peer queries the state of a collection.
|
|
198
|
-
*/
|
|
199
|
-
onCollectionStateQueried(collectionId, peerId) {
|
|
200
|
-
const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
|
|
201
|
-
if (perCollectionState.localState) {
|
|
202
|
-
this._sendCollectionState(collectionId, peerId, perCollectionState.localState);
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
/**
|
|
206
|
-
* Callback when a peer sends the state of a collection.
|
|
207
|
-
*/
|
|
208
|
-
onRemoteStateReceived(collectionId, peerId, state) {
|
|
209
|
-
log("onRemoteStateReceived", {
|
|
210
|
-
collectionId,
|
|
211
|
-
peerId,
|
|
212
|
-
state
|
|
213
|
-
}, {
|
|
214
|
-
F: __dxlog_file,
|
|
215
|
-
L: 171,
|
|
216
|
-
S: this,
|
|
217
|
-
C: (f, a) => f(...a)
|
|
218
|
-
});
|
|
219
|
-
validateCollectionState(state);
|
|
220
|
-
const perCollectionState = this._getOrCreatePerCollectionState(collectionId);
|
|
221
|
-
const existingState = perCollectionState.remoteStates.get(peerId) ?? {
|
|
222
|
-
documents: {}
|
|
223
|
-
};
|
|
224
|
-
const diff = diffCollectionState(existingState, state);
|
|
225
|
-
const spanId = getSpanName(peerId);
|
|
226
|
-
if (diff.different.length === 0) {
|
|
227
|
-
trace.spanEnd(spanId);
|
|
228
|
-
} else {
|
|
229
|
-
trace.spanStart({
|
|
230
|
-
id: spanId,
|
|
231
|
-
methodName: spanId,
|
|
232
|
-
instance: this,
|
|
233
|
-
parentCtx: this._ctx,
|
|
234
|
-
showInBrowserTimeline: true,
|
|
235
|
-
attributes: {
|
|
236
|
-
peerId
|
|
237
|
-
}
|
|
238
|
-
});
|
|
239
|
-
}
|
|
240
|
-
if (diff.missingOnLocal.length > 0 || diff.different.length > 0) {
|
|
241
|
-
perCollectionState.remoteStates.set(peerId, state);
|
|
242
|
-
this.remoteStateUpdated.emit({
|
|
243
|
-
peerId,
|
|
244
|
-
collectionId,
|
|
245
|
-
newDocsAppeared: diff.missingOnLocal.length > 0
|
|
246
|
-
});
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
_getOrCreatePerCollectionState(collectionId) {
|
|
250
|
-
return defaultMap(this._perCollectionStates, collectionId, () => ({
|
|
251
|
-
localState: void 0,
|
|
252
|
-
remoteStates: /* @__PURE__ */ new Map(),
|
|
253
|
-
interestedPeers: /* @__PURE__ */ new Set(),
|
|
254
|
-
lastQueried: /* @__PURE__ */ new Map()
|
|
255
|
-
}));
|
|
256
|
-
}
|
|
257
|
-
_refreshInterestedPeers(collectionId) {
|
|
258
|
-
for (const peerId of this._connectedPeers) {
|
|
259
|
-
if (this._shouldSyncCollection(collectionId, peerId)) {
|
|
260
|
-
this._getOrCreatePerCollectionState(collectionId).interestedPeers.add(peerId);
|
|
261
|
-
} else {
|
|
262
|
-
this._getOrCreatePerCollectionState(collectionId).interestedPeers.delete(peerId);
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
};
|
|
267
|
-
CollectionSynchronizer = _ts_decorate([
|
|
268
|
-
trace.resource()
|
|
269
|
-
], CollectionSynchronizer);
|
|
270
|
-
var diffCollectionState = (local, remote) => {
|
|
271
|
-
const allDocuments = /* @__PURE__ */ new Set([
|
|
272
|
-
...Object.keys(local.documents),
|
|
273
|
-
...Object.keys(remote.documents)
|
|
274
|
-
]);
|
|
275
|
-
const missingOnRemote = [];
|
|
276
|
-
const missingOnLocal = [];
|
|
277
|
-
const different = [];
|
|
278
|
-
for (const documentId of allDocuments) {
|
|
279
|
-
if (!local.documents[documentId]) {
|
|
280
|
-
missingOnLocal.push(documentId);
|
|
281
|
-
} else if (!remote.documents[documentId]) {
|
|
282
|
-
missingOnRemote.push(documentId);
|
|
283
|
-
} else if (!am.equals(local.documents[documentId], remote.documents[documentId])) {
|
|
284
|
-
different.push(documentId);
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
return {
|
|
288
|
-
missingOnRemote,
|
|
289
|
-
missingOnLocal,
|
|
290
|
-
different
|
|
291
|
-
};
|
|
292
|
-
};
|
|
293
|
-
var validateCollectionState = (state) => {
|
|
294
|
-
Object.entries(state.documents).forEach(([documentId, heads]) => {
|
|
295
|
-
if (!isValidDocumentId(documentId)) {
|
|
296
|
-
throw new Error(`Invalid documentId: ${documentId}`);
|
|
297
|
-
}
|
|
298
|
-
if (Array.isArray(heads) && heads.some((head) => typeof head !== "string")) {
|
|
299
|
-
throw new Error(`Invalid heads: ${heads}`);
|
|
300
|
-
}
|
|
301
|
-
});
|
|
302
|
-
};
|
|
303
|
-
var isValidDocumentId = (documentId) => {
|
|
304
|
-
return typeof documentId === "string" && !documentId.includes(":");
|
|
305
|
-
};
|
|
306
|
-
var getSpanName = (peerId) => {
|
|
307
|
-
return `collection-sync-${peerId}`;
|
|
308
|
-
};
|
|
309
|
-
|
|
310
|
-
// packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts
|
|
311
|
-
import { NetworkAdapter } from "@automerge/automerge-repo";
|
|
312
|
-
import { synchronized, Trigger } from "@dxos/async";
|
|
313
|
-
import { LifecycleState } from "@dxos/context";
|
|
314
|
-
import { invariant } from "@dxos/invariant";
|
|
315
|
-
import { log as log2 } from "@dxos/log";
|
|
316
|
-
import { isNonNullable } from "@dxos/util";
|
|
317
|
-
|
|
318
|
-
// packages/core/echo/echo-pipeline/src/automerge/network-protocol.ts
|
|
319
|
-
import { MESSAGE_TYPE_COLLECTION_QUERY, MESSAGE_TYPE_COLLECTION_STATE } from "@dxos/protocols";
|
|
320
|
-
var isCollectionQueryMessage = (message) => message.type === MESSAGE_TYPE_COLLECTION_QUERY;
|
|
321
|
-
var isCollectionStateMessage = (message) => message.type === MESSAGE_TYPE_COLLECTION_STATE;
|
|
322
|
-
|
|
323
|
-
// packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts
|
|
324
|
-
function _ts_decorate2(decorators, target, key, desc) {
|
|
325
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
326
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
327
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
328
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
329
|
-
}
|
|
330
|
-
var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/echo-network-adapter.ts";
|
|
331
|
-
var EchoNetworkAdapter = class extends NetworkAdapter {
|
|
332
|
-
constructor(_params) {
|
|
333
|
-
super(), this._params = _params, this._replicators = /* @__PURE__ */ new Set(), this._connections = /* @__PURE__ */ new Map(), this._lifecycleState = LifecycleState.CLOSED, this._connected = new Trigger(), this._ready = new Trigger();
|
|
334
|
-
}
|
|
335
|
-
isReady() {
|
|
336
|
-
return this._lifecycleState === LifecycleState.OPEN;
|
|
337
|
-
}
|
|
338
|
-
whenReady() {
|
|
339
|
-
return this._ready.wait();
|
|
340
|
-
}
|
|
341
|
-
connect(peerId, peerMetadata) {
|
|
342
|
-
this.peerId = peerId;
|
|
343
|
-
this.peerMetadata = peerMetadata;
|
|
344
|
-
this._connected.wake();
|
|
345
|
-
}
|
|
346
|
-
send(message) {
|
|
347
|
-
this._send(message);
|
|
348
|
-
}
|
|
349
|
-
disconnect() {
|
|
350
|
-
}
|
|
351
|
-
async open() {
|
|
352
|
-
if (this._lifecycleState === LifecycleState.OPEN) {
|
|
353
|
-
return;
|
|
354
|
-
}
|
|
355
|
-
this._lifecycleState = LifecycleState.OPEN;
|
|
356
|
-
this._ready.wake();
|
|
357
|
-
}
|
|
358
|
-
async close() {
|
|
359
|
-
if (this._lifecycleState === LifecycleState.CLOSED) {
|
|
360
|
-
return this;
|
|
361
|
-
}
|
|
362
|
-
for (const replicator of this._replicators) {
|
|
363
|
-
await replicator.disconnect();
|
|
364
|
-
}
|
|
365
|
-
this._replicators.clear();
|
|
366
|
-
this._ready.reset();
|
|
367
|
-
this._lifecycleState = LifecycleState.CLOSED;
|
|
368
|
-
}
|
|
369
|
-
async whenConnected() {
|
|
370
|
-
await this._connected.wait({
|
|
371
|
-
timeout: 1e4
|
|
372
|
-
});
|
|
373
|
-
}
|
|
374
|
-
onConnectionAuthScopeChanged(peer) {
|
|
375
|
-
const entry = this._connections.get(peer);
|
|
376
|
-
if (entry) {
|
|
377
|
-
this._onConnectionAuthScopeChanged(entry.connection);
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
async addReplicator(replicator) {
|
|
381
|
-
invariant(this._lifecycleState === LifecycleState.OPEN, void 0, {
|
|
382
|
-
F: __dxlog_file2,
|
|
383
|
-
L: 129,
|
|
384
|
-
S: this,
|
|
385
|
-
A: [
|
|
386
|
-
"this._lifecycleState === LifecycleState.OPEN",
|
|
387
|
-
""
|
|
388
|
-
]
|
|
389
|
-
});
|
|
390
|
-
invariant(this.peerId, void 0, {
|
|
391
|
-
F: __dxlog_file2,
|
|
392
|
-
L: 130,
|
|
393
|
-
S: this,
|
|
394
|
-
A: [
|
|
395
|
-
"this.peerId",
|
|
396
|
-
""
|
|
397
|
-
]
|
|
398
|
-
});
|
|
399
|
-
invariant(!this._replicators.has(replicator), void 0, {
|
|
400
|
-
F: __dxlog_file2,
|
|
401
|
-
L: 131,
|
|
402
|
-
S: this,
|
|
403
|
-
A: [
|
|
404
|
-
"!this._replicators.has(replicator)",
|
|
405
|
-
""
|
|
406
|
-
]
|
|
407
|
-
});
|
|
408
|
-
this._replicators.add(replicator);
|
|
409
|
-
await replicator.connect({
|
|
410
|
-
peerId: this.peerId,
|
|
411
|
-
onConnectionOpen: this._onConnectionOpen.bind(this),
|
|
412
|
-
onConnectionClosed: this._onConnectionClosed.bind(this),
|
|
413
|
-
onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
|
|
414
|
-
isDocumentInRemoteCollection: this._params.isDocumentInRemoteCollection,
|
|
415
|
-
getContainingSpaceForDocument: this._params.getContainingSpaceForDocument,
|
|
416
|
-
getContainingSpaceIdForDocument: async (documentId) => {
|
|
417
|
-
const key = await this._params.getContainingSpaceForDocument(documentId);
|
|
418
|
-
return key ? createIdFromSpaceKey(key) : null;
|
|
419
|
-
}
|
|
420
|
-
});
|
|
421
|
-
}
|
|
422
|
-
async removeReplicator(replicator) {
|
|
423
|
-
invariant(this._lifecycleState === LifecycleState.OPEN, void 0, {
|
|
424
|
-
F: __dxlog_file2,
|
|
425
|
-
L: 150,
|
|
426
|
-
S: this,
|
|
427
|
-
A: [
|
|
428
|
-
"this._lifecycleState === LifecycleState.OPEN",
|
|
429
|
-
""
|
|
430
|
-
]
|
|
431
|
-
});
|
|
432
|
-
invariant(this._replicators.has(replicator), void 0, {
|
|
433
|
-
F: __dxlog_file2,
|
|
434
|
-
L: 151,
|
|
435
|
-
S: this,
|
|
436
|
-
A: [
|
|
437
|
-
"this._replicators.has(replicator)",
|
|
438
|
-
""
|
|
439
|
-
]
|
|
440
|
-
});
|
|
441
|
-
await replicator.disconnect();
|
|
442
|
-
this._replicators.delete(replicator);
|
|
443
|
-
}
|
|
444
|
-
async shouldAdvertise(peerId, params) {
|
|
445
|
-
const connection = this._connections.get(peerId);
|
|
446
|
-
if (!connection) {
|
|
447
|
-
return false;
|
|
448
|
-
}
|
|
449
|
-
return connection.connection.shouldAdvertise(params);
|
|
450
|
-
}
|
|
451
|
-
shouldSyncCollection(peerId, params) {
|
|
452
|
-
const connection = this._connections.get(peerId);
|
|
453
|
-
if (!connection) {
|
|
454
|
-
return false;
|
|
455
|
-
}
|
|
456
|
-
return connection.connection.shouldSyncCollection(params);
|
|
457
|
-
}
|
|
458
|
-
queryCollectionState(collectionId, targetId) {
|
|
459
|
-
const message = {
|
|
460
|
-
type: "collection-query",
|
|
461
|
-
senderId: this.peerId,
|
|
462
|
-
targetId,
|
|
463
|
-
collectionId
|
|
464
|
-
};
|
|
465
|
-
this._send(message);
|
|
466
|
-
}
|
|
467
|
-
sendCollectionState(collectionId, targetId, state) {
|
|
468
|
-
const message = {
|
|
469
|
-
type: "collection-state",
|
|
470
|
-
senderId: this.peerId,
|
|
471
|
-
targetId,
|
|
472
|
-
collectionId,
|
|
473
|
-
state
|
|
474
|
-
};
|
|
475
|
-
this._send(message);
|
|
476
|
-
}
|
|
477
|
-
// TODO(dmaretskyi): Remove.
|
|
478
|
-
getPeersInterestedInCollection(collectionId) {
|
|
479
|
-
return Array.from(this._connections.values()).map((connection) => {
|
|
480
|
-
return connection.connection.shouldSyncCollection({
|
|
481
|
-
collectionId
|
|
482
|
-
}) ? connection.connection.peerId : null;
|
|
483
|
-
}).filter(isNonNullable);
|
|
484
|
-
}
|
|
485
|
-
_send(message) {
|
|
486
|
-
const connectionEntry = this._connections.get(message.targetId);
|
|
487
|
-
if (!connectionEntry) {
|
|
488
|
-
throw new Error("Connection not found.");
|
|
489
|
-
}
|
|
490
|
-
const start = Date.now();
|
|
491
|
-
connectionEntry.writer.write(message).then(() => {
|
|
492
|
-
this._params.monitor?.recordMessageSent(message, Date.now() - start);
|
|
493
|
-
}).catch((err) => {
|
|
494
|
-
if (connectionEntry.isOpen) {
|
|
495
|
-
log2.catch(err, void 0, {
|
|
496
|
-
F: __dxlog_file2,
|
|
497
|
-
L: 221,
|
|
498
|
-
S: this,
|
|
499
|
-
C: (f, a) => f(...a)
|
|
500
|
-
});
|
|
501
|
-
}
|
|
502
|
-
this._params.monitor?.recordMessageSendingFailed(message);
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
_onConnectionOpen(connection) {
|
|
506
|
-
log2("connection opened", {
|
|
507
|
-
peerId: connection.peerId
|
|
508
|
-
}, {
|
|
509
|
-
F: __dxlog_file2,
|
|
510
|
-
L: 229,
|
|
511
|
-
S: this,
|
|
512
|
-
C: (f, a) => f(...a)
|
|
513
|
-
});
|
|
514
|
-
invariant(!this._connections.has(connection.peerId), void 0, {
|
|
515
|
-
F: __dxlog_file2,
|
|
516
|
-
L: 230,
|
|
517
|
-
S: this,
|
|
518
|
-
A: [
|
|
519
|
-
"!this._connections.has(connection.peerId as PeerId)",
|
|
520
|
-
""
|
|
521
|
-
]
|
|
522
|
-
});
|
|
523
|
-
const connectionEntry = {
|
|
524
|
-
isOpen: true,
|
|
525
|
-
connection,
|
|
526
|
-
reader: connection.readable.getReader(),
|
|
527
|
-
writer: connection.writable.getWriter()
|
|
528
|
-
};
|
|
529
|
-
this._connections.set(connection.peerId, connectionEntry);
|
|
530
|
-
queueMicrotask(async () => {
|
|
531
|
-
try {
|
|
532
|
-
while (true) {
|
|
533
|
-
const { done, value } = await connectionEntry.reader.read();
|
|
534
|
-
if (done) {
|
|
535
|
-
break;
|
|
536
|
-
}
|
|
537
|
-
this._onMessage(value);
|
|
538
|
-
}
|
|
539
|
-
} catch (err) {
|
|
540
|
-
if (connectionEntry.isOpen) {
|
|
541
|
-
log2.catch(err, void 0, {
|
|
542
|
-
F: __dxlog_file2,
|
|
543
|
-
L: 254,
|
|
544
|
-
S: this,
|
|
545
|
-
C: (f, a) => f(...a)
|
|
546
|
-
});
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
});
|
|
550
|
-
log2("emit peer-candidate", {
|
|
551
|
-
peerId: connection.peerId
|
|
552
|
-
}, {
|
|
553
|
-
F: __dxlog_file2,
|
|
554
|
-
L: 259,
|
|
555
|
-
S: this,
|
|
556
|
-
C: (f, a) => f(...a)
|
|
557
|
-
});
|
|
558
|
-
this._emitPeerCandidate(connection);
|
|
559
|
-
this._params.monitor?.recordPeerConnected(connection.peerId);
|
|
560
|
-
}
|
|
561
|
-
_onMessage(message) {
|
|
562
|
-
if (isCollectionQueryMessage(message)) {
|
|
563
|
-
this._params.onCollectionStateQueried(message.collectionId, message.senderId);
|
|
564
|
-
} else if (isCollectionStateMessage(message)) {
|
|
565
|
-
this._params.onCollectionStateReceived(message.collectionId, message.senderId, message.state);
|
|
566
|
-
} else {
|
|
567
|
-
this.emit("message", message);
|
|
568
|
-
}
|
|
569
|
-
this._params.monitor?.recordMessageReceived(message);
|
|
570
|
-
}
|
|
571
|
-
_onConnectionClosed(connection) {
|
|
572
|
-
log2("connection closed", {
|
|
573
|
-
peerId: connection.peerId
|
|
574
|
-
}, {
|
|
575
|
-
F: __dxlog_file2,
|
|
576
|
-
L: 276,
|
|
577
|
-
S: this,
|
|
578
|
-
C: (f, a) => f(...a)
|
|
579
|
-
});
|
|
580
|
-
const entry = this._connections.get(connection.peerId);
|
|
581
|
-
invariant(entry, void 0, {
|
|
582
|
-
F: __dxlog_file2,
|
|
583
|
-
L: 278,
|
|
584
|
-
S: this,
|
|
585
|
-
A: [
|
|
586
|
-
"entry",
|
|
587
|
-
""
|
|
588
|
-
]
|
|
589
|
-
});
|
|
590
|
-
entry.isOpen = false;
|
|
591
|
-
this.emit("peer-disconnected", {
|
|
592
|
-
peerId: connection.peerId
|
|
593
|
-
});
|
|
594
|
-
this._params.monitor?.recordPeerDisconnected(connection.peerId);
|
|
595
|
-
void entry.reader.cancel().catch((err) => log2.catch(err, void 0, {
|
|
596
|
-
F: __dxlog_file2,
|
|
597
|
-
L: 284,
|
|
598
|
-
S: this,
|
|
599
|
-
C: (f, a) => f(...a)
|
|
600
|
-
}));
|
|
601
|
-
void entry.writer.abort().catch((err) => log2.catch(err, void 0, {
|
|
602
|
-
F: __dxlog_file2,
|
|
603
|
-
L: 285,
|
|
604
|
-
S: this,
|
|
605
|
-
C: (f, a) => f(...a)
|
|
606
|
-
}));
|
|
607
|
-
this._connections.delete(connection.peerId);
|
|
608
|
-
}
|
|
609
|
-
/**
|
|
610
|
-
* Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
|
|
611
|
-
* TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
|
|
612
|
-
*/
|
|
613
|
-
_onConnectionAuthScopeChanged(connection) {
|
|
614
|
-
log2("Connection auth scope changed", {
|
|
615
|
-
peerId: connection.peerId
|
|
616
|
-
}, {
|
|
617
|
-
F: __dxlog_file2,
|
|
618
|
-
L: 294,
|
|
619
|
-
S: this,
|
|
620
|
-
C: (f, a) => f(...a)
|
|
621
|
-
});
|
|
622
|
-
const entry = this._connections.get(connection.peerId);
|
|
623
|
-
invariant(entry, void 0, {
|
|
624
|
-
F: __dxlog_file2,
|
|
625
|
-
L: 296,
|
|
626
|
-
S: this,
|
|
627
|
-
A: [
|
|
628
|
-
"entry",
|
|
629
|
-
""
|
|
630
|
-
]
|
|
631
|
-
});
|
|
632
|
-
this.emit("peer-disconnected", {
|
|
633
|
-
peerId: connection.peerId
|
|
634
|
-
});
|
|
635
|
-
this._emitPeerCandidate(connection);
|
|
636
|
-
}
|
|
637
|
-
_emitPeerCandidate(connection) {
|
|
638
|
-
this.emit("peer-candidate", {
|
|
639
|
-
peerId: connection.peerId,
|
|
640
|
-
peerMetadata: createEchoPeerMetadata()
|
|
641
|
-
});
|
|
642
|
-
}
|
|
643
|
-
};
|
|
644
|
-
_ts_decorate2([
|
|
645
|
-
synchronized
|
|
646
|
-
], EchoNetworkAdapter.prototype, "open", null);
|
|
647
|
-
_ts_decorate2([
|
|
648
|
-
synchronized
|
|
649
|
-
], EchoNetworkAdapter.prototype, "close", null);
|
|
650
|
-
_ts_decorate2([
|
|
651
|
-
synchronized
|
|
652
|
-
], EchoNetworkAdapter.prototype, "addReplicator", null);
|
|
653
|
-
_ts_decorate2([
|
|
654
|
-
synchronized
|
|
655
|
-
], EchoNetworkAdapter.prototype, "removeReplicator", null);
|
|
656
|
-
var createEchoPeerMetadata = () => ({
|
|
657
|
-
// TODO(dmaretskyi): Refactor this.
|
|
658
|
-
dxos_peerSource: "EchoNetworkAdapter"
|
|
659
|
-
});
|
|
660
|
-
var isEchoPeerMetadata = (metadata) => metadata?.dxos_peerSource === "EchoNetworkAdapter";
|
|
661
|
-
|
|
662
|
-
// packages/core/echo/echo-pipeline/src/automerge/heads-store.ts
|
|
663
|
-
import { headsEncoding } from "@dxos/indexing";
|
|
664
|
-
var HeadsStore = class {
|
|
665
|
-
constructor({ db }) {
|
|
666
|
-
this._db = db;
|
|
667
|
-
}
|
|
668
|
-
setHeads(documentId, heads, batch) {
|
|
669
|
-
batch.put(documentId, heads, {
|
|
670
|
-
sublevel: this._db,
|
|
671
|
-
keyEncoding: "utf8",
|
|
672
|
-
valueEncoding: headsEncoding
|
|
673
|
-
});
|
|
674
|
-
}
|
|
675
|
-
// TODO(dmaretskyi): Make batched.
|
|
676
|
-
async getHeads(documentIds) {
|
|
677
|
-
return this._db.getMany(documentIds, {
|
|
678
|
-
keyEncoding: "utf8",
|
|
679
|
-
valueEncoding: headsEncoding
|
|
680
|
-
});
|
|
681
|
-
}
|
|
682
|
-
};
|
|
683
|
-
|
|
684
|
-
// packages/core/echo/echo-pipeline/src/automerge/leveldb-storage-adapter.ts
|
|
685
|
-
import { LifecycleState as LifecycleState2, Resource as Resource2 } from "@dxos/context";
|
|
686
|
-
var LevelDBStorageAdapter = class extends Resource2 {
|
|
687
|
-
constructor(_params) {
|
|
688
|
-
super(), this._params = _params;
|
|
689
|
-
}
|
|
690
|
-
async load(keyArray) {
|
|
691
|
-
try {
|
|
692
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
693
|
-
return void 0;
|
|
694
|
-
}
|
|
695
|
-
const startMs = Date.now();
|
|
696
|
-
const chunk = await this._params.db.get(keyArray, {
|
|
697
|
-
...encodingOptions
|
|
698
|
-
});
|
|
699
|
-
this._params.monitor?.recordBytesLoaded(chunk.byteLength);
|
|
700
|
-
this._params.monitor?.recordLoadDuration(Date.now() - startMs);
|
|
701
|
-
return chunk;
|
|
702
|
-
} catch (err) {
|
|
703
|
-
if (isLevelDbNotFoundError(err)) {
|
|
704
|
-
return void 0;
|
|
705
|
-
}
|
|
706
|
-
throw err;
|
|
707
|
-
}
|
|
708
|
-
}
|
|
709
|
-
async save(keyArray, binary) {
|
|
710
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
711
|
-
return void 0;
|
|
712
|
-
}
|
|
713
|
-
const startMs = Date.now();
|
|
714
|
-
const batch = this._params.db.batch();
|
|
715
|
-
await this._params.callbacks?.beforeSave?.({
|
|
716
|
-
path: keyArray,
|
|
717
|
-
batch
|
|
718
|
-
});
|
|
719
|
-
batch.put(keyArray, Buffer.from(binary), {
|
|
720
|
-
...encodingOptions
|
|
721
|
-
});
|
|
722
|
-
await batch.write();
|
|
723
|
-
this._params.monitor?.recordBytesStored(binary.byteLength);
|
|
724
|
-
await this._params.callbacks?.afterSave?.(keyArray);
|
|
725
|
-
this._params.monitor?.recordStoreDuration(Date.now() - startMs);
|
|
726
|
-
}
|
|
727
|
-
async remove(keyArray) {
|
|
728
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
729
|
-
return void 0;
|
|
730
|
-
}
|
|
731
|
-
await this._params.db.del(keyArray, {
|
|
732
|
-
...encodingOptions
|
|
733
|
-
});
|
|
734
|
-
}
|
|
735
|
-
async loadRange(keyPrefix) {
|
|
736
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
737
|
-
return [];
|
|
738
|
-
}
|
|
739
|
-
const startMs = Date.now();
|
|
740
|
-
const result = [];
|
|
741
|
-
for await (const [key, value] of this._params.db.iterator({
|
|
742
|
-
gte: keyPrefix,
|
|
743
|
-
lte: [
|
|
744
|
-
...keyPrefix,
|
|
745
|
-
"\uFFFF"
|
|
746
|
-
],
|
|
747
|
-
...encodingOptions
|
|
748
|
-
})) {
|
|
749
|
-
result.push({
|
|
750
|
-
key,
|
|
751
|
-
data: value
|
|
752
|
-
});
|
|
753
|
-
this._params.monitor?.recordBytesLoaded(value.byteLength);
|
|
754
|
-
}
|
|
755
|
-
this._params.monitor?.recordLoadDuration(Date.now() - startMs);
|
|
756
|
-
return result;
|
|
757
|
-
}
|
|
758
|
-
async removeRange(keyPrefix) {
|
|
759
|
-
if (this._lifecycleState !== LifecycleState2.OPEN) {
|
|
760
|
-
return void 0;
|
|
761
|
-
}
|
|
762
|
-
const batch = this._params.db.batch();
|
|
763
|
-
for await (const [key] of this._params.db.iterator({
|
|
764
|
-
gte: keyPrefix,
|
|
765
|
-
lte: [
|
|
766
|
-
...keyPrefix,
|
|
767
|
-
"\uFFFF"
|
|
768
|
-
],
|
|
769
|
-
...encodingOptions
|
|
770
|
-
})) {
|
|
771
|
-
batch.del(key, {
|
|
772
|
-
...encodingOptions
|
|
773
|
-
});
|
|
774
|
-
}
|
|
775
|
-
await batch.write();
|
|
776
|
-
}
|
|
777
|
-
};
|
|
778
|
-
var keyEncoder = {
|
|
779
|
-
encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
|
|
780
|
-
decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%")),
|
|
781
|
-
format: "buffer"
|
|
782
|
-
};
|
|
783
|
-
var encodingOptions = {
|
|
784
|
-
keyEncoding: keyEncoder,
|
|
785
|
-
valueEncoding: "buffer"
|
|
786
|
-
};
|
|
787
|
-
var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
|
|
788
|
-
|
|
789
|
-
// packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
|
|
790
|
-
function _ts_decorate3(decorators, target, key, desc) {
|
|
791
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
792
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
793
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
794
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
795
|
-
}
|
|
796
|
-
var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
|
|
797
|
-
var FIND_PARAMS = {
|
|
798
|
-
allowableStates: [
|
|
799
|
-
"ready",
|
|
800
|
-
"requesting"
|
|
801
|
-
]
|
|
802
|
-
};
|
|
803
|
-
var AutomergeHost = class extends Resource3 {
|
|
804
|
-
constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider, getSpaceKeyByRootDocumentId }) {
|
|
805
|
-
super();
|
|
806
|
-
this._collectionSynchronizer = new CollectionSynchronizer({
|
|
807
|
-
queryCollectionState: this._queryCollectionState.bind(this),
|
|
808
|
-
sendCollectionState: this._sendCollectionState.bind(this),
|
|
809
|
-
shouldSyncCollection: this._shouldSyncCollection.bind(this)
|
|
810
|
-
});
|
|
811
|
-
this.collectionStateUpdated = new Event2();
|
|
812
|
-
/**
|
|
813
|
-
* Fired after a batch of documents was saved to disk.
|
|
814
|
-
*/
|
|
815
|
-
this.documentsSaved = new Event2();
|
|
816
|
-
this._db = db;
|
|
817
|
-
this._storage = new LevelDBStorageAdapter({
|
|
818
|
-
db: db.sublevel("automerge"),
|
|
819
|
-
callbacks: {
|
|
820
|
-
beforeSave: async (params) => this._beforeSave(params),
|
|
821
|
-
afterSave: async (key) => this._afterSave(key)
|
|
822
|
-
},
|
|
823
|
-
monitor: dataMonitor
|
|
824
|
-
});
|
|
825
|
-
this._echoNetworkAdapter = new EchoNetworkAdapter({
|
|
826
|
-
getContainingSpaceForDocument: this._getContainingSpaceForDocument.bind(this),
|
|
827
|
-
isDocumentInRemoteCollection: this._isDocumentInRemoteCollection.bind(this),
|
|
828
|
-
onCollectionStateQueried: this._onCollectionStateQueried.bind(this),
|
|
829
|
-
onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
|
|
830
|
-
monitor: dataMonitor
|
|
831
|
-
});
|
|
832
|
-
this._headsStore = new HeadsStore({
|
|
833
|
-
db: db.sublevel("heads")
|
|
834
|
-
});
|
|
835
|
-
this._indexMetadataStore = indexMetadataStore;
|
|
836
|
-
this._peerIdProvider = peerIdProvider;
|
|
837
|
-
this._getSpaceKeyByRootDocumentId = getSpaceKeyByRootDocumentId;
|
|
838
|
-
}
|
|
839
|
-
async _open() {
|
|
840
|
-
this._peerId = `host-${this._peerIdProvider?.() ?? PublicKey.random().toHex()}`;
|
|
841
|
-
await this._storage.open?.();
|
|
842
|
-
this._repo = new Repo({
|
|
843
|
-
peerId: this._peerId,
|
|
844
|
-
sharePolicy: this._sharePolicy.bind(this),
|
|
845
|
-
storage: this._storage,
|
|
846
|
-
network: [
|
|
847
|
-
// Upstream swarm.
|
|
848
|
-
this._echoNetworkAdapter
|
|
849
|
-
]
|
|
850
|
-
});
|
|
851
|
-
let updatingAuthScope = false;
|
|
852
|
-
Event2.wrap(this._echoNetworkAdapter, "peer-candidate").on(this._ctx, (e) => !updatingAuthScope && this._onPeerConnected(e.peerId));
|
|
853
|
-
Event2.wrap(this._echoNetworkAdapter, "peer-disconnected").on(this._ctx, (e) => !updatingAuthScope && this._onPeerDisconnected(e.peerId));
|
|
854
|
-
this._collectionSynchronizer.remoteStateUpdated.on(this._ctx, ({ collectionId, peerId, newDocsAppeared }) => {
|
|
855
|
-
this._onRemoteCollectionStateUpdated(collectionId, peerId);
|
|
856
|
-
this.collectionStateUpdated.emit({
|
|
857
|
-
collectionId
|
|
858
|
-
});
|
|
859
|
-
if (newDocsAppeared) {
|
|
860
|
-
updatingAuthScope = true;
|
|
861
|
-
try {
|
|
862
|
-
this._echoNetworkAdapter.onConnectionAuthScopeChanged(peerId);
|
|
863
|
-
} finally {
|
|
864
|
-
updatingAuthScope = false;
|
|
865
|
-
}
|
|
866
|
-
}
|
|
867
|
-
});
|
|
868
|
-
await this._echoNetworkAdapter.open();
|
|
869
|
-
await this._collectionSynchronizer.open();
|
|
870
|
-
await this._echoNetworkAdapter.open();
|
|
871
|
-
await this._echoNetworkAdapter.whenConnected();
|
|
872
|
-
}
|
|
873
|
-
async _close() {
|
|
874
|
-
await this._collectionSynchronizer.close();
|
|
875
|
-
await this._storage.close?.();
|
|
876
|
-
await this._echoNetworkAdapter.close();
|
|
877
|
-
await this._ctx.dispose();
|
|
878
|
-
}
|
|
879
|
-
/**
|
|
880
|
-
* @deprecated To be abstracted away.
|
|
881
|
-
*/
|
|
882
|
-
get repo() {
|
|
883
|
-
return this._repo;
|
|
884
|
-
}
|
|
885
|
-
get peerId() {
|
|
886
|
-
return this._peerId;
|
|
887
|
-
}
|
|
888
|
-
get loadedDocsCount() {
|
|
889
|
-
return Object.keys(this._repo.handles).length;
|
|
890
|
-
}
|
|
891
|
-
async addReplicator(replicator) {
|
|
892
|
-
await this._echoNetworkAdapter.addReplicator(replicator);
|
|
893
|
-
}
|
|
894
|
-
async removeReplicator(replicator) {
|
|
895
|
-
await this._echoNetworkAdapter.removeReplicator(replicator);
|
|
896
|
-
}
|
|
897
|
-
/**
|
|
898
|
-
* Loads the document handle from the repo and waits for it to be ready.
|
|
899
|
-
*/
|
|
900
|
-
async loadDoc(ctx, documentId, opts) {
|
|
901
|
-
let handle;
|
|
902
|
-
if (typeof documentId === "string") {
|
|
903
|
-
handle = this._repo.handles[documentId];
|
|
904
|
-
}
|
|
905
|
-
if (!handle) {
|
|
906
|
-
handle = await this._repo.find(documentId, FIND_PARAMS);
|
|
907
|
-
}
|
|
908
|
-
if (!handle.isReady()) {
|
|
909
|
-
if (!opts?.timeout) {
|
|
910
|
-
await cancelWithContext(ctx, handle.whenReady());
|
|
911
|
-
} else {
|
|
912
|
-
await cancelWithContext(ctx, asyncTimeout(handle.whenReady(), opts.timeout));
|
|
913
|
-
}
|
|
914
|
-
}
|
|
915
|
-
return handle;
|
|
916
|
-
}
|
|
917
|
-
async exportDoc(ctx, id) {
|
|
918
|
-
const documentId = interpretAsDocumentId(id);
|
|
919
|
-
const chunks = await this._storage.loadRange([
|
|
920
|
-
documentId
|
|
921
|
-
]);
|
|
922
|
-
return bufferToArray(Buffer.concat(chunks.map((c) => c.data)));
|
|
923
|
-
}
|
|
924
|
-
/**
|
|
925
|
-
* Create new persisted document.
|
|
926
|
-
*/
|
|
927
|
-
createDoc(initialValue, opts) {
|
|
928
|
-
if (opts?.preserveHistory) {
|
|
929
|
-
if (initialValue instanceof Uint8Array) {
|
|
930
|
-
return this._repo.import(initialValue);
|
|
931
|
-
}
|
|
932
|
-
if (!isAutomerge(initialValue)) {
|
|
933
|
-
throw new TypeError("Initial value must be an Automerge document");
|
|
934
|
-
}
|
|
935
|
-
return this._repo.import(save(initialValue));
|
|
936
|
-
} else {
|
|
937
|
-
if (initialValue instanceof Uint8Array) {
|
|
938
|
-
throw new Error("Cannot create document from Uint8Array without preserving history");
|
|
939
|
-
}
|
|
940
|
-
return this._repo.create(initialValue);
|
|
941
|
-
}
|
|
942
|
-
}
|
|
943
|
-
async waitUntilHeadsReplicated(heads) {
|
|
944
|
-
const entries = heads.entries;
|
|
945
|
-
if (!entries?.length) {
|
|
946
|
-
return;
|
|
947
|
-
}
|
|
948
|
-
const documentIds = entries.map((entry) => entry.documentId);
|
|
949
|
-
const documentHeads = await this.getHeads(documentIds);
|
|
950
|
-
const headsToWait = entries.filter((entry, index) => {
|
|
951
|
-
const targetHeads = entry.heads;
|
|
952
|
-
if (!targetHeads || targetHeads.length === 0) {
|
|
953
|
-
return false;
|
|
954
|
-
}
|
|
955
|
-
const currentHeads = documentHeads[index];
|
|
956
|
-
return !(currentHeads !== null && headsEquals(currentHeads, targetHeads));
|
|
957
|
-
});
|
|
958
|
-
if (headsToWait.length > 0) {
|
|
959
|
-
await Promise.all(headsToWait.map(async (entry, index) => {
|
|
960
|
-
const handle = await this.loadDoc(Context.default(void 0, {
|
|
961
|
-
F: __dxlog_file3,
|
|
962
|
-
L: 293
|
|
963
|
-
}), entry.documentId);
|
|
964
|
-
await waitForHeads(handle, entry.heads);
|
|
965
|
-
}));
|
|
966
|
-
}
|
|
967
|
-
await this._repo.flush(documentIds.filter((documentId) => this._repo.handles[documentId] && this._repo.handles[documentId].isReady()));
|
|
968
|
-
}
|
|
969
|
-
async reIndexHeads(documentIds) {
|
|
970
|
-
for (const documentId of documentIds) {
|
|
971
|
-
log3("re-indexing heads for document", {
|
|
972
|
-
documentId
|
|
973
|
-
}, {
|
|
974
|
-
F: __dxlog_file3,
|
|
975
|
-
L: 307,
|
|
976
|
-
S: this,
|
|
977
|
-
C: (f, a) => f(...a)
|
|
978
|
-
});
|
|
979
|
-
const handle = await this._repo.find(documentId, FIND_PARAMS);
|
|
980
|
-
if (!handle.isReady()) {
|
|
981
|
-
log3.warn("document is not available locally, skipping", {
|
|
982
|
-
documentId
|
|
983
|
-
}, {
|
|
984
|
-
F: __dxlog_file3,
|
|
985
|
-
L: 310,
|
|
986
|
-
S: this,
|
|
987
|
-
C: (f, a) => f(...a)
|
|
988
|
-
});
|
|
989
|
-
continue;
|
|
990
|
-
}
|
|
991
|
-
const heads = handle.heads();
|
|
992
|
-
const batch = this._db.batch();
|
|
993
|
-
this._headsStore.setHeads(documentId, heads, batch);
|
|
994
|
-
await batch.write();
|
|
995
|
-
}
|
|
996
|
-
log3("done re-indexing heads", void 0, {
|
|
997
|
-
F: __dxlog_file3,
|
|
998
|
-
L: 319,
|
|
999
|
-
S: this,
|
|
1000
|
-
C: (f, a) => f(...a)
|
|
1001
|
-
});
|
|
1002
|
-
}
|
|
1003
|
-
// TODO(dmaretskyi): Share based on HALO permissions and space affinity.
|
|
1004
|
-
// Hosts, running in the worker, don't share documents unless requested by other peers.
|
|
1005
|
-
// NOTE: If both peers return sharePolicy=false the replication will not happen
|
|
1006
|
-
// https://github.com/automerge/automerge-repo/pull/292
|
|
1007
|
-
async _sharePolicy(peerId, documentId) {
|
|
1008
|
-
if (peerId.startsWith("client-")) {
|
|
1009
|
-
return false;
|
|
1010
|
-
}
|
|
1011
|
-
if (!documentId) {
|
|
1012
|
-
return false;
|
|
1013
|
-
}
|
|
1014
|
-
const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
|
|
1015
|
-
if (isEchoPeerMetadata(peerMetadata)) {
|
|
1016
|
-
return this._echoNetworkAdapter.shouldAdvertise(peerId, {
|
|
1017
|
-
documentId
|
|
1018
|
-
});
|
|
1019
|
-
}
|
|
1020
|
-
return false;
|
|
1021
|
-
}
|
|
1022
|
-
async _beforeSave({ path, batch }) {
|
|
1023
|
-
const handle = this._repo.handles[path[0]];
|
|
1024
|
-
if (!handle || !handle.isReady()) {
|
|
1025
|
-
return;
|
|
1026
|
-
}
|
|
1027
|
-
const doc = handle.doc();
|
|
1028
|
-
if (!doc) {
|
|
1029
|
-
return;
|
|
1030
|
-
}
|
|
1031
|
-
const heads = getHeads(doc);
|
|
1032
|
-
this._headsStore.setHeads(handle.documentId, heads, batch);
|
|
1033
|
-
const spaceKey = DatabaseDirectory.getSpaceKey(doc) ?? void 0;
|
|
1034
|
-
const objectIds = Object.keys(doc.objects ?? {});
|
|
1035
|
-
const encodedIds = objectIds.map((objectId) => objectPointerCodec.encode({
|
|
1036
|
-
documentId: handle.documentId,
|
|
1037
|
-
objectId,
|
|
1038
|
-
spaceKey
|
|
1039
|
-
}));
|
|
1040
|
-
const idToLastHash = new Map(encodedIds.map((id) => [
|
|
1041
|
-
id,
|
|
1042
|
-
heads
|
|
1043
|
-
]));
|
|
1044
|
-
this._indexMetadataStore.markDirty(idToLastHash, batch);
|
|
1045
|
-
}
|
|
1046
|
-
_shouldSyncCollection(collectionId, peerId) {
|
|
1047
|
-
const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
|
|
1048
|
-
if (isEchoPeerMetadata(peerMetadata)) {
|
|
1049
|
-
return this._echoNetworkAdapter.shouldSyncCollection(peerId, {
|
|
1050
|
-
collectionId
|
|
1051
|
-
});
|
|
1052
|
-
}
|
|
1053
|
-
return false;
|
|
1054
|
-
}
|
|
1055
|
-
/**
|
|
1056
|
-
* Called by AutomergeStorageAdapter after levelDB batch commit.
|
|
1057
|
-
*/
|
|
1058
|
-
async _afterSave(path) {
|
|
1059
|
-
this._indexMetadataStore.notifyMarkedDirty();
|
|
1060
|
-
const documentId = path[0];
|
|
1061
|
-
const document = this._repo.handles[documentId]?.doc();
|
|
1062
|
-
if (document) {
|
|
1063
|
-
const heads = getHeads(document);
|
|
1064
|
-
this._onHeadsChanged(documentId, heads);
|
|
1065
|
-
}
|
|
1066
|
-
this.documentsSaved.emit();
|
|
1067
|
-
}
|
|
1068
|
-
_automergePeers() {
|
|
1069
|
-
return this._repo.peers;
|
|
1070
|
-
}
|
|
1071
|
-
async _isDocumentInRemoteCollection(params) {
|
|
1072
|
-
for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
|
|
1073
|
-
const remoteCollections = this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
|
|
1074
|
-
const remotePeerDocs = remoteCollections.get(params.peerId)?.documents;
|
|
1075
|
-
if (remotePeerDocs && params.documentId in remotePeerDocs) {
|
|
1076
|
-
return true;
|
|
1077
|
-
}
|
|
1078
|
-
}
|
|
1079
|
-
return false;
|
|
1080
|
-
}
|
|
1081
|
-
async _getContainingSpaceForDocument(documentId) {
|
|
1082
|
-
const handle = this._repo.handles[documentId];
|
|
1083
|
-
if (handle.state === "loading") {
|
|
1084
|
-
await handle.whenReady();
|
|
1085
|
-
}
|
|
1086
|
-
if (handle && handle.isReady() && handle.doc()) {
|
|
1087
|
-
const spaceKeyHex = DatabaseDirectory.getSpaceKey(handle.doc());
|
|
1088
|
-
if (spaceKeyHex) {
|
|
1089
|
-
return PublicKey.from(spaceKeyHex);
|
|
1090
|
-
}
|
|
1091
|
-
}
|
|
1092
|
-
const rootDocSpaceKey = this._getSpaceKeyByRootDocumentId?.(documentId);
|
|
1093
|
-
if (rootDocSpaceKey) {
|
|
1094
|
-
return rootDocSpaceKey;
|
|
1095
|
-
}
|
|
1096
|
-
return null;
|
|
1097
|
-
}
|
|
1098
|
-
/**
|
|
1099
|
-
* Flush documents to disk.
|
|
1100
|
-
*/
|
|
1101
|
-
async flush({ documentIds } = {}) {
|
|
1102
|
-
const loadedDocuments = documentIds?.filter((documentId) => {
|
|
1103
|
-
const handle = this._repo.handles[documentId];
|
|
1104
|
-
return handle && handle.isReady();
|
|
1105
|
-
});
|
|
1106
|
-
await this._repo.flush(loadedDocuments);
|
|
1107
|
-
}
|
|
1108
|
-
async getHeads(documentIds) {
|
|
1109
|
-
const result = [];
|
|
1110
|
-
const storeRequestIds = [];
|
|
1111
|
-
const storeResultIndices = [];
|
|
1112
|
-
for (const documentId of documentIds) {
|
|
1113
|
-
const handle = this._repo.handles[documentId];
|
|
1114
|
-
if (handle && handle.isReady() && handle.doc()) {
|
|
1115
|
-
result.push(getHeads(handle.doc()));
|
|
1116
|
-
} else {
|
|
1117
|
-
storeRequestIds.push(documentId);
|
|
1118
|
-
storeResultIndices.push(result.length);
|
|
1119
|
-
result.push(void 0);
|
|
1120
|
-
}
|
|
1121
|
-
}
|
|
1122
|
-
if (storeRequestIds.length > 0) {
|
|
1123
|
-
const storedHeads = await this._headsStore.getHeads(storeRequestIds);
|
|
1124
|
-
for (let i = 0; i < storedHeads.length; i++) {
|
|
1125
|
-
result[storeResultIndices[i]] = storedHeads[i];
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
return result;
|
|
1129
|
-
}
|
|
1130
|
-
//
|
|
1131
|
-
// Collection sync.
|
|
1132
|
-
//
|
|
1133
|
-
getLocalCollectionState(collectionId) {
|
|
1134
|
-
return this._collectionSynchronizer.getLocalCollectionState(collectionId);
|
|
1135
|
-
}
|
|
1136
|
-
getRemoteCollectionStates(collectionId) {
|
|
1137
|
-
return this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
|
|
1138
|
-
}
|
|
1139
|
-
refreshCollection(collectionId) {
|
|
1140
|
-
this._collectionSynchronizer.refreshCollection(collectionId);
|
|
1141
|
-
}
|
|
1142
|
-
async getCollectionSyncState(collectionId) {
|
|
1143
|
-
const result = {
|
|
1144
|
-
peers: []
|
|
1145
|
-
};
|
|
1146
|
-
const localState = this.getLocalCollectionState(collectionId);
|
|
1147
|
-
const remoteState = this.getRemoteCollectionStates(collectionId);
|
|
1148
|
-
if (!localState) {
|
|
1149
|
-
return result;
|
|
1150
|
-
}
|
|
1151
|
-
for (const [peerId, state] of remoteState) {
|
|
1152
|
-
const diff = diffCollectionState(localState, state);
|
|
1153
|
-
result.peers.push({
|
|
1154
|
-
peerId,
|
|
1155
|
-
missingOnRemote: diff.missingOnRemote.length,
|
|
1156
|
-
missingOnLocal: diff.missingOnLocal.length,
|
|
1157
|
-
differentDocuments: diff.different.length,
|
|
1158
|
-
localDocumentCount: Object.keys(localState.documents).length,
|
|
1159
|
-
remoteDocumentCount: Object.keys(state.documents).length
|
|
1160
|
-
});
|
|
1161
|
-
}
|
|
1162
|
-
return result;
|
|
1163
|
-
}
|
|
1164
|
-
/**
|
|
1165
|
-
* Update the local collection state based on the locally stored document heads.
|
|
1166
|
-
*/
|
|
1167
|
-
async updateLocalCollectionState(collectionId, documentIds) {
|
|
1168
|
-
const heads = await this.getHeads(documentIds);
|
|
1169
|
-
const documents = Object.fromEntries(heads.map((heads2, index) => [
|
|
1170
|
-
documentIds[index],
|
|
1171
|
-
heads2 ?? []
|
|
1172
|
-
]));
|
|
1173
|
-
this._collectionSynchronizer.setLocalCollectionState(collectionId, {
|
|
1174
|
-
documents
|
|
1175
|
-
});
|
|
1176
|
-
}
|
|
1177
|
-
async clearLocalCollectionState(collectionId) {
|
|
1178
|
-
this._collectionSynchronizer.clearLocalCollectionState(collectionId);
|
|
1179
|
-
}
|
|
1180
|
-
_onCollectionStateQueried(collectionId, peerId) {
|
|
1181
|
-
this._collectionSynchronizer.onCollectionStateQueried(collectionId, peerId);
|
|
1182
|
-
}
|
|
1183
|
-
_onCollectionStateReceived(collectionId, peerId, state) {
|
|
1184
|
-
this._collectionSynchronizer.onRemoteStateReceived(collectionId, peerId, decodeCollectionState(state));
|
|
1185
|
-
}
|
|
1186
|
-
_queryCollectionState(collectionId, peerId) {
|
|
1187
|
-
this._echoNetworkAdapter.queryCollectionState(collectionId, peerId);
|
|
1188
|
-
}
|
|
1189
|
-
_sendCollectionState(collectionId, peerId, state) {
|
|
1190
|
-
this._echoNetworkAdapter.sendCollectionState(collectionId, peerId, encodeCollectionState(state));
|
|
1191
|
-
}
|
|
1192
|
-
_onPeerConnected(peerId) {
|
|
1193
|
-
this._collectionSynchronizer.onConnectionOpen(peerId);
|
|
1194
|
-
}
|
|
1195
|
-
_onPeerDisconnected(peerId) {
|
|
1196
|
-
this._collectionSynchronizer.onConnectionClosed(peerId);
|
|
1197
|
-
}
|
|
1198
|
-
_onRemoteCollectionStateUpdated(collectionId, peerId) {
|
|
1199
|
-
const localState = this._collectionSynchronizer.getLocalCollectionState(collectionId);
|
|
1200
|
-
const remoteState = this._collectionSynchronizer.getRemoteCollectionStates(collectionId).get(peerId);
|
|
1201
|
-
if (!localState || !remoteState) {
|
|
1202
|
-
return;
|
|
1203
|
-
}
|
|
1204
|
-
const { different, missingOnLocal, missingOnRemote } = diffCollectionState(localState, remoteState);
|
|
1205
|
-
const toReplicate = [
|
|
1206
|
-
...missingOnLocal,
|
|
1207
|
-
...missingOnRemote,
|
|
1208
|
-
...different
|
|
1209
|
-
];
|
|
1210
|
-
if (toReplicate.length === 0) {
|
|
1211
|
-
return;
|
|
1212
|
-
}
|
|
1213
|
-
log3("replicating documents after collection sync", {
|
|
1214
|
-
collectionId,
|
|
1215
|
-
peerId,
|
|
1216
|
-
toReplicate,
|
|
1217
|
-
count: toReplicate.length
|
|
1218
|
-
}, {
|
|
1219
|
-
F: __dxlog_file3,
|
|
1220
|
-
L: 563,
|
|
1221
|
-
S: this,
|
|
1222
|
-
C: (f, a) => f(...a)
|
|
1223
|
-
});
|
|
1224
|
-
for (const documentId of toReplicate) {
|
|
1225
|
-
this._repo.findWithProgress(documentId);
|
|
1226
|
-
}
|
|
1227
|
-
}
|
|
1228
|
-
_onHeadsChanged(documentId, heads) {
|
|
1229
|
-
const collectionsChanged = /* @__PURE__ */ new Set();
|
|
1230
|
-
for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
|
|
1231
|
-
const state = this._collectionSynchronizer.getLocalCollectionState(collectionId);
|
|
1232
|
-
if (state?.documents[documentId]) {
|
|
1233
|
-
const newState = structuredClone(state);
|
|
1234
|
-
newState.documents[documentId] = heads;
|
|
1235
|
-
this._collectionSynchronizer.setLocalCollectionState(collectionId, newState);
|
|
1236
|
-
collectionsChanged.add(collectionId);
|
|
1237
|
-
}
|
|
1238
|
-
}
|
|
1239
|
-
for (const collectionId of collectionsChanged) {
|
|
1240
|
-
this.collectionStateUpdated.emit({
|
|
1241
|
-
collectionId
|
|
1242
|
-
});
|
|
1243
|
-
}
|
|
1244
|
-
}
|
|
1245
|
-
};
|
|
1246
|
-
_ts_decorate3([
|
|
1247
|
-
trace2.info()
|
|
1248
|
-
], AutomergeHost.prototype, "_peerId", void 0);
|
|
1249
|
-
_ts_decorate3([
|
|
1250
|
-
trace2.info({
|
|
1251
|
-
depth: null
|
|
1252
|
-
})
|
|
1253
|
-
], AutomergeHost.prototype, "_automergePeers", null);
|
|
1254
|
-
_ts_decorate3([
|
|
1255
|
-
trace2.span({
|
|
1256
|
-
showInBrowserTimeline: true
|
|
1257
|
-
})
|
|
1258
|
-
], AutomergeHost.prototype, "flush", null);
|
|
1259
|
-
AutomergeHost = _ts_decorate3([
|
|
1260
|
-
trace2.resource()
|
|
1261
|
-
], AutomergeHost);
|
|
1262
|
-
var waitForHeads = async (handle, heads) => {
|
|
1263
|
-
const unavailableHeads = new Set(heads);
|
|
1264
|
-
await handle.whenReady();
|
|
1265
|
-
await Event2.wrap(handle, "change").waitForCondition(() => {
|
|
1266
|
-
for (const changeHash of unavailableHeads.values()) {
|
|
1267
|
-
if (changeIsPresentInDoc(handle.doc(), changeHash)) {
|
|
1268
|
-
unavailableHeads.delete(changeHash);
|
|
1269
|
-
}
|
|
1270
|
-
}
|
|
1271
|
-
return unavailableHeads.size === 0;
|
|
1272
|
-
});
|
|
1273
|
-
};
|
|
1274
|
-
var changeIsPresentInDoc = (doc, changeHash) => {
|
|
1275
|
-
return !!getBackend(doc).getChangeByHash(changeHash);
|
|
1276
|
-
};
|
|
1277
|
-
var decodeCollectionState = (state) => {
|
|
1278
|
-
invariant2(typeof state === "object" && state !== null, "Invalid state", {
|
|
1279
|
-
F: __dxlog_file3,
|
|
1280
|
-
L: 614,
|
|
1281
|
-
S: void 0,
|
|
1282
|
-
A: [
|
|
1283
|
-
"typeof state === 'object' && state !== null",
|
|
1284
|
-
"'Invalid state'"
|
|
1285
|
-
]
|
|
1286
|
-
});
|
|
1287
|
-
return state;
|
|
1288
|
-
};
|
|
1289
|
-
var encodeCollectionState = (state) => {
|
|
1290
|
-
return state;
|
|
1291
|
-
};
|
|
1292
|
-
|
|
1293
|
-
// packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts
|
|
1294
|
-
import { invariant as invariant5 } from "@dxos/invariant";
|
|
1295
|
-
import { PublicKey as PublicKey2 } from "@dxos/keys";
|
|
1296
|
-
import { log as log5 } from "@dxos/log";
|
|
1297
|
-
import { ComplexSet, defaultMap as defaultMap2 } from "@dxos/util";
|
|
1298
|
-
|
|
1299
|
-
// packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts
|
|
1300
|
-
import * as A from "@automerge/automerge";
|
|
1301
|
-
import { cbor } from "@automerge/automerge-repo";
|
|
1302
|
-
import { Resource as Resource4 } from "@dxos/context";
|
|
1303
|
-
import { invariant as invariant3 } from "@dxos/invariant";
|
|
1304
|
-
import { log as log4 } from "@dxos/log";
|
|
1305
|
-
import { AutomergeReplicator } from "@dxos/teleport-extension-automerge-replicator";
|
|
1306
|
-
var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator-connection.ts";
|
|
1307
|
-
var DEFAULT_FACTORY = (params) => new AutomergeReplicator(...params);
|
|
1308
|
-
var MeshReplicatorConnection = class extends Resource4 {
|
|
1309
|
-
constructor(_params) {
|
|
1310
|
-
super(), this._params = _params, this.remoteDeviceKey = null, this._remotePeerId = null, this._isEnabled = false;
|
|
1311
|
-
let readableStreamController;
|
|
1312
|
-
this.readable = new ReadableStream({
|
|
1313
|
-
start: (controller) => {
|
|
1314
|
-
readableStreamController = controller;
|
|
1315
|
-
this._ctx.onDispose(() => controller.close());
|
|
1316
|
-
}
|
|
1317
|
-
});
|
|
1318
|
-
this.writable = new WritableStream({
|
|
1319
|
-
write: async (message, controller) => {
|
|
1320
|
-
invariant3(this._isEnabled, "Writing to a disabled connection", {
|
|
1321
|
-
F: __dxlog_file4,
|
|
1322
|
-
L: 51,
|
|
1323
|
-
S: this,
|
|
1324
|
-
A: [
|
|
1325
|
-
"this._isEnabled",
|
|
1326
|
-
"'Writing to a disabled connection'"
|
|
1327
|
-
]
|
|
1328
|
-
});
|
|
1329
|
-
try {
|
|
1330
|
-
logSendSync(message);
|
|
1331
|
-
await this.replicatorExtension.sendSyncMessage({
|
|
1332
|
-
payload: cbor.encode(message)
|
|
1333
|
-
});
|
|
1334
|
-
} catch (err) {
|
|
1335
|
-
controller.error(err);
|
|
1336
|
-
this._disconnectIfEnabled();
|
|
1337
|
-
}
|
|
1338
|
-
}
|
|
1339
|
-
});
|
|
1340
|
-
const createAutomergeReplicator = this._params.replicatorFactory ?? DEFAULT_FACTORY;
|
|
1341
|
-
this.replicatorExtension = createAutomergeReplicator([
|
|
1342
|
-
{
|
|
1343
|
-
peerId: this._params.ownPeerId
|
|
1344
|
-
},
|
|
1345
|
-
{
|
|
1346
|
-
onStartReplication: async (info, remotePeerId) => {
|
|
1347
|
-
this.remoteDeviceKey = remotePeerId;
|
|
1348
|
-
this._remotePeerId = info.id;
|
|
1349
|
-
log4("onStartReplication", {
|
|
1350
|
-
id: info.id,
|
|
1351
|
-
thisPeerId: this.peerId,
|
|
1352
|
-
remotePeerId: remotePeerId.toHex()
|
|
1353
|
-
}, {
|
|
1354
|
-
F: __dxlog_file4,
|
|
1355
|
-
L: 80,
|
|
1356
|
-
S: this,
|
|
1357
|
-
C: (f, a) => f(...a)
|
|
1358
|
-
});
|
|
1359
|
-
this._params.onRemoteConnected();
|
|
1360
|
-
},
|
|
1361
|
-
onSyncMessage: async ({ payload }) => {
|
|
1362
|
-
if (!this._isEnabled) {
|
|
1363
|
-
return;
|
|
1364
|
-
}
|
|
1365
|
-
const message = cbor.decode(payload);
|
|
1366
|
-
readableStreamController.enqueue(message);
|
|
1367
|
-
},
|
|
1368
|
-
onClose: async () => {
|
|
1369
|
-
this._disconnectIfEnabled();
|
|
1370
|
-
}
|
|
1371
|
-
}
|
|
1372
|
-
]);
|
|
1373
|
-
}
|
|
1374
|
-
_disconnectIfEnabled() {
|
|
1375
|
-
if (this._isEnabled) {
|
|
1376
|
-
this._params.onRemoteDisconnected();
|
|
1377
|
-
}
|
|
1378
|
-
}
|
|
1379
|
-
get peerId() {
|
|
1380
|
-
invariant3(this._remotePeerId != null, "Remote peer has not connected yet.", {
|
|
1381
|
-
F: __dxlog_file4,
|
|
1382
|
-
L: 106,
|
|
1383
|
-
S: this,
|
|
1384
|
-
A: [
|
|
1385
|
-
"this._remotePeerId != null",
|
|
1386
|
-
"'Remote peer has not connected yet.'"
|
|
1387
|
-
]
|
|
1388
|
-
});
|
|
1389
|
-
return this._remotePeerId;
|
|
1390
|
-
}
|
|
1391
|
-
get isEnabled() {
|
|
1392
|
-
return this._isEnabled;
|
|
1393
|
-
}
|
|
1394
|
-
async shouldAdvertise(params) {
|
|
1395
|
-
return this._params.shouldAdvertise(params);
|
|
1396
|
-
}
|
|
1397
|
-
shouldSyncCollection(params) {
|
|
1398
|
-
return this._params.shouldSyncCollection(params);
|
|
1399
|
-
}
|
|
1400
|
-
/**
|
|
1401
|
-
* Start exchanging messages with the remote peer.
|
|
1402
|
-
* Call after the remote peer has connected.
|
|
1403
|
-
*/
|
|
1404
|
-
enable() {
|
|
1405
|
-
invariant3(this._remotePeerId != null, "Remote peer has not connected yet.", {
|
|
1406
|
-
F: __dxlog_file4,
|
|
1407
|
-
L: 127,
|
|
1408
|
-
S: this,
|
|
1409
|
-
A: [
|
|
1410
|
-
"this._remotePeerId != null",
|
|
1411
|
-
"'Remote peer has not connected yet.'"
|
|
1412
|
-
]
|
|
1413
|
-
});
|
|
1414
|
-
this._isEnabled = true;
|
|
1415
|
-
}
|
|
1416
|
-
/**
|
|
1417
|
-
* Stop exchanging messages with the remote peer.
|
|
1418
|
-
*/
|
|
1419
|
-
disable() {
|
|
1420
|
-
this._isEnabled = false;
|
|
1421
|
-
}
|
|
1422
|
-
};
|
|
1423
|
-
var logSendSync = (message) => {
|
|
1424
|
-
log4("sendSyncMessage", () => {
|
|
1425
|
-
const decodedSyncMessage = message.type === "sync" && message.data ? A.decodeSyncMessage(message.data) : void 0;
|
|
1426
|
-
return {
|
|
1427
|
-
sync: decodedSyncMessage && {
|
|
1428
|
-
headsLength: decodedSyncMessage.heads.length,
|
|
1429
|
-
requesting: decodedSyncMessage.need.length > 0,
|
|
1430
|
-
sendingChanges: decodedSyncMessage.changes.length > 0
|
|
1431
|
-
},
|
|
1432
|
-
type: message.type,
|
|
1433
|
-
from: message.senderId,
|
|
1434
|
-
to: message.targetId
|
|
1435
|
-
};
|
|
1436
|
-
}, {
|
|
1437
|
-
F: __dxlog_file4,
|
|
1438
|
-
L: 140,
|
|
1439
|
-
S: void 0,
|
|
1440
|
-
C: (f, a) => f(...a)
|
|
1441
|
-
});
|
|
1442
|
-
};
|
|
1443
|
-
|
|
1444
|
-
// packages/core/echo/echo-pipeline/src/automerge/space-collection.ts
|
|
1445
|
-
import { invariant as invariant4 } from "@dxos/invariant";
|
|
1446
|
-
import { SpaceId } from "@dxos/keys";
|
|
1447
|
-
var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/space-collection.ts";
|
|
1448
|
-
var deriveCollectionIdFromSpaceId = (spaceId, rootDocumentId) => rootDocumentId ? `space:${spaceId}:${rootDocumentId}` : `space:${spaceId}`;
|
|
1449
|
-
var getSpaceIdFromCollectionId = (collectionId) => {
|
|
1450
|
-
const spaceId = collectionId.split(":")[1];
|
|
1451
|
-
invariant4(SpaceId.isValid(spaceId), void 0, {
|
|
1452
|
-
F: __dxlog_file5,
|
|
1453
|
-
L: 16,
|
|
1454
|
-
S: void 0,
|
|
1455
|
-
A: [
|
|
1456
|
-
"SpaceId.isValid(spaceId)",
|
|
1457
|
-
""
|
|
1458
|
-
]
|
|
1459
|
-
});
|
|
1460
|
-
return spaceId;
|
|
1461
|
-
};
|
|
1462
|
-
|
|
1463
|
-
// packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts
|
|
1464
|
-
var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-echo-replicator.ts";
|
|
1465
|
-
var MeshEchoReplicator = class {
|
|
1466
|
-
constructor() {
|
|
1467
|
-
/**
|
|
1468
|
-
* We might have multiple connections open with a peer (one per space), but there'll be only one enabled
|
|
1469
|
-
* connection at any given moment, because there's a single repo for all the spaces.
|
|
1470
|
-
* When a connection closes (space was closed) it gets removed from the list and the next connection
|
|
1471
|
-
* in the line gets enabled.
|
|
1472
|
-
*/
|
|
1473
|
-
this._connectionsPerPeer = /* @__PURE__ */ new Map();
|
|
1474
|
-
/**
|
|
1475
|
-
* A set of all connections (enabled and disabled).
|
|
1476
|
-
*/
|
|
1477
|
-
this._connections = /* @__PURE__ */ new Set();
|
|
1478
|
-
/**
|
|
1479
|
-
* spaceId -> deviceKey[]
|
|
1480
|
-
*/
|
|
1481
|
-
this._authorizedDevices = /* @__PURE__ */ new Map();
|
|
1482
|
-
this._context = null;
|
|
1483
|
-
}
|
|
1484
|
-
async connect(context) {
|
|
1485
|
-
this._context = context;
|
|
1486
|
-
}
|
|
1487
|
-
async disconnect() {
|
|
1488
|
-
for (const connection of this._connections) {
|
|
1489
|
-
if (connection.isEnabled) {
|
|
1490
|
-
this._context?.onConnectionClosed(connection);
|
|
1491
|
-
}
|
|
1492
|
-
}
|
|
1493
|
-
for (const connection of this._connections) {
|
|
1494
|
-
await connection.close();
|
|
1495
|
-
}
|
|
1496
|
-
this._connections.clear();
|
|
1497
|
-
this._connectionsPerPeer.clear();
|
|
1498
|
-
this._context = null;
|
|
1499
|
-
}
|
|
1500
|
-
createExtension(extensionFactory) {
|
|
1501
|
-
invariant5(this._context, void 0, {
|
|
1502
|
-
F: __dxlog_file6,
|
|
1503
|
-
L: 67,
|
|
1504
|
-
S: this,
|
|
1505
|
-
A: [
|
|
1506
|
-
"this._context",
|
|
1507
|
-
""
|
|
1508
|
-
]
|
|
1509
|
-
});
|
|
1510
|
-
const connection = new MeshReplicatorConnection({
|
|
1511
|
-
ownPeerId: this._context.peerId,
|
|
1512
|
-
replicatorFactory: extensionFactory,
|
|
1513
|
-
onRemoteConnected: async () => {
|
|
1514
|
-
log5("onRemoteConnected", {
|
|
1515
|
-
peerId: connection.peerId
|
|
1516
|
-
}, {
|
|
1517
|
-
F: __dxlog_file6,
|
|
1518
|
-
L: 73,
|
|
1519
|
-
S: this,
|
|
1520
|
-
C: (f, a) => f(...a)
|
|
1521
|
-
});
|
|
1522
|
-
invariant5(this._context, void 0, {
|
|
1523
|
-
F: __dxlog_file6,
|
|
1524
|
-
L: 74,
|
|
1525
|
-
S: this,
|
|
1526
|
-
A: [
|
|
1527
|
-
"this._context",
|
|
1528
|
-
""
|
|
1529
|
-
]
|
|
1530
|
-
});
|
|
1531
|
-
const existingConnections = this._connectionsPerPeer.get(connection.peerId);
|
|
1532
|
-
if (existingConnections?.length) {
|
|
1533
|
-
const enabledConnection = existingConnections[0];
|
|
1534
|
-
this._context.onConnectionAuthScopeChanged(enabledConnection);
|
|
1535
|
-
existingConnections.push(connection);
|
|
1536
|
-
} else {
|
|
1537
|
-
this._connectionsPerPeer.set(connection.peerId, [
|
|
1538
|
-
connection
|
|
1539
|
-
]);
|
|
1540
|
-
this._context.onConnectionOpen(connection);
|
|
1541
|
-
connection.enable();
|
|
1542
|
-
}
|
|
1543
|
-
},
|
|
1544
|
-
onRemoteDisconnected: async () => {
|
|
1545
|
-
log5("onRemoteDisconnected", {
|
|
1546
|
-
peerId: connection.peerId
|
|
1547
|
-
}, {
|
|
1548
|
-
F: __dxlog_file6,
|
|
1549
|
-
L: 88,
|
|
1550
|
-
S: this,
|
|
1551
|
-
C: (f, a) => f(...a)
|
|
1552
|
-
});
|
|
1553
|
-
this._connections.delete(connection);
|
|
1554
|
-
const existingConnections = this._connectionsPerPeer.get(connection.peerId) ?? [];
|
|
1555
|
-
const index = existingConnections.indexOf(connection);
|
|
1556
|
-
if (index < 0) {
|
|
1557
|
-
log5.warn("disconnected connection not found", {
|
|
1558
|
-
peerId: connection.peerId
|
|
1559
|
-
}, {
|
|
1560
|
-
F: __dxlog_file6,
|
|
1561
|
-
L: 96,
|
|
1562
|
-
S: this,
|
|
1563
|
-
C: (f, a) => f(...a)
|
|
1564
|
-
});
|
|
1565
|
-
return;
|
|
1566
|
-
}
|
|
1567
|
-
existingConnections.splice(index, 1);
|
|
1568
|
-
if (connection.isEnabled) {
|
|
1569
|
-
this._context?.onConnectionClosed(connection);
|
|
1570
|
-
connection.disable();
|
|
1571
|
-
if (existingConnections.length > 0) {
|
|
1572
|
-
this._context?.onConnectionOpen(existingConnections[0]);
|
|
1573
|
-
existingConnections[0].enable();
|
|
1574
|
-
}
|
|
1575
|
-
}
|
|
1576
|
-
},
|
|
1577
|
-
shouldAdvertise: async (params) => {
|
|
1578
|
-
log5("shouldAdvertise", {
|
|
1579
|
-
peerId: connection.peerId,
|
|
1580
|
-
documentId: params.documentId
|
|
1581
|
-
}, {
|
|
1582
|
-
F: __dxlog_file6,
|
|
1583
|
-
L: 114,
|
|
1584
|
-
S: this,
|
|
1585
|
-
C: (f, a) => f(...a)
|
|
1586
|
-
});
|
|
1587
|
-
invariant5(this._context, void 0, {
|
|
1588
|
-
F: __dxlog_file6,
|
|
1589
|
-
L: 115,
|
|
1590
|
-
S: this,
|
|
1591
|
-
A: [
|
|
1592
|
-
"this._context",
|
|
1593
|
-
""
|
|
1594
|
-
]
|
|
1595
|
-
});
|
|
1596
|
-
try {
|
|
1597
|
-
const spaceKey = await this._context.getContainingSpaceForDocument(params.documentId);
|
|
1598
|
-
if (!spaceKey) {
|
|
1599
|
-
const remoteDocumentExists = await this._context.isDocumentInRemoteCollection({
|
|
1600
|
-
documentId: params.documentId,
|
|
1601
|
-
peerId: connection.peerId
|
|
1602
|
-
});
|
|
1603
|
-
log5("document not found locally for share policy check", {
|
|
1604
|
-
peerId: connection.peerId,
|
|
1605
|
-
documentId: params.documentId,
|
|
1606
|
-
acceptDocument: remoteDocumentExists
|
|
1607
|
-
}, {
|
|
1608
|
-
F: __dxlog_file6,
|
|
1609
|
-
L: 123,
|
|
1610
|
-
S: this,
|
|
1611
|
-
C: (f, a) => f(...a)
|
|
1612
|
-
});
|
|
1613
|
-
return remoteDocumentExists;
|
|
1614
|
-
}
|
|
1615
|
-
const spaceId = await createIdFromSpaceKey(spaceKey);
|
|
1616
|
-
const authorizedDevices = this._authorizedDevices.get(spaceId);
|
|
1617
|
-
if (!connection.remoteDeviceKey) {
|
|
1618
|
-
log5("device key not found for share policy check", {
|
|
1619
|
-
peerId: connection.peerId,
|
|
1620
|
-
documentId: params.documentId
|
|
1621
|
-
}, {
|
|
1622
|
-
F: __dxlog_file6,
|
|
1623
|
-
L: 140,
|
|
1624
|
-
S: this,
|
|
1625
|
-
C: (f, a) => f(...a)
|
|
1626
|
-
});
|
|
1627
|
-
return false;
|
|
1628
|
-
}
|
|
1629
|
-
const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
|
|
1630
|
-
log5("share policy check", {
|
|
1631
|
-
localPeer: this._context.peerId,
|
|
1632
|
-
remotePeer: connection.peerId,
|
|
1633
|
-
documentId: params.documentId,
|
|
1634
|
-
deviceKey: connection.remoteDeviceKey,
|
|
1635
|
-
spaceKey,
|
|
1636
|
-
isAuthorized
|
|
1637
|
-
}, {
|
|
1638
|
-
F: __dxlog_file6,
|
|
1639
|
-
L: 148,
|
|
1640
|
-
S: this,
|
|
1641
|
-
C: (f, a) => f(...a)
|
|
1642
|
-
});
|
|
1643
|
-
return isAuthorized;
|
|
1644
|
-
} catch (err) {
|
|
1645
|
-
log5.catch(err, void 0, {
|
|
1646
|
-
F: __dxlog_file6,
|
|
1647
|
-
L: 158,
|
|
1648
|
-
S: this,
|
|
1649
|
-
C: (f, a) => f(...a)
|
|
1650
|
-
});
|
|
1651
|
-
return false;
|
|
1652
|
-
}
|
|
1653
|
-
},
|
|
1654
|
-
shouldSyncCollection: ({ collectionId }) => {
|
|
1655
|
-
const spaceId = getSpaceIdFromCollectionId(collectionId);
|
|
1656
|
-
const authorizedDevices = this._authorizedDevices.get(spaceId);
|
|
1657
|
-
if (!connection.remoteDeviceKey) {
|
|
1658
|
-
log5("device key not found for collection sync check", {
|
|
1659
|
-
peerId: connection.peerId,
|
|
1660
|
-
collectionId
|
|
1661
|
-
}, {
|
|
1662
|
-
F: __dxlog_file6,
|
|
1663
|
-
L: 168,
|
|
1664
|
-
S: this,
|
|
1665
|
-
C: (f, a) => f(...a)
|
|
1666
|
-
});
|
|
1667
|
-
return false;
|
|
1668
|
-
}
|
|
1669
|
-
const isAuthorized = authorizedDevices?.has(connection.remoteDeviceKey) ?? false;
|
|
1670
|
-
return isAuthorized;
|
|
1671
|
-
}
|
|
1672
|
-
});
|
|
1673
|
-
this._connections.add(connection);
|
|
1674
|
-
return connection.replicatorExtension;
|
|
1675
|
-
}
|
|
1676
|
-
async authorizeDevice(spaceKey, deviceKey) {
|
|
1677
|
-
log5("authorizeDevice", {
|
|
1678
|
-
spaceKey,
|
|
1679
|
-
deviceKey
|
|
1680
|
-
}, {
|
|
1681
|
-
F: __dxlog_file6,
|
|
1682
|
-
L: 185,
|
|
1683
|
-
S: this,
|
|
1684
|
-
C: (f, a) => f(...a)
|
|
1685
|
-
});
|
|
1686
|
-
const spaceId = await createIdFromSpaceKey(spaceKey);
|
|
1687
|
-
defaultMap2(this._authorizedDevices, spaceId, () => new ComplexSet(PublicKey2.hash)).add(deviceKey);
|
|
1688
|
-
for (const connection of this._connections) {
|
|
1689
|
-
if (connection.isEnabled && connection.remoteDeviceKey && connection.remoteDeviceKey.equals(deviceKey)) {
|
|
1690
|
-
if (this._connectionsPerPeer.has(connection.peerId)) {
|
|
1691
|
-
this._context?.onConnectionAuthScopeChanged(connection);
|
|
1692
|
-
}
|
|
1693
|
-
}
|
|
1694
|
-
}
|
|
1695
|
-
}
|
|
1696
|
-
};
|
|
1697
|
-
|
|
1698
|
-
// packages/core/echo/echo-pipeline/src/automerge/echo-data-monitor.ts
|
|
1699
|
-
import { trace as trace3 } from "@dxos/tracing";
|
|
1700
|
-
import { CircularBuffer, mapValues, SlidingWindowSummary } from "@dxos/util";
|
|
1701
|
-
function _ts_decorate4(decorators, target, key, desc) {
|
|
1702
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1703
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
1704
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1705
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1706
|
-
}
|
|
1707
|
-
var PER_SECOND_RATE_AVG_WINDOW_SIZE = 5;
|
|
1708
|
-
var DEFAULT_AVG_WINDOW_SIZE = 25;
|
|
1709
|
-
var EchoDataMonitor = class {
|
|
1710
|
-
constructor(_params = {
|
|
1711
|
-
timeSeriesLength: 30
|
|
1712
|
-
}) {
|
|
1713
|
-
this._params = _params;
|
|
1714
|
-
this._lastTick = 0;
|
|
1715
|
-
this._activeCounters = createLocalCounters();
|
|
1716
|
-
this._localTimeSeries = createLocalTimeSeries();
|
|
1717
|
-
this._storageAverages = createStorageAverages();
|
|
1718
|
-
this._replicationAverages = createNetworkAverages();
|
|
1719
|
-
this._sizeByMessage = {};
|
|
1720
|
-
this._lastReceivedMessages = new CircularBuffer(100);
|
|
1721
|
-
this._lastSentMessages = new CircularBuffer(100);
|
|
1722
|
-
this._connectionsCount = 0;
|
|
1723
|
-
}
|
|
1724
|
-
tick(timeMs) {
|
|
1725
|
-
this._advanceTimeWindow(timeMs - this._lastTick);
|
|
1726
|
-
this._lastTick = timeMs;
|
|
1727
|
-
}
|
|
1728
|
-
computeStats() {
|
|
1729
|
-
return {
|
|
1730
|
-
meta: {
|
|
1731
|
-
rateAverageOverSeconds: PER_SECOND_RATE_AVG_WINDOW_SIZE
|
|
1732
|
-
},
|
|
1733
|
-
storage: {
|
|
1734
|
-
reads: {
|
|
1735
|
-
payloadSize: this._storageAverages.loadedChunkSize.average(),
|
|
1736
|
-
opDuration: this._storageAverages.loadDuration.average(),
|
|
1737
|
-
countPerSecond: this._storageAverages.loadsPerSecond.average()
|
|
1738
|
-
},
|
|
1739
|
-
writes: {
|
|
1740
|
-
payloadSize: this._storageAverages.storedChunkSize.average(),
|
|
1741
|
-
opDuration: this._storageAverages.storeDuration.average(),
|
|
1742
|
-
countPerSecond: this._storageAverages.storesPerSecond.average()
|
|
1743
|
-
}
|
|
1744
|
-
},
|
|
1745
|
-
replicator: {
|
|
1746
|
-
connections: this._connectionsCount,
|
|
1747
|
-
receivedMessages: {
|
|
1748
|
-
payloadSize: this._replicationAverages.receivedMessageSize.average(),
|
|
1749
|
-
countPerSecond: this._replicationAverages.receivedPerSecond.average()
|
|
1750
|
-
},
|
|
1751
|
-
sentMessages: {
|
|
1752
|
-
payloadSize: this._replicationAverages.sentMessageSize.average(),
|
|
1753
|
-
opDuration: this._replicationAverages.sendDuration.average(),
|
|
1754
|
-
countPerSecond: this._replicationAverages.sentPerSecond.average(),
|
|
1755
|
-
failedPerSecond: this._replicationAverages.sendsFailedPerSecond.average()
|
|
1756
|
-
},
|
|
1757
|
-
countByMessage: this._computeMessageHistogram("type"),
|
|
1758
|
-
avgSizeByMessage: mapValues(this._sizeByMessage, (summary) => summary.average())
|
|
1759
|
-
}
|
|
1760
|
-
};
|
|
1761
|
-
}
|
|
1762
|
-
get connectionsCount() {
|
|
1763
|
-
return this._connectionsCount;
|
|
1764
|
-
}
|
|
1765
|
-
/**
|
|
1766
|
-
* @internal
|
|
1767
|
-
*/
|
|
1768
|
-
get lastPerSecondStats() {
|
|
1769
|
-
return this._lastCompleteCounters;
|
|
1770
|
-
}
|
|
1771
|
-
/**
|
|
1772
|
-
* @internal
|
|
1773
|
-
*/
|
|
1774
|
-
get timeSeries() {
|
|
1775
|
-
return {
|
|
1776
|
-
...this._localTimeSeries.storage,
|
|
1777
|
-
...this._localTimeSeries.replication
|
|
1778
|
-
};
|
|
1779
|
-
}
|
|
1780
|
-
/**
|
|
1781
|
-
* @internal
|
|
1782
|
-
*/
|
|
1783
|
-
get messagesByPeerId() {
|
|
1784
|
-
return this._computeMessageHistogram("peerId");
|
|
1785
|
-
}
|
|
1786
|
-
_advanceTimeWindow(millisPassed) {
|
|
1787
|
-
const oldMetrics = Object.freeze(this._activeCounters);
|
|
1788
|
-
this._activeCounters = createLocalCounters();
|
|
1789
|
-
this._lastCompleteCounters = oldMetrics;
|
|
1790
|
-
for (const peerId of Object.keys(oldMetrics.byPeerId)) {
|
|
1791
|
-
this._activeCounters.byPeerId[peerId] = createMessageCounter();
|
|
1792
|
-
}
|
|
1793
|
-
this._addToTimeSeries(oldMetrics.replication, this._localTimeSeries.replication);
|
|
1794
|
-
this._addToTimeSeries(oldMetrics.storage, this._localTimeSeries.storage);
|
|
1795
|
-
if (Math.abs(millisPassed - 1e3) < 100) {
|
|
1796
|
-
this._reportPerSecondRate(oldMetrics);
|
|
1797
|
-
}
|
|
1798
|
-
}
|
|
1799
|
-
_addToTimeSeries(values, timeSeries) {
|
|
1800
|
-
for (const [key, value] of Object.entries(values)) {
|
|
1801
|
-
const values2 = timeSeries[key];
|
|
1802
|
-
values2.push(value);
|
|
1803
|
-
if (values2.length > this._params.timeSeriesLength) {
|
|
1804
|
-
values2.shift();
|
|
1805
|
-
}
|
|
1806
|
-
}
|
|
1807
|
-
}
|
|
1808
|
-
_reportPerSecondRate(metrics) {
|
|
1809
|
-
const toReport = [
|
|
1810
|
-
[
|
|
1811
|
-
"storage.load",
|
|
1812
|
-
metrics.storage.loadedChunks,
|
|
1813
|
-
this._storageAverages.loadsPerSecond
|
|
1814
|
-
],
|
|
1815
|
-
[
|
|
1816
|
-
"storage.store",
|
|
1817
|
-
metrics.storage.storedChunks,
|
|
1818
|
-
this._storageAverages.storesPerSecond
|
|
1819
|
-
],
|
|
1820
|
-
[
|
|
1821
|
-
"network.receive",
|
|
1822
|
-
metrics.replication.received,
|
|
1823
|
-
this._replicationAverages.receivedPerSecond
|
|
1824
|
-
],
|
|
1825
|
-
[
|
|
1826
|
-
"network.send",
|
|
1827
|
-
metrics.replication.sent,
|
|
1828
|
-
this._replicationAverages.sentPerSecond
|
|
1829
|
-
]
|
|
1830
|
-
];
|
|
1831
|
-
for (const [metricName, metric, summary] of toReport) {
|
|
1832
|
-
summary.record(metric);
|
|
1833
|
-
if (metric > 0) {
|
|
1834
|
-
trace3.metrics.distribution(`dxos.echo.${metricName}-rate`, metric);
|
|
1835
|
-
trace3.metrics.increment(`dxos.echo.${metricName}`, 1, {
|
|
1836
|
-
tags: {
|
|
1837
|
-
status: "busy"
|
|
1838
|
-
}
|
|
1839
|
-
});
|
|
1840
|
-
} else {
|
|
1841
|
-
trace3.metrics.increment(`dxos.echo.${metricName}`, 1, {
|
|
1842
|
-
tags: {
|
|
1843
|
-
status: "idle"
|
|
1844
|
-
}
|
|
1845
|
-
});
|
|
1846
|
-
}
|
|
1847
|
-
}
|
|
1848
|
-
this._replicationAverages.sendsFailedPerSecond.record(metrics.replication.failed);
|
|
1849
|
-
}
|
|
1850
|
-
recordPeerConnected(peerId) {
|
|
1851
|
-
this._activeCounters.byPeerId[peerId] = createMessageCounter();
|
|
1852
|
-
this._connectionsCount++;
|
|
1853
|
-
}
|
|
1854
|
-
recordPeerDisconnected(peerId) {
|
|
1855
|
-
this._connectionsCount--;
|
|
1856
|
-
delete this._activeCounters.byPeerId[peerId];
|
|
1857
|
-
}
|
|
1858
|
-
recordBytesStored(count) {
|
|
1859
|
-
this._activeCounters.storage.storedChunks++;
|
|
1860
|
-
this._activeCounters.storage.storedBytes += count;
|
|
1861
|
-
this._storageAverages.storedChunkSize.record(count);
|
|
1862
|
-
trace3.metrics.distribution("dxos.echo.storage.bytes-stored", count, {
|
|
1863
|
-
unit: "bytes"
|
|
1864
|
-
});
|
|
1865
|
-
}
|
|
1866
|
-
recordLoadDuration(durationMs) {
|
|
1867
|
-
this._storageAverages.loadDuration.record(durationMs);
|
|
1868
|
-
}
|
|
1869
|
-
recordStoreDuration(durationMs) {
|
|
1870
|
-
this._storageAverages.storeDuration.record(durationMs);
|
|
1871
|
-
}
|
|
1872
|
-
recordBytesLoaded(count) {
|
|
1873
|
-
this._activeCounters.storage.loadedChunks++;
|
|
1874
|
-
this._activeCounters.storage.loadedBytes += count;
|
|
1875
|
-
this._storageAverages.loadedChunkSize.record(count);
|
|
1876
|
-
trace3.metrics.distribution("dxos.echo.storage.bytes-loaded", count, {
|
|
1877
|
-
unit: "bytes"
|
|
1878
|
-
});
|
|
1879
|
-
}
|
|
1880
|
-
recordMessageSent(message, duration) {
|
|
1881
|
-
let metricsGroupName;
|
|
1882
|
-
const bytes = getByteCount(message);
|
|
1883
|
-
const tags = {
|
|
1884
|
-
type: message.type
|
|
1885
|
-
};
|
|
1886
|
-
if (isAutomergeProtocolMessage(message)) {
|
|
1887
|
-
this._activeCounters.replication.sent++;
|
|
1888
|
-
this._replicationAverages.sendDuration.record(duration);
|
|
1889
|
-
this._replicationAverages.sentMessageSize.record(bytes);
|
|
1890
|
-
metricsGroupName = "replication";
|
|
1891
|
-
} else {
|
|
1892
|
-
metricsGroupName = "collection-sync";
|
|
1893
|
-
}
|
|
1894
|
-
trace3.metrics.distribution(`dxos.echo.${metricsGroupName}.bytes-sent`, bytes, {
|
|
1895
|
-
unit: "bytes",
|
|
1896
|
-
tags
|
|
1897
|
-
});
|
|
1898
|
-
trace3.metrics.distribution(`dxos.echo.${metricsGroupName}.send-duration`, duration, {
|
|
1899
|
-
unit: "millisecond",
|
|
1900
|
-
tags
|
|
1901
|
-
});
|
|
1902
|
-
trace3.metrics.increment(`dxos.echo.${metricsGroupName}.send-status`, 1, {
|
|
1903
|
-
tags: {
|
|
1904
|
-
...tags,
|
|
1905
|
-
success: true
|
|
1906
|
-
}
|
|
1907
|
-
});
|
|
1908
|
-
const { messageSize, messageCounts } = this._getStatsForType(message);
|
|
1909
|
-
messageSize.record(bytes);
|
|
1910
|
-
messageCounts.sent++;
|
|
1911
|
-
this._lastSentMessages.push({
|
|
1912
|
-
type: message.type,
|
|
1913
|
-
peerId: message.targetId
|
|
1914
|
-
});
|
|
1915
|
-
}
|
|
1916
|
-
recordMessageReceived(message) {
|
|
1917
|
-
const bytes = getByteCount(message);
|
|
1918
|
-
const tags = {
|
|
1919
|
-
type: message.type
|
|
1920
|
-
};
|
|
1921
|
-
if (isAutomergeProtocolMessage(message)) {
|
|
1922
|
-
this._activeCounters.replication.received++;
|
|
1923
|
-
this._replicationAverages.receivedMessageSize.record(bytes);
|
|
1924
|
-
trace3.metrics.distribution("dxos.echo.replication.bytes-received", bytes, {
|
|
1925
|
-
unit: "bytes",
|
|
1926
|
-
tags
|
|
1927
|
-
});
|
|
1928
|
-
} else {
|
|
1929
|
-
trace3.metrics.distribution("dxos.echo.collection-sync.bytes-received", bytes, {
|
|
1930
|
-
unit: "bytes",
|
|
1931
|
-
tags
|
|
1932
|
-
});
|
|
1933
|
-
}
|
|
1934
|
-
const { messageSize, messageCounts } = this._getStatsForType(message);
|
|
1935
|
-
messageSize.record(bytes);
|
|
1936
|
-
messageCounts.received++;
|
|
1937
|
-
this._lastReceivedMessages.push({
|
|
1938
|
-
type: message.type,
|
|
1939
|
-
peerId: message.senderId
|
|
1940
|
-
});
|
|
1941
|
-
}
|
|
1942
|
-
recordMessageSendingFailed(message) {
|
|
1943
|
-
const tags = {
|
|
1944
|
-
type: message.type,
|
|
1945
|
-
success: false
|
|
1946
|
-
};
|
|
1947
|
-
if (isAutomergeProtocolMessage(message)) {
|
|
1948
|
-
this._activeCounters.replication.failed++;
|
|
1949
|
-
trace3.metrics.increment("dxos.echo.replication.send-status", 1, {
|
|
1950
|
-
unit: "bytes",
|
|
1951
|
-
tags
|
|
1952
|
-
});
|
|
1953
|
-
} else {
|
|
1954
|
-
trace3.metrics.increment("dxos.echo.collection-sync.send-status", 1, {
|
|
1955
|
-
unit: "bytes",
|
|
1956
|
-
tags
|
|
1957
|
-
});
|
|
1958
|
-
}
|
|
1959
|
-
const { messageCounts } = this._getStatsForType(message);
|
|
1960
|
-
messageCounts.failed++;
|
|
1961
|
-
}
|
|
1962
|
-
_getStatsForType(message) {
|
|
1963
|
-
const messageSize = this._sizeByMessage[message.type] ??= createSlidingWindow();
|
|
1964
|
-
const messageCounts = this._activeCounters.byType[message.type] ??= createMessageCounter();
|
|
1965
|
-
return {
|
|
1966
|
-
messageCounts,
|
|
1967
|
-
messageSize
|
|
1968
|
-
};
|
|
1969
|
-
}
|
|
1970
|
-
_computeMessageHistogram(groupKey) {
|
|
1971
|
-
const result = {};
|
|
1972
|
-
for (const receivedMessage of this._lastReceivedMessages) {
|
|
1973
|
-
const counters = result[receivedMessage[groupKey]] ??= {
|
|
1974
|
-
received: 0,
|
|
1975
|
-
sent: 0
|
|
1976
|
-
};
|
|
1977
|
-
counters.received++;
|
|
1978
|
-
}
|
|
1979
|
-
for (const receivedMessage of this._lastSentMessages) {
|
|
1980
|
-
const counters = result[receivedMessage[groupKey]] ??= {
|
|
1981
|
-
received: 0,
|
|
1982
|
-
sent: 0
|
|
1983
|
-
};
|
|
1984
|
-
counters.sent++;
|
|
1985
|
-
}
|
|
1986
|
-
return result;
|
|
1987
|
-
}
|
|
1988
|
-
};
|
|
1989
|
-
EchoDataMonitor = _ts_decorate4([
|
|
1990
|
-
trace3.resource()
|
|
1991
|
-
], EchoDataMonitor);
|
|
1992
|
-
var isAutomergeProtocolMessage = (message) => {
|
|
1993
|
-
return !(isCollectionQueryMessage(message) || isCollectionStateMessage(message));
|
|
1994
|
-
};
|
|
1995
|
-
var createSlidingWindow = (overrides) => new SlidingWindowSummary({
|
|
1996
|
-
dataPoints: DEFAULT_AVG_WINDOW_SIZE,
|
|
1997
|
-
precision: 2,
|
|
1998
|
-
...overrides
|
|
1999
|
-
});
|
|
2000
|
-
var createLocalCounters = () => ({
|
|
2001
|
-
storage: {
|
|
2002
|
-
loadedBytes: 0,
|
|
2003
|
-
storedBytes: 0,
|
|
2004
|
-
storedChunks: 0,
|
|
2005
|
-
loadedChunks: 0
|
|
2006
|
-
},
|
|
2007
|
-
replication: createMessageCounter(),
|
|
2008
|
-
byPeerId: {},
|
|
2009
|
-
byType: {}
|
|
2010
|
-
});
|
|
2011
|
-
var createLocalTimeSeries = () => ({
|
|
2012
|
-
storage: {
|
|
2013
|
-
loadedBytes: [],
|
|
2014
|
-
storedBytes: [],
|
|
2015
|
-
storedChunks: [],
|
|
2016
|
-
loadedChunks: []
|
|
2017
|
-
},
|
|
2018
|
-
replication: {
|
|
2019
|
-
sent: [],
|
|
2020
|
-
failed: [],
|
|
2021
|
-
received: []
|
|
2022
|
-
}
|
|
2023
|
-
});
|
|
2024
|
-
var createMessageCounter = () => ({
|
|
2025
|
-
sent: 0,
|
|
2026
|
-
received: 0,
|
|
2027
|
-
failed: 0
|
|
2028
|
-
});
|
|
2029
|
-
var createNetworkAverages = () => ({
|
|
2030
|
-
receivedMessageSize: createSlidingWindow(),
|
|
2031
|
-
sentMessageSize: createSlidingWindow(),
|
|
2032
|
-
sendDuration: createSlidingWindow(),
|
|
2033
|
-
receivedPerSecond: createSlidingWindow({
|
|
2034
|
-
dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
|
|
2035
|
-
}),
|
|
2036
|
-
sentPerSecond: createSlidingWindow({
|
|
2037
|
-
dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
|
|
2038
|
-
}),
|
|
2039
|
-
sendsFailedPerSecond: createSlidingWindow({
|
|
2040
|
-
dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
|
|
2041
|
-
})
|
|
2042
|
-
});
|
|
2043
|
-
var createStorageAverages = () => ({
|
|
2044
|
-
storedChunkSize: createSlidingWindow(),
|
|
2045
|
-
loadedChunkSize: createSlidingWindow(),
|
|
2046
|
-
loadDuration: createSlidingWindow(),
|
|
2047
|
-
storeDuration: createSlidingWindow(),
|
|
2048
|
-
loadsPerSecond: createSlidingWindow({
|
|
2049
|
-
dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
|
|
2050
|
-
}),
|
|
2051
|
-
storesPerSecond: createSlidingWindow({
|
|
2052
|
-
dataPoints: PER_SECOND_RATE_AVG_WINDOW_SIZE
|
|
2053
|
-
})
|
|
2054
|
-
});
|
|
2055
|
-
var getByteCount = (message) => {
|
|
2056
|
-
return message.type.length + message.senderId.length + message.targetId.length + (message.data?.byteLength ?? 0) + (message.documentId?.length ?? 0);
|
|
2057
|
-
};
|
|
2058
|
-
|
|
2059
|
-
// packages/core/echo/echo-pipeline/src/db-host/documents-synchronizer.ts
|
|
2060
|
-
var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/documents-synchronizer.ts";
|
|
2061
|
-
var MAX_UPDATE_FREQ = 10;
|
|
2062
|
-
var DocumentsSynchronizer = class extends Resource5 {
|
|
2063
|
-
constructor(_params) {
|
|
2064
|
-
super(), this._params = _params, this._syncStates = /* @__PURE__ */ new Map(), this._pendingUpdates = /* @__PURE__ */ new Set(), this._sendUpdatesJob = void 0;
|
|
2065
|
-
}
|
|
2066
|
-
addDocuments(documentIds, retryCounter = 0) {
|
|
2067
|
-
if (retryCounter > 3) {
|
|
2068
|
-
log6.warn("Failed to load document, retry limit reached", {
|
|
2069
|
-
documentIds
|
|
2070
|
-
}, {
|
|
2071
|
-
F: __dxlog_file7,
|
|
2072
|
-
L: 52,
|
|
2073
|
-
S: this,
|
|
2074
|
-
C: (f, a) => f(...a)
|
|
2075
|
-
});
|
|
2076
|
-
return;
|
|
2077
|
-
}
|
|
2078
|
-
for (const documentId of documentIds) {
|
|
2079
|
-
this._params.repo.find(documentId).then(async (doc) => {
|
|
2080
|
-
await doc.whenReady();
|
|
2081
|
-
this._startSync(doc);
|
|
2082
|
-
this._pendingUpdates.add(doc.documentId);
|
|
2083
|
-
this._sendUpdatesJob.trigger();
|
|
2084
|
-
}).catch((error) => {
|
|
2085
|
-
log6.warn("Failed to load document, wraparound", {
|
|
2086
|
-
documentId,
|
|
2087
|
-
error
|
|
2088
|
-
}, {
|
|
2089
|
-
F: __dxlog_file7,
|
|
2090
|
-
L: 66,
|
|
2091
|
-
S: this,
|
|
2092
|
-
C: (f, a) => f(...a)
|
|
2093
|
-
});
|
|
2094
|
-
this.addDocuments([
|
|
2095
|
-
documentId
|
|
2096
|
-
], retryCounter + 1);
|
|
2097
|
-
});
|
|
2098
|
-
}
|
|
2099
|
-
}
|
|
2100
|
-
removeDocuments(documentIds) {
|
|
2101
|
-
for (const documentId of documentIds) {
|
|
2102
|
-
this._syncStates.get(documentId)?.clearSubscriptions?.();
|
|
2103
|
-
this._syncStates.delete(documentId);
|
|
2104
|
-
this._pendingUpdates.delete(documentId);
|
|
2105
|
-
}
|
|
2106
|
-
}
|
|
2107
|
-
async _open() {
|
|
2108
|
-
this._sendUpdatesJob = new UpdateScheduler(this._ctx, this._checkAndSendUpdates.bind(this), {
|
|
2109
|
-
maxFrequency: MAX_UPDATE_FREQ
|
|
2110
|
-
});
|
|
2111
|
-
}
|
|
2112
|
-
async _close() {
|
|
2113
|
-
await this._sendUpdatesJob.join();
|
|
2114
|
-
this._syncStates.clear();
|
|
2115
|
-
}
|
|
2116
|
-
async update(updates) {
|
|
2117
|
-
for (const { documentId, mutation, isNew } of updates) {
|
|
2118
|
-
if (isNew) {
|
|
2119
|
-
const doc = await this._params.repo.find(documentId, FIND_PARAMS);
|
|
2120
|
-
doc.update((doc2) => A2.loadIncremental(doc2, mutation));
|
|
2121
|
-
this._startSync(doc);
|
|
2122
|
-
} else {
|
|
2123
|
-
this._writeMutation(documentId, mutation);
|
|
2124
|
-
}
|
|
2125
|
-
}
|
|
2126
|
-
}
|
|
2127
|
-
_startSync(doc) {
|
|
2128
|
-
if (this._syncStates.has(doc.documentId)) {
|
|
2129
|
-
log6("Document already being synced", {
|
|
2130
|
-
documentId: doc.documentId
|
|
2131
|
-
}, {
|
|
2132
|
-
F: __dxlog_file7,
|
|
2133
|
-
L: 105,
|
|
2134
|
-
S: this,
|
|
2135
|
-
C: (f, a) => f(...a)
|
|
2136
|
-
});
|
|
2137
|
-
return;
|
|
2138
|
-
}
|
|
2139
|
-
const syncState = {
|
|
2140
|
-
handle: doc
|
|
2141
|
-
};
|
|
2142
|
-
this._subscribeForChanges(syncState);
|
|
2143
|
-
this._syncStates.set(doc.documentId, syncState);
|
|
2144
|
-
}
|
|
2145
|
-
_subscribeForChanges(syncState) {
|
|
2146
|
-
const handler = () => {
|
|
2147
|
-
this._pendingUpdates.add(syncState.handle.documentId);
|
|
2148
|
-
this._sendUpdatesJob.trigger();
|
|
2149
|
-
};
|
|
2150
|
-
syncState.handle.on("heads-changed", handler);
|
|
2151
|
-
syncState.clearSubscriptions = () => syncState.handle.off("heads-changed", handler);
|
|
2152
|
-
}
|
|
2153
|
-
async _checkAndSendUpdates() {
|
|
2154
|
-
const updates = [];
|
|
2155
|
-
const docsWithPendingUpdates = Array.from(this._pendingUpdates);
|
|
2156
|
-
this._pendingUpdates.clear();
|
|
2157
|
-
for (const documentId of docsWithPendingUpdates) {
|
|
2158
|
-
const update = this._getPendingChanges(documentId);
|
|
2159
|
-
if (update) {
|
|
2160
|
-
updates.push({
|
|
2161
|
-
documentId,
|
|
2162
|
-
mutation: update
|
|
2163
|
-
});
|
|
2164
|
-
}
|
|
2165
|
-
}
|
|
2166
|
-
if (updates.length > 0) {
|
|
2167
|
-
this._params.sendUpdates({
|
|
2168
|
-
updates
|
|
2169
|
-
});
|
|
2170
|
-
}
|
|
2171
|
-
}
|
|
2172
|
-
_getPendingChanges(documentId) {
|
|
2173
|
-
const syncState = this._syncStates.get(documentId);
|
|
2174
|
-
invariant6(syncState, "Sync state for document not found", {
|
|
2175
|
-
F: __dxlog_file7,
|
|
2176
|
-
L: 146,
|
|
2177
|
-
S: this,
|
|
2178
|
-
A: [
|
|
2179
|
-
"syncState",
|
|
2180
|
-
"'Sync state for document not found'"
|
|
2181
|
-
]
|
|
2182
|
-
});
|
|
2183
|
-
const handle = syncState.handle;
|
|
2184
|
-
if (!handle || !handle.isReady() || !handle.doc()) {
|
|
2185
|
-
return;
|
|
2186
|
-
}
|
|
2187
|
-
const doc = handle.doc();
|
|
2188
|
-
const mutation = syncState.lastSentHead ? A2.saveSince(doc, syncState.lastSentHead) : A2.save(doc);
|
|
2189
|
-
if (mutation.length === 0) {
|
|
2190
|
-
return;
|
|
2191
|
-
}
|
|
2192
|
-
syncState.lastSentHead = A2.getHeads(doc);
|
|
2193
|
-
return mutation;
|
|
2194
|
-
}
|
|
2195
|
-
_writeMutation(documentId, mutation) {
|
|
2196
|
-
const syncState = this._syncStates.get(documentId);
|
|
2197
|
-
invariant6(syncState, "Sync state for document not found", {
|
|
2198
|
-
F: __dxlog_file7,
|
|
2199
|
-
L: 162,
|
|
2200
|
-
S: this,
|
|
2201
|
-
A: [
|
|
2202
|
-
"syncState",
|
|
2203
|
-
"'Sync state for document not found'"
|
|
2204
|
-
]
|
|
2205
|
-
});
|
|
2206
|
-
syncState.handle.update((doc) => {
|
|
2207
|
-
const headsBefore = A2.getHeads(doc);
|
|
2208
|
-
const newDoc = A2.loadIncremental(doc, mutation);
|
|
2209
|
-
if (A2.equals(headsBefore, syncState.lastSentHead)) {
|
|
2210
|
-
syncState.lastSentHead = A2.getHeads(newDoc);
|
|
2211
|
-
}
|
|
2212
|
-
return newDoc;
|
|
2213
|
-
});
|
|
2214
|
-
}
|
|
2215
|
-
};
|
|
2216
|
-
|
|
2217
|
-
// packages/core/echo/echo-pipeline/src/db-host/data-service.ts
|
|
2218
|
-
var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
|
|
2219
|
-
var DataServiceImpl = class {
|
|
2220
|
-
constructor(params) {
|
|
2221
|
-
/**
|
|
2222
|
-
* Map of subscriptions.
|
|
2223
|
-
* subscriptionId -> DocumentsSynchronizer
|
|
2224
|
-
*/
|
|
2225
|
-
this._subscriptions = /* @__PURE__ */ new Map();
|
|
2226
|
-
this._automergeHost = params.automergeHost;
|
|
2227
|
-
this._spaceStateManager = params.spaceStateManager;
|
|
2228
|
-
this._updateIndexes = params.updateIndexes;
|
|
2229
|
-
}
|
|
2230
|
-
subscribe(request) {
|
|
2231
|
-
return new Stream(({ next, ready }) => {
|
|
2232
|
-
const synchronizer = new DocumentsSynchronizer({
|
|
2233
|
-
repo: this._automergeHost.repo,
|
|
2234
|
-
sendUpdates: (updates) => next(updates)
|
|
2235
|
-
});
|
|
2236
|
-
synchronizer.open().then(() => {
|
|
2237
|
-
this._subscriptions.set(request.subscriptionId, synchronizer);
|
|
2238
|
-
ready();
|
|
2239
|
-
}).catch((err) => log7.catch(err, void 0, {
|
|
2240
|
-
F: __dxlog_file8,
|
|
2241
|
-
L: 71,
|
|
2242
|
-
S: this,
|
|
2243
|
-
C: (f, a) => f(...a)
|
|
2244
|
-
}));
|
|
2245
|
-
return () => synchronizer.close();
|
|
2246
|
-
});
|
|
2247
|
-
}
|
|
2248
|
-
async updateSubscription(request) {
|
|
2249
|
-
const synchronizer = this._subscriptions.get(request.subscriptionId);
|
|
2250
|
-
invariant7(synchronizer, "Subscription not found", {
|
|
2251
|
-
F: __dxlog_file8,
|
|
2252
|
-
L: 78,
|
|
2253
|
-
S: this,
|
|
2254
|
-
A: [
|
|
2255
|
-
"synchronizer",
|
|
2256
|
-
"'Subscription not found'"
|
|
2257
|
-
]
|
|
2258
|
-
});
|
|
2259
|
-
if (request.addIds?.length) {
|
|
2260
|
-
await synchronizer.addDocuments(request.addIds);
|
|
2261
|
-
}
|
|
2262
|
-
if (request.removeIds?.length) {
|
|
2263
|
-
await synchronizer.removeDocuments(request.removeIds);
|
|
2264
|
-
}
|
|
2265
|
-
}
|
|
2266
|
-
async update(request) {
|
|
2267
|
-
if (!request.updates) {
|
|
2268
|
-
return;
|
|
2269
|
-
}
|
|
2270
|
-
const synchronizer = this._subscriptions.get(request.subscriptionId);
|
|
2271
|
-
invariant7(synchronizer, "Subscription not found", {
|
|
2272
|
-
F: __dxlog_file8,
|
|
2273
|
-
L: 93,
|
|
2274
|
-
S: this,
|
|
2275
|
-
A: [
|
|
2276
|
-
"synchronizer",
|
|
2277
|
-
"'Subscription not found'"
|
|
2278
|
-
]
|
|
2279
|
-
});
|
|
2280
|
-
await synchronizer.update(request.updates);
|
|
2281
|
-
}
|
|
2282
|
-
async flush(request) {
|
|
2283
|
-
await this._automergeHost.flush(request);
|
|
2284
|
-
}
|
|
2285
|
-
async getDocumentHeads(request) {
|
|
2286
|
-
const documentIds = request.documentIds;
|
|
2287
|
-
if (!documentIds) {
|
|
2288
|
-
return {
|
|
2289
|
-
heads: {
|
|
2290
|
-
entries: []
|
|
2291
|
-
}
|
|
2292
|
-
};
|
|
2293
|
-
}
|
|
2294
|
-
const heads = await this._automergeHost.getHeads(documentIds);
|
|
2295
|
-
return {
|
|
2296
|
-
heads: {
|
|
2297
|
-
entries: heads.map((heads2, idx) => ({
|
|
2298
|
-
documentId: documentIds[idx],
|
|
2299
|
-
heads: heads2
|
|
2300
|
-
}))
|
|
2301
|
-
}
|
|
2302
|
-
};
|
|
2303
|
-
}
|
|
2304
|
-
async waitUntilHeadsReplicated(request, options) {
|
|
2305
|
-
await this._automergeHost.waitUntilHeadsReplicated(request.heads);
|
|
2306
|
-
}
|
|
2307
|
-
async reIndexHeads(request, options) {
|
|
2308
|
-
await this._automergeHost.reIndexHeads(request.documentIds ?? []);
|
|
2309
|
-
}
|
|
2310
|
-
async updateIndexes() {
|
|
2311
|
-
await this._updateIndexes();
|
|
2312
|
-
}
|
|
2313
|
-
subscribeSpaceSyncState(request) {
|
|
2314
|
-
return new Stream(({ ctx, next, ready }) => {
|
|
2315
|
-
const spaceId = request.spaceId;
|
|
2316
|
-
invariant7(SpaceId2.isValid(spaceId), void 0, {
|
|
2317
|
-
F: __dxlog_file8,
|
|
2318
|
-
L: 133,
|
|
2319
|
-
S: this,
|
|
2320
|
-
A: [
|
|
2321
|
-
"SpaceId.isValid(spaceId)",
|
|
2322
|
-
""
|
|
2323
|
-
]
|
|
2324
|
-
});
|
|
2325
|
-
const rootDocumentId = this._spaceStateManager.getSpaceRootDocumentId(spaceId);
|
|
2326
|
-
let collectionId = rootDocumentId && deriveCollectionIdFromSpaceId(spaceId, rootDocumentId);
|
|
2327
|
-
this._spaceStateManager.spaceDocumentListUpdated.on(ctx, (event) => {
|
|
2328
|
-
const newId = deriveCollectionIdFromSpaceId(spaceId, event.spaceRootId);
|
|
2329
|
-
if (newId !== collectionId) {
|
|
2330
|
-
collectionId = newId;
|
|
2331
|
-
scheduler.trigger();
|
|
2332
|
-
}
|
|
2333
|
-
});
|
|
2334
|
-
const scheduler = new UpdateScheduler2(ctx, async () => {
|
|
2335
|
-
const state = collectionId ? await this._automergeHost.getCollectionSyncState(collectionId) : {
|
|
2336
|
-
peers: []
|
|
2337
|
-
};
|
|
2338
|
-
next({
|
|
2339
|
-
peers: state.peers.map((peer) => ({
|
|
2340
|
-
peerId: peer.peerId,
|
|
2341
|
-
missingOnRemote: peer.missingOnRemote,
|
|
2342
|
-
missingOnLocal: peer.missingOnLocal,
|
|
2343
|
-
differentDocuments: peer.differentDocuments,
|
|
2344
|
-
localDocumentCount: peer.localDocumentCount,
|
|
2345
|
-
remoteDocumentCount: peer.remoteDocumentCount
|
|
2346
|
-
}))
|
|
2347
|
-
});
|
|
2348
|
-
});
|
|
2349
|
-
this._automergeHost.collectionStateUpdated.on(ctx, (e) => {
|
|
2350
|
-
if (e.collectionId === collectionId) {
|
|
2351
|
-
scheduler.trigger();
|
|
2352
|
-
}
|
|
2353
|
-
});
|
|
2354
|
-
scheduler.trigger();
|
|
2355
|
-
});
|
|
2356
|
-
}
|
|
2357
|
-
};
|
|
2358
|
-
|
|
2359
|
-
// packages/core/echo/echo-pipeline/src/db-host/echo-host.ts
|
|
2360
|
-
import { LifecycleState as LifecycleState5, Resource as Resource9 } from "@dxos/context";
|
|
2361
|
-
import { todo } from "@dxos/debug";
|
|
2362
|
-
import { createIdFromSpaceKey as createIdFromSpaceKey2, SpaceDocVersion as SpaceDocVersion3 } from "@dxos/echo-protocol";
|
|
2363
|
-
import { IndexMetadataStore, IndexStore, Indexer } from "@dxos/indexing";
|
|
2364
|
-
import { invariant as invariant13 } from "@dxos/invariant";
|
|
2365
|
-
import { IndexKind } from "@dxos/protocols/proto/dxos/echo/indexing";
|
|
2366
|
-
import { trace as trace5 } from "@dxos/tracing";
|
|
2367
|
-
|
|
2368
|
-
// packages/core/echo/echo-pipeline/src/db-host/documents-iterator.ts
|
|
2369
|
-
import * as A3 from "@automerge/automerge";
|
|
2370
|
-
import { Context as Context2 } from "@dxos/context";
|
|
2371
|
-
import { DatabaseDirectory as DatabaseDirectory2, SpaceDocVersion } from "@dxos/echo-protocol";
|
|
2372
|
-
import { invariant as invariant8 } from "@dxos/invariant";
|
|
2373
|
-
import { log as log8 } from "@dxos/log";
|
|
2374
|
-
import { ObjectPointerVersion, objectPointerCodec as objectPointerCodec2 } from "@dxos/protocols";
|
|
2375
|
-
var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/documents-iterator.ts";
|
|
2376
|
-
var LOG_VIEW_OPERATION_THRESHOLD = 300;
|
|
2377
|
-
var createSelectedDocumentsIterator = (automergeHost) => (
|
|
2378
|
-
/**
|
|
2379
|
-
* Get object data blobs from Automerge Repo by ids.
|
|
2380
|
-
*/
|
|
2381
|
-
// TODO(mykola): Unload automerge handles after usage.
|
|
2382
|
-
async function* loadDocuments(objects) {
|
|
2383
|
-
for (const [id, heads] of objects.entries()) {
|
|
2384
|
-
try {
|
|
2385
|
-
const { documentId, objectId } = objectPointerCodec2.decode(id);
|
|
2386
|
-
const handle = await automergeHost.loadDoc(Context2.default(void 0, {
|
|
2387
|
-
F: __dxlog_file9,
|
|
2388
|
-
L: 31
|
|
2389
|
-
}), documentId);
|
|
2390
|
-
let doc = handle.doc();
|
|
2391
|
-
invariant8(doc, void 0, {
|
|
2392
|
-
F: __dxlog_file9,
|
|
2393
|
-
L: 34,
|
|
2394
|
-
S: this,
|
|
2395
|
-
A: [
|
|
2396
|
-
"doc",
|
|
2397
|
-
""
|
|
2398
|
-
]
|
|
2399
|
-
});
|
|
2400
|
-
const currentHeads = A3.getHeads(doc);
|
|
2401
|
-
if (!A3.equals(currentHeads, heads)) {
|
|
2402
|
-
const begin = Date.now();
|
|
2403
|
-
doc = A3.view(doc, heads);
|
|
2404
|
-
const end = Date.now();
|
|
2405
|
-
if (end - begin > LOG_VIEW_OPERATION_THRESHOLD) {
|
|
2406
|
-
log8("Checking out document version is taking too long", {
|
|
2407
|
-
duration: end - begin,
|
|
2408
|
-
requestedHeads: heads,
|
|
2409
|
-
originalHeads: currentHeads
|
|
2410
|
-
}, {
|
|
2411
|
-
F: __dxlog_file9,
|
|
2412
|
-
L: 45,
|
|
2413
|
-
S: this,
|
|
2414
|
-
C: (f, a) => f(...a)
|
|
2415
|
-
});
|
|
2416
|
-
}
|
|
2417
|
-
}
|
|
2418
|
-
if (doc.version !== SpaceDocVersion.CURRENT) {
|
|
2419
|
-
continue;
|
|
2420
|
-
}
|
|
2421
|
-
if (!doc.objects?.[objectId]) {
|
|
2422
|
-
continue;
|
|
2423
|
-
}
|
|
2424
|
-
let newId = id;
|
|
2425
|
-
if (objectPointerCodec2.getVersion(id) === ObjectPointerVersion.V0) {
|
|
2426
|
-
const spaceKey = DatabaseDirectory2.getSpaceKey(doc) ?? void 0;
|
|
2427
|
-
newId = objectPointerCodec2.encode({
|
|
2428
|
-
documentId,
|
|
2429
|
-
objectId,
|
|
2430
|
-
spaceKey
|
|
2431
|
-
});
|
|
2432
|
-
}
|
|
2433
|
-
yield [
|
|
2434
|
-
{
|
|
2435
|
-
id: newId,
|
|
2436
|
-
object: doc.objects[objectId],
|
|
2437
|
-
heads
|
|
2438
|
-
}
|
|
2439
|
-
];
|
|
2440
|
-
} catch (error) {
|
|
2441
|
-
log8.error("Error loading document", {
|
|
2442
|
-
heads,
|
|
2443
|
-
id,
|
|
2444
|
-
error
|
|
2445
|
-
}, {
|
|
2446
|
-
F: __dxlog_file9,
|
|
2447
|
-
L: 71,
|
|
2448
|
-
S: this,
|
|
2449
|
-
C: (f, a) => f(...a)
|
|
2450
|
-
});
|
|
2451
|
-
}
|
|
2452
|
-
}
|
|
2453
|
-
}
|
|
2454
|
-
);
|
|
2455
|
-
|
|
2456
|
-
// packages/core/echo/echo-pipeline/src/db-host/query-service.ts
|
|
2457
|
-
import { getHeads as getHeads3 } from "@automerge/automerge";
|
|
2458
|
-
import { Schema } from "effect";
|
|
2459
|
-
import { DeferredTask, scheduleMicroTask, synchronized as synchronized2 } from "@dxos/async";
|
|
2460
|
-
import { Stream as Stream2 } from "@dxos/codec-protobuf/stream";
|
|
2461
|
-
import { Context as Context4, Resource as Resource7 } from "@dxos/context";
|
|
2462
|
-
import { raise } from "@dxos/debug";
|
|
2463
|
-
import { DatabaseDirectory as DatabaseDirectory4, QueryAST } from "@dxos/echo-protocol";
|
|
2464
|
-
import { log as log10 } from "@dxos/log";
|
|
2465
|
-
import { objectPointerCodec as objectPointerCodec4 } from "@dxos/protocols";
|
|
2466
|
-
import { trace as trace4 } from "@dxos/tracing";
|
|
2467
|
-
|
|
2468
|
-
// packages/core/echo/echo-pipeline/src/query/query-executor.ts
|
|
2469
|
-
import { Match } from "effect";
|
|
2470
|
-
import { Context as Context3, ContextDisposedError, LifecycleState as LifecycleState3, Resource as Resource6 } from "@dxos/context";
|
|
2471
|
-
import { DatabaseDirectory as DatabaseDirectory3, isEncodedReference, ObjectStructure } from "@dxos/echo-protocol";
|
|
2472
|
-
import { EscapedPropPath } from "@dxos/indexing";
|
|
2473
|
-
import { invariant as invariant10 } from "@dxos/invariant";
|
|
2474
|
-
import { DXN, PublicKey as PublicKey3 } from "@dxos/keys";
|
|
2475
|
-
import { log as log9 } from "@dxos/log";
|
|
2476
|
-
import { objectPointerCodec as objectPointerCodec3 } from "@dxos/protocols";
|
|
2477
|
-
import { getDeep, isNonNullable as isNonNullable2 } from "@dxos/util";
|
|
2478
|
-
|
|
2479
|
-
// packages/core/echo/echo-pipeline/src/query/query-planner.ts
|
|
2480
|
-
import { invariant as invariant9 } from "@dxos/invariant";
|
|
2481
|
-
|
|
2482
|
-
// packages/core/echo/echo-pipeline/src/query/errors.ts
|
|
2483
|
-
import { BaseError } from "@dxos/errors";
|
|
2484
|
-
var QueryError = class extends BaseError.extend("QUERY_ERROR") {
|
|
2485
|
-
};
|
|
2486
|
-
|
|
2487
|
-
// packages/core/echo/echo-pipeline/src/query/plan.ts
|
|
2488
|
-
(function(QueryPlan2) {
|
|
2489
|
-
QueryPlan2.Plan = Object.freeze({
|
|
2490
|
-
make: (steps) => ({
|
|
2491
|
-
steps
|
|
2492
|
-
})
|
|
2493
|
-
});
|
|
2494
|
-
QueryPlan2.FilterStep = Object.freeze({
|
|
2495
|
-
isNoop: (step) => {
|
|
2496
|
-
switch (step.filter.type) {
|
|
2497
|
-
case "object": {
|
|
2498
|
-
return step.filter.typename === null && (step.filter.id === void 0 || step.filter.id.length === 0) && (step.filter.props === void 0 || Object.keys(step.filter.props).length === 0) && (step.filter.foreignKeys === void 0 || step.filter.foreignKeys.length === 0);
|
|
2499
|
-
}
|
|
2500
|
-
default:
|
|
2501
|
-
return false;
|
|
2502
|
-
}
|
|
2503
|
-
}
|
|
2504
|
-
});
|
|
2505
|
-
})(QueryPlan || (QueryPlan = {}));
|
|
2506
|
-
var QueryPlan;
|
|
2507
|
-
|
|
2508
|
-
// packages/core/echo/echo-pipeline/src/query/query-planner.ts
|
|
2509
|
-
var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/query/query-planner.ts";
|
|
2510
|
-
var DEFAULT_OPTIONS = {
|
|
2511
|
-
defaultTextSearchKind: "full-text"
|
|
2512
|
-
};
|
|
2513
|
-
var QueryPlanner = class {
|
|
2514
|
-
constructor(options) {
|
|
2515
|
-
this._options = {
|
|
2516
|
-
...DEFAULT_OPTIONS,
|
|
2517
|
-
...options
|
|
2518
|
-
};
|
|
2519
|
-
}
|
|
2520
|
-
createPlan(query) {
|
|
2521
|
-
let plan = this._generate(query, {
|
|
2522
|
-
...DEFAULT_CONTEXT,
|
|
2523
|
-
originalQuery: query
|
|
2524
|
-
});
|
|
2525
|
-
plan = this._optimizeEmptyFilters(plan);
|
|
2526
|
-
plan = this._optimizeSoloUnions(plan);
|
|
2527
|
-
return plan;
|
|
2528
|
-
}
|
|
2529
|
-
_generate(query, context) {
|
|
2530
|
-
switch (query.type) {
|
|
2531
|
-
case "options":
|
|
2532
|
-
return this._generateOptionsClause(query, context);
|
|
2533
|
-
case "select":
|
|
2534
|
-
return this._generateSelectClause(query, context);
|
|
2535
|
-
case "filter":
|
|
2536
|
-
return this._generateFilterClause(query, context);
|
|
2537
|
-
case "incoming-references":
|
|
2538
|
-
return this._generateIncomingReferencesClause(query, context);
|
|
2539
|
-
case "relation":
|
|
2540
|
-
return this._generateRelationClause(query, context);
|
|
2541
|
-
case "relation-traversal":
|
|
2542
|
-
return this._generateRelationTraversalClause(query, context);
|
|
2543
|
-
case "reference-traversal":
|
|
2544
|
-
return this._generateReferenceTraversalClause(query, context);
|
|
2545
|
-
case "union":
|
|
2546
|
-
return this._generateUnionClause(query, context);
|
|
2547
|
-
case "set-difference":
|
|
2548
|
-
return this._generateSetDifferenceClause(query, context);
|
|
2549
|
-
default:
|
|
2550
|
-
throw new QueryError(`Unsupported query type: ${query.type}`, {
|
|
2551
|
-
context: {
|
|
2552
|
-
query: context.originalQuery
|
|
2553
|
-
}
|
|
2554
|
-
});
|
|
2555
|
-
}
|
|
2556
|
-
}
|
|
2557
|
-
_generateOptionsClause(query, context) {
|
|
2558
|
-
const newContext = {
|
|
2559
|
-
...context
|
|
2560
|
-
};
|
|
2561
|
-
if (query.options.spaceIds) {
|
|
2562
|
-
newContext.selectionSpaces = query.options.spaceIds;
|
|
2563
|
-
}
|
|
2564
|
-
if (query.options.deleted) {
|
|
2565
|
-
newContext.deletedHandling = query.options.deleted;
|
|
2566
|
-
}
|
|
2567
|
-
return this._generate(query.query, newContext);
|
|
2568
|
-
}
|
|
2569
|
-
_generateSelectClause(query, context) {
|
|
2570
|
-
return this._generateSelectionFromFilter(query.filter, context);
|
|
2571
|
-
}
|
|
2572
|
-
// TODO(dmaretskyi): This can be rewritten as a function of (filter[]) -> (selection ? undefined, rest: filter[]) that recurses onto itself.
|
|
2573
|
-
// TODO(dmaretskyi): If the tip of the query ast is a [select, ...filter] shape we can reorder the filters so the query is most efficient.
|
|
2574
|
-
_generateSelectionFromFilter(filter, context) {
|
|
2575
|
-
switch (filter.type) {
|
|
2576
|
-
case "object": {
|
|
2577
|
-
if (context.selectionInverted && filter.id === void 0 && filter.typename === null && Object.keys(filter.props).length === 0) {
|
|
2578
|
-
return QueryPlan.Plan.make([
|
|
2579
|
-
{
|
|
2580
|
-
_tag: "ClearWorkingSetStep"
|
|
2581
|
-
},
|
|
2582
|
-
...this._generateDeletedHandlingSteps(context)
|
|
2583
|
-
]);
|
|
2584
|
-
}
|
|
2585
|
-
if (context.selectionInverted) {
|
|
2586
|
-
throw new QueryError("Query too complex", {
|
|
2587
|
-
context: {
|
|
2588
|
-
query: context.originalQuery
|
|
2589
|
-
}
|
|
2590
|
-
});
|
|
2591
|
-
}
|
|
2592
|
-
if (filter.id && filter.id?.length > 0) {
|
|
2593
|
-
return QueryPlan.Plan.make([
|
|
2594
|
-
{
|
|
2595
|
-
_tag: "SelectStep",
|
|
2596
|
-
spaces: context.selectionSpaces,
|
|
2597
|
-
selector: {
|
|
2598
|
-
_tag: "IdSelector",
|
|
2599
|
-
objectIds: filter.id
|
|
2600
|
-
}
|
|
2601
|
-
},
|
|
2602
|
-
...this._generateDeletedHandlingSteps(context),
|
|
2603
|
-
{
|
|
2604
|
-
_tag: "FilterStep",
|
|
2605
|
-
filter: {
|
|
2606
|
-
...filter,
|
|
2607
|
-
id: void 0
|
|
2608
|
-
}
|
|
2609
|
-
}
|
|
2610
|
-
]);
|
|
2611
|
-
} else if (filter.typename) {
|
|
2612
|
-
return QueryPlan.Plan.make([
|
|
2613
|
-
{
|
|
2614
|
-
_tag: "SelectStep",
|
|
2615
|
-
spaces: context.selectionSpaces,
|
|
2616
|
-
selector: {
|
|
2617
|
-
_tag: "TypeSelector",
|
|
2618
|
-
typename: [
|
|
2619
|
-
filter.typename
|
|
2620
|
-
],
|
|
2621
|
-
inverted: false
|
|
2622
|
-
}
|
|
2623
|
-
},
|
|
2624
|
-
...this._generateDeletedHandlingSteps(context),
|
|
2625
|
-
{
|
|
2626
|
-
_tag: "FilterStep",
|
|
2627
|
-
filter: {
|
|
2628
|
-
...filter,
|
|
2629
|
-
typename: null
|
|
2630
|
-
}
|
|
2631
|
-
}
|
|
2632
|
-
]);
|
|
2633
|
-
} else {
|
|
2634
|
-
return QueryPlan.Plan.make([
|
|
2635
|
-
{
|
|
2636
|
-
_tag: "SelectStep",
|
|
2637
|
-
spaces: context.selectionSpaces,
|
|
2638
|
-
selector: {
|
|
2639
|
-
_tag: "WildcardSelector"
|
|
2640
|
-
}
|
|
2641
|
-
},
|
|
2642
|
-
...this._generateDeletedHandlingSteps(context),
|
|
2643
|
-
{
|
|
2644
|
-
_tag: "FilterStep",
|
|
2645
|
-
filter: {
|
|
2646
|
-
...filter
|
|
2647
|
-
}
|
|
2648
|
-
}
|
|
2649
|
-
]);
|
|
2650
|
-
}
|
|
2651
|
-
}
|
|
2652
|
-
case "text-search": {
|
|
2653
|
-
return QueryPlan.Plan.make([
|
|
2654
|
-
{
|
|
2655
|
-
_tag: "SelectStep",
|
|
2656
|
-
spaces: context.selectionSpaces,
|
|
2657
|
-
selector: {
|
|
2658
|
-
_tag: "TextSelector",
|
|
2659
|
-
text: filter.text,
|
|
2660
|
-
searchKind: filter.searchKind ?? this._options.defaultTextSearchKind
|
|
2661
|
-
}
|
|
2662
|
-
},
|
|
2663
|
-
...this._generateDeletedHandlingSteps(context)
|
|
2664
|
-
]);
|
|
2665
|
-
}
|
|
2666
|
-
case "compare":
|
|
2667
|
-
throw new QueryError("Query too complex", {
|
|
2668
|
-
context: {
|
|
2669
|
-
query: context.originalQuery
|
|
2670
|
-
}
|
|
2671
|
-
});
|
|
2672
|
-
case "in":
|
|
2673
|
-
throw new QueryError("Query too complex", {
|
|
2674
|
-
context: {
|
|
2675
|
-
query: context.originalQuery
|
|
2676
|
-
}
|
|
2677
|
-
});
|
|
2678
|
-
case "range":
|
|
2679
|
-
throw new QueryError("Query too complex", {
|
|
2680
|
-
context: {
|
|
2681
|
-
query: context.originalQuery
|
|
2682
|
-
}
|
|
2683
|
-
});
|
|
2684
|
-
case "not":
|
|
2685
|
-
return this._generateSelectionFromFilter(filter.filter, {
|
|
2686
|
-
...context,
|
|
2687
|
-
selectionInverted: !context.selectionInverted
|
|
2688
|
-
});
|
|
2689
|
-
case "and":
|
|
2690
|
-
throw new QueryError("Query too complex", {
|
|
2691
|
-
context: {
|
|
2692
|
-
query: context.originalQuery
|
|
2693
|
-
}
|
|
2694
|
-
});
|
|
2695
|
-
case "or":
|
|
2696
|
-
if (filter.filters.every(isTrivialTypenameFilter)) {
|
|
2697
|
-
const typenames = filter.filters.map((f) => {
|
|
2698
|
-
invariant9(f.type === "object" && f.typename !== null, void 0, {
|
|
2699
|
-
F: __dxlog_file10,
|
|
2700
|
-
L: 191,
|
|
2701
|
-
S: this,
|
|
2702
|
-
A: [
|
|
2703
|
-
"f.type === 'object' && f.typename !== null",
|
|
2704
|
-
""
|
|
2705
|
-
]
|
|
2706
|
-
});
|
|
2707
|
-
return f.typename;
|
|
2708
|
-
});
|
|
2709
|
-
return QueryPlan.Plan.make([
|
|
2710
|
-
{
|
|
2711
|
-
_tag: "SelectStep",
|
|
2712
|
-
spaces: context.selectionSpaces,
|
|
2713
|
-
selector: {
|
|
2714
|
-
_tag: "TypeSelector",
|
|
2715
|
-
typename: typenames,
|
|
2716
|
-
inverted: context.selectionInverted
|
|
2717
|
-
}
|
|
2718
|
-
},
|
|
2719
|
-
...this._generateDeletedHandlingSteps(context)
|
|
2720
|
-
]);
|
|
2721
|
-
} else {
|
|
2722
|
-
throw new QueryError("Query too complex", {
|
|
2723
|
-
context: {
|
|
2724
|
-
query: context.originalQuery
|
|
2725
|
-
}
|
|
2726
|
-
});
|
|
2727
|
-
}
|
|
2728
|
-
default:
|
|
2729
|
-
throw new QueryError(`Unsupported filter type: ${filter.type}`, {
|
|
2730
|
-
context: {
|
|
2731
|
-
query: context.originalQuery
|
|
2732
|
-
}
|
|
2733
|
-
});
|
|
2734
|
-
}
|
|
2735
|
-
}
|
|
2736
|
-
_generateDeletedHandlingSteps(context) {
|
|
2737
|
-
switch (context.deletedHandling) {
|
|
2738
|
-
case "include":
|
|
2739
|
-
return [];
|
|
2740
|
-
case "exclude":
|
|
2741
|
-
return [
|
|
2742
|
-
{
|
|
2743
|
-
_tag: "FilterDeletedStep",
|
|
2744
|
-
mode: "only-non-deleted"
|
|
2745
|
-
}
|
|
2746
|
-
];
|
|
2747
|
-
case "only":
|
|
2748
|
-
return [
|
|
2749
|
-
{
|
|
2750
|
-
_tag: "FilterDeletedStep",
|
|
2751
|
-
mode: "only-deleted"
|
|
2752
|
-
}
|
|
2753
|
-
];
|
|
2754
|
-
}
|
|
2755
|
-
}
|
|
2756
|
-
_generateUnionClause(query, context) {
|
|
2757
|
-
return QueryPlan.Plan.make([
|
|
2758
|
-
{
|
|
2759
|
-
_tag: "UnionStep",
|
|
2760
|
-
plans: query.queries.map((query2) => this._generate(query2, context))
|
|
2761
|
-
}
|
|
2762
|
-
]);
|
|
2763
|
-
}
|
|
2764
|
-
_generateSetDifferenceClause(query, context) {
|
|
2765
|
-
return QueryPlan.Plan.make([
|
|
2766
|
-
{
|
|
2767
|
-
_tag: "SetDifferenceStep",
|
|
2768
|
-
source: this._generate(query.source, context),
|
|
2769
|
-
exclude: this._generate(query.exclude, context)
|
|
2770
|
-
}
|
|
2771
|
-
]);
|
|
2772
|
-
}
|
|
2773
|
-
_generateReferenceTraversalClause(query, context) {
|
|
2774
|
-
return QueryPlan.Plan.make([
|
|
2775
|
-
...this._generate(query.anchor, context).steps,
|
|
2776
|
-
{
|
|
2777
|
-
_tag: "TraverseStep",
|
|
2778
|
-
traversal: {
|
|
2779
|
-
_tag: "ReferenceTraversal",
|
|
2780
|
-
direction: "outgoing",
|
|
2781
|
-
property: query.property
|
|
2782
|
-
}
|
|
2783
|
-
},
|
|
2784
|
-
...this._generateDeletedHandlingSteps(context)
|
|
2785
|
-
]);
|
|
2786
|
-
}
|
|
2787
|
-
_generateIncomingReferencesClause(query, context) {
|
|
2788
|
-
return QueryPlan.Plan.make([
|
|
2789
|
-
...this._generate(query.anchor, context).steps,
|
|
2790
|
-
{
|
|
2791
|
-
_tag: "TraverseStep",
|
|
2792
|
-
traversal: {
|
|
2793
|
-
_tag: "ReferenceTraversal",
|
|
2794
|
-
direction: "incoming",
|
|
2795
|
-
property: query.property
|
|
2796
|
-
}
|
|
2797
|
-
},
|
|
2798
|
-
...this._generateDeletedHandlingSteps(context),
|
|
2799
|
-
{
|
|
2800
|
-
_tag: "FilterStep",
|
|
2801
|
-
filter: {
|
|
2802
|
-
type: "object",
|
|
2803
|
-
typename: query.typename,
|
|
2804
|
-
props: {}
|
|
2805
|
-
}
|
|
2806
|
-
}
|
|
2807
|
-
]);
|
|
2808
|
-
}
|
|
2809
|
-
_generateRelationTraversalClause(query, context) {
|
|
2810
|
-
switch (query.direction) {
|
|
2811
|
-
case "source": {
|
|
2812
|
-
return QueryPlan.Plan.make([
|
|
2813
|
-
...this._generate(query.anchor, context).steps,
|
|
2814
|
-
createRelationTraversalStep("relation-to-source"),
|
|
2815
|
-
...this._generateDeletedHandlingSteps(context)
|
|
2816
|
-
]);
|
|
2817
|
-
}
|
|
2818
|
-
case "target": {
|
|
2819
|
-
return QueryPlan.Plan.make([
|
|
2820
|
-
...this._generate(query.anchor, context).steps,
|
|
2821
|
-
createRelationTraversalStep("relation-to-target"),
|
|
2822
|
-
...this._generateDeletedHandlingSteps(context)
|
|
2823
|
-
]);
|
|
2824
|
-
}
|
|
2825
|
-
case "both": {
|
|
2826
|
-
const anchorPlan = this._generate(query.anchor, context);
|
|
2827
|
-
return QueryPlan.Plan.make([
|
|
2828
|
-
...anchorPlan.steps,
|
|
2829
|
-
{
|
|
2830
|
-
_tag: "UnionStep",
|
|
2831
|
-
plans: [
|
|
2832
|
-
QueryPlan.Plan.make([
|
|
2833
|
-
createRelationTraversalStep("relation-to-source")
|
|
2834
|
-
]),
|
|
2835
|
-
QueryPlan.Plan.make([
|
|
2836
|
-
createRelationTraversalStep("relation-to-target")
|
|
2837
|
-
])
|
|
2838
|
-
]
|
|
2839
|
-
},
|
|
2840
|
-
...this._generateDeletedHandlingSteps(context)
|
|
2841
|
-
]);
|
|
2842
|
-
}
|
|
2843
|
-
}
|
|
2844
|
-
}
|
|
2845
|
-
_generateRelationClause(query, context) {
|
|
2846
|
-
switch (query.direction) {
|
|
2847
|
-
case "outgoing": {
|
|
2848
|
-
return QueryPlan.Plan.make([
|
|
2849
|
-
...this._generate(query.anchor, context).steps,
|
|
2850
|
-
createRelationTraversalStep("source-to-relation"),
|
|
2851
|
-
...this._generateDeletedHandlingSteps(context),
|
|
2852
|
-
{
|
|
2853
|
-
_tag: "FilterStep",
|
|
2854
|
-
filter: query.filter ?? NOOP_FILTER
|
|
2855
|
-
}
|
|
2856
|
-
]);
|
|
2857
|
-
}
|
|
2858
|
-
case "incoming": {
|
|
2859
|
-
return QueryPlan.Plan.make([
|
|
2860
|
-
...this._generate(query.anchor, context).steps,
|
|
2861
|
-
createRelationTraversalStep("target-to-relation"),
|
|
2862
|
-
...this._generateDeletedHandlingSteps(context),
|
|
2863
|
-
{
|
|
2864
|
-
_tag: "FilterStep",
|
|
2865
|
-
filter: query.filter ?? NOOP_FILTER
|
|
2866
|
-
}
|
|
2867
|
-
]);
|
|
2868
|
-
}
|
|
2869
|
-
case "both": {
|
|
2870
|
-
const anchorPlan = this._generate(query.anchor, context);
|
|
2871
|
-
return QueryPlan.Plan.make([
|
|
2872
|
-
...anchorPlan.steps,
|
|
2873
|
-
{
|
|
2874
|
-
_tag: "UnionStep",
|
|
2875
|
-
plans: [
|
|
2876
|
-
QueryPlan.Plan.make([
|
|
2877
|
-
createRelationTraversalStep("source-to-relation")
|
|
2878
|
-
]),
|
|
2879
|
-
QueryPlan.Plan.make([
|
|
2880
|
-
createRelationTraversalStep("target-to-relation")
|
|
2881
|
-
])
|
|
2882
|
-
]
|
|
2883
|
-
},
|
|
2884
|
-
...this._generateDeletedHandlingSteps(context),
|
|
2885
|
-
{
|
|
2886
|
-
_tag: "FilterStep",
|
|
2887
|
-
filter: query.filter ?? NOOP_FILTER
|
|
2888
|
-
}
|
|
2889
|
-
]);
|
|
2890
|
-
}
|
|
2891
|
-
}
|
|
2892
|
-
}
|
|
2893
|
-
_generateFilterClause(query, context) {
|
|
2894
|
-
return QueryPlan.Plan.make([
|
|
2895
|
-
...this._generate(query.selection, context).steps,
|
|
2896
|
-
{
|
|
2897
|
-
_tag: "FilterStep",
|
|
2898
|
-
filter: query.filter
|
|
2899
|
-
}
|
|
2900
|
-
]);
|
|
2901
|
-
}
|
|
2902
|
-
/**
|
|
2903
|
-
* Removes filter steps that have no predicates.
|
|
2904
|
-
*/
|
|
2905
|
-
_optimizeEmptyFilters(plan) {
|
|
2906
|
-
return QueryPlan.Plan.make(plan.steps.filter((step) => {
|
|
2907
|
-
if (step._tag === "FilterStep") {
|
|
2908
|
-
return !QueryPlan.FilterStep.isNoop(step);
|
|
2909
|
-
} else {
|
|
2910
|
-
return true;
|
|
2911
|
-
}
|
|
2912
|
-
}).map((step) => {
|
|
2913
|
-
if (step._tag === "UnionStep") {
|
|
2914
|
-
return {
|
|
2915
|
-
_tag: "UnionStep",
|
|
2916
|
-
plans: step.plans.map((plan2) => this._optimizeEmptyFilters(plan2))
|
|
2917
|
-
};
|
|
2918
|
-
} else {
|
|
2919
|
-
return step;
|
|
2920
|
-
}
|
|
2921
|
-
}));
|
|
2922
|
-
}
|
|
2923
|
-
/**
|
|
2924
|
-
* Removes union steps that have only one child.
|
|
2925
|
-
*/
|
|
2926
|
-
_optimizeSoloUnions(plan) {
|
|
2927
|
-
return plan;
|
|
2928
|
-
}
|
|
2929
|
-
};
|
|
2930
|
-
var DEFAULT_CONTEXT = {
|
|
2931
|
-
originalQuery: null,
|
|
2932
|
-
selectionSpaces: [],
|
|
2933
|
-
deletedHandling: "exclude",
|
|
2934
|
-
selectionInverted: false
|
|
2935
|
-
};
|
|
2936
|
-
var NOOP_FILTER = {
|
|
2937
|
-
type: "object",
|
|
2938
|
-
typename: null,
|
|
2939
|
-
id: [],
|
|
2940
|
-
props: {}
|
|
2941
|
-
};
|
|
2942
|
-
var createRelationTraversalStep = (direction) => ({
|
|
2943
|
-
_tag: "TraverseStep",
|
|
2944
|
-
traversal: {
|
|
2945
|
-
_tag: "RelationTraversal",
|
|
2946
|
-
direction
|
|
2947
|
-
}
|
|
2948
|
-
});
|
|
2949
|
-
var isTrivialTypenameFilter = (filter) => {
|
|
2950
|
-
return filter.type === "object" && filter.typename !== null && Object.keys(filter.props).length === 0 && (filter.id === void 0 || filter.id.length === 0) && (filter.foreignKeys === void 0 || filter.foreignKeys.length === 0);
|
|
2951
|
-
};
|
|
2952
|
-
|
|
2953
|
-
// packages/core/echo/echo-pipeline/src/query/query-executor.ts
|
|
2954
|
-
var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/query/query-executor.ts";
|
|
2955
|
-
var ExecutionTrace = Object.freeze({
|
|
2956
|
-
makeEmpty: () => ({
|
|
2957
|
-
name: "Empty",
|
|
2958
|
-
details: "",
|
|
2959
|
-
objectCount: 0,
|
|
2960
|
-
documentsLoaded: 0,
|
|
2961
|
-
indexHits: 0,
|
|
2962
|
-
indexQueryTime: 0,
|
|
2963
|
-
documentLoadTime: 0,
|
|
2964
|
-
executionTime: 0,
|
|
2965
|
-
children: []
|
|
2966
|
-
}),
|
|
2967
|
-
format: (trace6) => {
|
|
2968
|
-
const go = (trace7, indent) => {
|
|
2969
|
-
return [
|
|
2970
|
-
`${" ".repeat(indent)} - ${trace7.name}(${trace7.details})`,
|
|
2971
|
-
`${" ".repeat(indent)} objects: ${trace7.objectCount} docs: ${trace7.documentsLoaded} index hits: ${trace7.indexHits} | total: ${trace7.executionTime.toFixed(0)}ms index: ${trace7.indexQueryTime.toFixed(0)}ms load: ${trace7.documentLoadTime.toFixed(0)}ms`,
|
|
2972
|
-
"",
|
|
2973
|
-
...trace7.children.map((child) => go(child, indent + 2))
|
|
2974
|
-
].join("\n");
|
|
2975
|
-
};
|
|
2976
|
-
return go(trace6, 0);
|
|
2977
|
-
}
|
|
2978
|
-
});
|
|
2979
|
-
var TRACE_QUERY_EXECUTION = false;
|
|
2980
|
-
var QueryExecutor = class extends Resource6 {
|
|
2981
|
-
constructor(options) {
|
|
2982
|
-
super();
|
|
2983
|
-
this._trace = ExecutionTrace.makeEmpty();
|
|
2984
|
-
this._lastResultSet = [];
|
|
2985
|
-
this._indexer = options.indexer;
|
|
2986
|
-
this._automergeHost = options.automergeHost;
|
|
2987
|
-
this._spaceStateManager = options.spaceStateManager;
|
|
2988
|
-
this._id = options.queryId;
|
|
2989
|
-
this._query = options.query;
|
|
2990
|
-
this._reactivity = options.reactivity;
|
|
2991
|
-
const queryPlanner = new QueryPlanner();
|
|
2992
|
-
this._plan = queryPlanner.createPlan(this._query);
|
|
2993
|
-
}
|
|
2994
|
-
get query() {
|
|
2995
|
-
return this._query;
|
|
2996
|
-
}
|
|
2997
|
-
get plan() {
|
|
2998
|
-
return this._plan;
|
|
2999
|
-
}
|
|
3000
|
-
get trace() {
|
|
3001
|
-
return this._trace;
|
|
3002
|
-
}
|
|
3003
|
-
async _open(ctx) {
|
|
3004
|
-
}
|
|
3005
|
-
async _close(ctx) {
|
|
3006
|
-
}
|
|
3007
|
-
getResults() {
|
|
3008
|
-
return this._lastResultSet.map((item) => ({
|
|
3009
|
-
id: item.objectId,
|
|
3010
|
-
documentId: item.documentId,
|
|
3011
|
-
spaceId: item.spaceId,
|
|
3012
|
-
// TODO(dmaretskyi): Plumb through the rank.
|
|
3013
|
-
rank: 0
|
|
3014
|
-
}));
|
|
3015
|
-
}
|
|
3016
|
-
async execQuery() {
|
|
3017
|
-
invariant10(this._lifecycleState === LifecycleState3.OPEN, void 0, {
|
|
3018
|
-
F: __dxlog_file11,
|
|
3019
|
-
L: 173,
|
|
3020
|
-
S: this,
|
|
3021
|
-
A: [
|
|
3022
|
-
"this._lifecycleState === LifecycleState.OPEN",
|
|
3023
|
-
""
|
|
3024
|
-
]
|
|
3025
|
-
});
|
|
3026
|
-
const prevResultSet = this._lastResultSet;
|
|
3027
|
-
const { workingSet, trace: trace6 } = await this._execPlan(this._plan, []);
|
|
3028
|
-
this._lastResultSet = workingSet;
|
|
3029
|
-
trace6.name = "Root";
|
|
3030
|
-
trace6.details = JSON.stringify({
|
|
3031
|
-
id: this._id
|
|
3032
|
-
});
|
|
3033
|
-
this._trace = trace6;
|
|
3034
|
-
const changed = prevResultSet.length !== workingSet.length || prevResultSet.some((item, index) => workingSet[index].objectId !== item.objectId || workingSet[index].spaceId !== item.spaceId || workingSet[index].documentId !== item.documentId);
|
|
3035
|
-
if (TRACE_QUERY_EXECUTION) {
|
|
3036
|
-
console.log(ExecutionTrace.format(trace6));
|
|
3037
|
-
}
|
|
3038
|
-
return {
|
|
3039
|
-
changed
|
|
3040
|
-
};
|
|
3041
|
-
}
|
|
3042
|
-
async _execPlan(plan, workingSet) {
|
|
3043
|
-
const trace6 = ExecutionTrace.makeEmpty();
|
|
3044
|
-
const begin = performance.now();
|
|
3045
|
-
for (const step of plan.steps) {
|
|
3046
|
-
if (this._ctx.disposed) {
|
|
3047
|
-
throw new ContextDisposedError();
|
|
3048
|
-
}
|
|
3049
|
-
const result = await this._execStep(step, workingSet);
|
|
3050
|
-
workingSet = result.workingSet;
|
|
3051
|
-
trace6.children.push(result.trace);
|
|
3052
|
-
}
|
|
3053
|
-
trace6.objectCount = workingSet.length;
|
|
3054
|
-
trace6.executionTime = performance.now() - begin;
|
|
3055
|
-
return {
|
|
3056
|
-
workingSet,
|
|
3057
|
-
trace: trace6
|
|
3058
|
-
};
|
|
3059
|
-
}
|
|
3060
|
-
async _execStep(step, workingSet) {
|
|
3061
|
-
if (this._ctx.disposed) {
|
|
3062
|
-
return {
|
|
3063
|
-
workingSet,
|
|
3064
|
-
trace: ExecutionTrace.makeEmpty()
|
|
3065
|
-
};
|
|
3066
|
-
}
|
|
3067
|
-
let newWorkingSet, trace6;
|
|
3068
|
-
const begin = performance.now();
|
|
3069
|
-
switch (step._tag) {
|
|
3070
|
-
case "ClearWorkingSetStep":
|
|
3071
|
-
newWorkingSet = [];
|
|
3072
|
-
trace6 = ExecutionTrace.makeEmpty();
|
|
3073
|
-
break;
|
|
3074
|
-
case "SelectStep":
|
|
3075
|
-
({ workingSet: newWorkingSet, trace: trace6 } = await this._execSelectStep(step, workingSet));
|
|
3076
|
-
break;
|
|
3077
|
-
case "FilterStep":
|
|
3078
|
-
({ workingSet: newWorkingSet, trace: trace6 } = await this._execFilterStep(step, workingSet));
|
|
3079
|
-
break;
|
|
3080
|
-
case "FilterDeletedStep":
|
|
3081
|
-
({ workingSet: newWorkingSet, trace: trace6 } = await this._execFilterDeletedStep(step, workingSet));
|
|
3082
|
-
break;
|
|
3083
|
-
case "UnionStep":
|
|
3084
|
-
({ workingSet: newWorkingSet, trace: trace6 } = await this._execUnionStep(step, workingSet));
|
|
3085
|
-
break;
|
|
3086
|
-
case "SetDifferenceStep":
|
|
3087
|
-
({ workingSet: newWorkingSet, trace: trace6 } = await this._execSetDifferenceStep(step, workingSet));
|
|
3088
|
-
break;
|
|
3089
|
-
case "TraverseStep":
|
|
3090
|
-
({ workingSet: newWorkingSet, trace: trace6 } = await this._execTraverseStep(step, workingSet));
|
|
3091
|
-
break;
|
|
3092
|
-
default:
|
|
3093
|
-
throw new Error(`Unknown step type: ${step._tag}`);
|
|
3094
|
-
}
|
|
3095
|
-
trace6.executionTime = performance.now() - begin;
|
|
3096
|
-
return {
|
|
3097
|
-
workingSet: newWorkingSet,
|
|
3098
|
-
trace: trace6
|
|
3099
|
-
};
|
|
3100
|
-
}
|
|
3101
|
-
async _execSelectStep(step, workingSet) {
|
|
3102
|
-
workingSet = [
|
|
3103
|
-
...workingSet
|
|
3104
|
-
];
|
|
3105
|
-
const trace6 = {
|
|
3106
|
-
...ExecutionTrace.makeEmpty(),
|
|
3107
|
-
name: "Select",
|
|
3108
|
-
details: JSON.stringify(step.selector)
|
|
3109
|
-
};
|
|
3110
|
-
switch (step.selector._tag) {
|
|
3111
|
-
case "WildcardSelector": {
|
|
3112
|
-
const beginIndexQuery = performance.now();
|
|
3113
|
-
const indexHits = await this._indexer.execQuery({
|
|
3114
|
-
typenames: [],
|
|
3115
|
-
inverted: false
|
|
3116
|
-
});
|
|
3117
|
-
trace6.indexHits = +indexHits.length;
|
|
3118
|
-
trace6.indexQueryTime += performance.now() - beginIndexQuery;
|
|
3119
|
-
if (this._ctx.disposed) {
|
|
3120
|
-
return {
|
|
3121
|
-
workingSet,
|
|
3122
|
-
trace: trace6
|
|
3123
|
-
};
|
|
3124
|
-
}
|
|
3125
|
-
const documentLoadStart = performance.now();
|
|
3126
|
-
const results = await this._loadDocumentsAfterIndexQuery(indexHits);
|
|
3127
|
-
trace6.documentsLoaded += results.length;
|
|
3128
|
-
trace6.documentLoadTime += performance.now() - documentLoadStart;
|
|
3129
|
-
workingSet.push(...results.filter(isNonNullable2).filter((item) => step.spaces.includes(item.spaceId)));
|
|
3130
|
-
trace6.objectCount = workingSet.length;
|
|
3131
|
-
break;
|
|
3132
|
-
}
|
|
3133
|
-
case "IdSelector": {
|
|
3134
|
-
const beginLoad = performance.now();
|
|
3135
|
-
const items = await Promise.all(step.selector.objectIds.map((id) => this._loadFromDXN(DXN.fromLocalObjectId(id), {
|
|
3136
|
-
sourceSpaceId: step.spaces[0]
|
|
3137
|
-
})));
|
|
3138
|
-
trace6.documentLoadTime += performance.now() - beginLoad;
|
|
3139
|
-
workingSet.push(...items.filter(isNonNullable2));
|
|
3140
|
-
trace6.objectCount = workingSet.length;
|
|
3141
|
-
break;
|
|
3142
|
-
}
|
|
3143
|
-
case "TypeSelector": {
|
|
3144
|
-
const beginIndexQuery = performance.now();
|
|
3145
|
-
const indexHits = await this._indexer.execQuery({
|
|
3146
|
-
typenames: step.selector.typename,
|
|
3147
|
-
inverted: step.selector.inverted
|
|
3148
|
-
});
|
|
3149
|
-
trace6.indexHits = +indexHits.length;
|
|
3150
|
-
trace6.indexQueryTime += performance.now() - beginIndexQuery;
|
|
3151
|
-
if (this._ctx.disposed) {
|
|
3152
|
-
return {
|
|
3153
|
-
workingSet,
|
|
3154
|
-
trace: trace6
|
|
3155
|
-
};
|
|
3156
|
-
}
|
|
3157
|
-
const documentLoadStart = performance.now();
|
|
3158
|
-
const results = await this._loadDocumentsAfterIndexQuery(indexHits);
|
|
3159
|
-
trace6.documentsLoaded += results.length;
|
|
3160
|
-
trace6.documentLoadTime += performance.now() - documentLoadStart;
|
|
3161
|
-
workingSet.push(...results.filter(isNonNullable2).filter((item) => step.spaces.includes(item.spaceId)));
|
|
3162
|
-
trace6.objectCount = workingSet.length;
|
|
3163
|
-
break;
|
|
3164
|
-
}
|
|
3165
|
-
case "TextSelector": {
|
|
3166
|
-
const beginIndexQuery = performance.now();
|
|
3167
|
-
const indexHits = await this._indexer.execQuery({
|
|
3168
|
-
typenames: [],
|
|
3169
|
-
text: {
|
|
3170
|
-
query: step.selector.text,
|
|
3171
|
-
kind: Match.type().pipe(Match.withReturnType(), Match.when("full-text", () => "text"), Match.when("vector", () => "vector"), Match.orElseAbsurd)(step.selector.searchKind)
|
|
3172
|
-
}
|
|
3173
|
-
});
|
|
3174
|
-
trace6.indexHits = +indexHits.length;
|
|
3175
|
-
trace6.indexQueryTime += performance.now() - beginIndexQuery;
|
|
3176
|
-
if (this._ctx.disposed) {
|
|
3177
|
-
return {
|
|
3178
|
-
workingSet,
|
|
3179
|
-
trace: trace6
|
|
3180
|
-
};
|
|
3181
|
-
}
|
|
3182
|
-
const documentLoadStart = performance.now();
|
|
3183
|
-
const results = await this._loadDocumentsAfterIndexQuery(indexHits);
|
|
3184
|
-
trace6.documentsLoaded += results.length;
|
|
3185
|
-
trace6.documentLoadTime += performance.now() - documentLoadStart;
|
|
3186
|
-
workingSet.push(...results.filter(isNonNullable2).filter((item) => step.spaces.includes(item.spaceId)));
|
|
3187
|
-
trace6.objectCount = workingSet.length;
|
|
3188
|
-
break;
|
|
3189
|
-
}
|
|
3190
|
-
default:
|
|
3191
|
-
throw new Error(`Unknown selector type: ${step.selector._tag}`);
|
|
3192
|
-
}
|
|
3193
|
-
return {
|
|
3194
|
-
workingSet,
|
|
3195
|
-
trace: trace6
|
|
3196
|
-
};
|
|
3197
|
-
}
|
|
3198
|
-
async _execFilterStep(step, workingSet) {
|
|
3199
|
-
const result = workingSet.filter((item) => filterMatchObject(step.filter, {
|
|
3200
|
-
id: item.objectId,
|
|
3201
|
-
spaceId: item.spaceId,
|
|
3202
|
-
doc: item.doc
|
|
3203
|
-
}));
|
|
3204
|
-
return {
|
|
3205
|
-
workingSet: result,
|
|
3206
|
-
trace: {
|
|
3207
|
-
...ExecutionTrace.makeEmpty(),
|
|
3208
|
-
name: "Filter",
|
|
3209
|
-
details: JSON.stringify(step.filter),
|
|
3210
|
-
objectCount: result.length
|
|
3211
|
-
}
|
|
3212
|
-
};
|
|
3213
|
-
}
|
|
3214
|
-
async _execFilterDeletedStep(step, workingSet) {
|
|
3215
|
-
if (workingSet.length === 6) {
|
|
3216
|
-
log9.info("FilterDeletedStep", {
|
|
3217
|
-
step,
|
|
3218
|
-
workingSet
|
|
3219
|
-
}, {
|
|
3220
|
-
F: __dxlog_file11,
|
|
3221
|
-
L: 386,
|
|
3222
|
-
S: this,
|
|
3223
|
-
C: (f, a) => f(...a)
|
|
3224
|
-
});
|
|
3225
|
-
}
|
|
3226
|
-
const expected = step.mode === "only-deleted";
|
|
3227
|
-
const result = workingSet.filter((item) => ObjectStructure.isDeleted(item.doc) === expected);
|
|
3228
|
-
return {
|
|
3229
|
-
workingSet: result,
|
|
3230
|
-
trace: {
|
|
3231
|
-
...ExecutionTrace.makeEmpty(),
|
|
3232
|
-
name: "FilterDeleted",
|
|
3233
|
-
details: step.mode,
|
|
3234
|
-
objectCount: result.length
|
|
3235
|
-
}
|
|
3236
|
-
};
|
|
3237
|
-
}
|
|
3238
|
-
// TODO(dmaretskyi): This needs to be completed.
|
|
3239
|
-
async _execTraverseStep(step, workingSet) {
|
|
3240
|
-
const trace6 = {
|
|
3241
|
-
...ExecutionTrace.makeEmpty(),
|
|
3242
|
-
name: "Traverse",
|
|
3243
|
-
details: JSON.stringify(step.traversal)
|
|
3244
|
-
};
|
|
3245
|
-
const newWorkingSet = [];
|
|
3246
|
-
switch (step.traversal._tag) {
|
|
3247
|
-
case "ReferenceTraversal": {
|
|
3248
|
-
switch (step.traversal.direction) {
|
|
3249
|
-
case "outgoing": {
|
|
3250
|
-
const property = EscapedPropPath.unescape(step.traversal.property);
|
|
3251
|
-
const refs = workingSet.flatMap((item) => {
|
|
3252
|
-
const ref = getDeep(item.doc.data, property);
|
|
3253
|
-
const refs2 = Array.isArray(ref) ? ref : [
|
|
3254
|
-
ref
|
|
3255
|
-
];
|
|
3256
|
-
return refs2.map((ref2) => {
|
|
3257
|
-
try {
|
|
3258
|
-
return isEncodedReference(ref2) ? {
|
|
3259
|
-
ref: DXN.parse(ref2["/"]),
|
|
3260
|
-
spaceId: item.spaceId
|
|
3261
|
-
} : null;
|
|
3262
|
-
} catch {
|
|
3263
|
-
log9.warn("Invalid reference", {
|
|
3264
|
-
ref: ref2["/"]
|
|
3265
|
-
}, {
|
|
3266
|
-
F: __dxlog_file11,
|
|
3267
|
-
L: 431,
|
|
3268
|
-
S: this,
|
|
3269
|
-
C: (f, a) => f(...a)
|
|
3270
|
-
});
|
|
3271
|
-
return null;
|
|
3272
|
-
}
|
|
3273
|
-
});
|
|
3274
|
-
}).filter(isNonNullable2);
|
|
3275
|
-
const beginLoad = performance.now();
|
|
3276
|
-
const items = await Promise.all(refs.map(({ ref, spaceId }) => this._loadFromDXN(ref, {
|
|
3277
|
-
sourceSpaceId: spaceId
|
|
3278
|
-
})));
|
|
3279
|
-
trace6.documentLoadTime += performance.now() - beginLoad;
|
|
3280
|
-
newWorkingSet.push(...items.filter(isNonNullable2));
|
|
3281
|
-
trace6.objectCount = newWorkingSet.length;
|
|
3282
|
-
break;
|
|
3283
|
-
}
|
|
3284
|
-
case "incoming": {
|
|
3285
|
-
const indexHits = await this._indexer.execQuery({
|
|
3286
|
-
typenames: [],
|
|
3287
|
-
inverted: false,
|
|
3288
|
-
graph: {
|
|
3289
|
-
kind: "inbound-reference",
|
|
3290
|
-
property: step.traversal.property,
|
|
3291
|
-
anchors: workingSet.map((item) => item.objectId)
|
|
3292
|
-
}
|
|
3293
|
-
});
|
|
3294
|
-
trace6.indexHits += indexHits.length;
|
|
3295
|
-
const documentLoadStart = performance.now();
|
|
3296
|
-
const results = await this._loadDocumentsAfterIndexQuery(indexHits);
|
|
3297
|
-
trace6.documentsLoaded += results.length;
|
|
3298
|
-
trace6.documentLoadTime += performance.now() - documentLoadStart;
|
|
3299
|
-
newWorkingSet.push(...results.filter(isNonNullable2));
|
|
3300
|
-
trace6.objectCount = newWorkingSet.length;
|
|
3301
|
-
break;
|
|
3302
|
-
}
|
|
3303
|
-
}
|
|
3304
|
-
break;
|
|
3305
|
-
}
|
|
3306
|
-
case "RelationTraversal": {
|
|
3307
|
-
switch (step.traversal.direction) {
|
|
3308
|
-
case "relation-to-source":
|
|
3309
|
-
case "relation-to-target": {
|
|
3310
|
-
const refs = workingSet.map((item) => {
|
|
3311
|
-
const ref = step.traversal.direction === "relation-to-source" ? ObjectStructure.getRelationSource(item.doc) : ObjectStructure.getRelationTarget(item.doc);
|
|
3312
|
-
if (!isEncodedReference(ref)) {
|
|
3313
|
-
return null;
|
|
3314
|
-
}
|
|
3315
|
-
try {
|
|
3316
|
-
return {
|
|
3317
|
-
ref: DXN.parse(ref["/"]),
|
|
3318
|
-
spaceId: item.spaceId
|
|
3319
|
-
};
|
|
3320
|
-
} catch {
|
|
3321
|
-
log9.warn("Invalid reference", {
|
|
3322
|
-
ref: ref["/"]
|
|
3323
|
-
}, {
|
|
3324
|
-
F: __dxlog_file11,
|
|
3325
|
-
L: 494,
|
|
3326
|
-
S: this,
|
|
3327
|
-
C: (f, a) => f(...a)
|
|
3328
|
-
});
|
|
3329
|
-
return null;
|
|
3330
|
-
}
|
|
3331
|
-
}).filter(isNonNullable2);
|
|
3332
|
-
const beginLoad = performance.now();
|
|
3333
|
-
const items = await Promise.all(refs.map(({ ref, spaceId }) => this._loadFromDXN(ref, {
|
|
3334
|
-
sourceSpaceId: spaceId
|
|
3335
|
-
})));
|
|
3336
|
-
trace6.documentLoadTime += performance.now() - beginLoad;
|
|
3337
|
-
newWorkingSet.push(...items.filter(isNonNullable2));
|
|
3338
|
-
trace6.objectCount = newWorkingSet.length;
|
|
3339
|
-
break;
|
|
3340
|
-
}
|
|
3341
|
-
case "source-to-relation":
|
|
3342
|
-
case "target-to-relation": {
|
|
3343
|
-
const indexHits = await this._indexer.execQuery({
|
|
3344
|
-
typenames: [],
|
|
3345
|
-
inverted: false,
|
|
3346
|
-
graph: {
|
|
3347
|
-
kind: step.traversal.direction === "source-to-relation" ? "relation-source" : "relation-target",
|
|
3348
|
-
anchors: workingSet.map((item) => item.objectId),
|
|
3349
|
-
property: null
|
|
3350
|
-
}
|
|
3351
|
-
});
|
|
3352
|
-
trace6.indexHits += indexHits.length;
|
|
3353
|
-
const documentLoadStart = performance.now();
|
|
3354
|
-
const results = await this._loadDocumentsAfterIndexQuery(indexHits);
|
|
3355
|
-
trace6.documentsLoaded += results.length;
|
|
3356
|
-
trace6.documentLoadTime += performance.now() - documentLoadStart;
|
|
3357
|
-
newWorkingSet.push(...results.filter(isNonNullable2));
|
|
3358
|
-
trace6.objectCount = newWorkingSet.length;
|
|
3359
|
-
break;
|
|
3360
|
-
}
|
|
3361
|
-
}
|
|
3362
|
-
break;
|
|
3363
|
-
}
|
|
3364
|
-
default:
|
|
3365
|
-
throw new Error(`Unknown traversal type: ${step.traversal._tag}`);
|
|
3366
|
-
}
|
|
3367
|
-
return {
|
|
3368
|
-
workingSet: newWorkingSet,
|
|
3369
|
-
trace: trace6
|
|
3370
|
-
};
|
|
3371
|
-
}
|
|
3372
|
-
async _execUnionStep(step, workingSet) {
|
|
3373
|
-
const results = /* @__PURE__ */ new Map();
|
|
3374
|
-
const resultSets = await Promise.all(step.plans.map((plan) => this._execPlan(plan, [
|
|
3375
|
-
...workingSet
|
|
3376
|
-
])));
|
|
3377
|
-
const trace6 = {
|
|
3378
|
-
...ExecutionTrace.makeEmpty(),
|
|
3379
|
-
name: "Union"
|
|
3380
|
-
};
|
|
3381
|
-
for (const resultSet of resultSets) {
|
|
3382
|
-
for (const item of resultSet.workingSet) {
|
|
3383
|
-
results.set(`${item.spaceId}:${item.documentId}:${item.objectId}`, item);
|
|
3384
|
-
}
|
|
3385
|
-
trace6.children.push(resultSet.trace);
|
|
3386
|
-
}
|
|
3387
|
-
return {
|
|
3388
|
-
workingSet: [
|
|
3389
|
-
...results.values()
|
|
3390
|
-
],
|
|
3391
|
-
trace: trace6
|
|
3392
|
-
};
|
|
3393
|
-
}
|
|
3394
|
-
async _execSetDifferenceStep(step, workingSet) {
|
|
3395
|
-
const trace6 = {
|
|
3396
|
-
...ExecutionTrace.makeEmpty(),
|
|
3397
|
-
name: "SetDifference"
|
|
3398
|
-
};
|
|
3399
|
-
const sourceResult = await this._execPlan(step.source, [
|
|
3400
|
-
...workingSet
|
|
3401
|
-
]);
|
|
3402
|
-
const excludeResult = await this._execPlan(step.exclude, [
|
|
3403
|
-
...workingSet
|
|
3404
|
-
]);
|
|
3405
|
-
trace6.children.push(sourceResult.trace, excludeResult.trace);
|
|
3406
|
-
return {
|
|
3407
|
-
workingSet: sourceResult.workingSet.filter((item) => {
|
|
3408
|
-
const index = excludeResult.workingSet.findIndex((i) => i.objectId === item.objectId);
|
|
3409
|
-
return index === -1;
|
|
3410
|
-
}),
|
|
3411
|
-
trace: trace6
|
|
3412
|
-
};
|
|
3413
|
-
}
|
|
3414
|
-
async _loadDocumentsAfterIndexQuery(indexHits) {
|
|
3415
|
-
return Promise.all(indexHits.map(async (hit) => {
|
|
3416
|
-
return this._loadFromIndexHit(hit);
|
|
3417
|
-
}));
|
|
3418
|
-
}
|
|
3419
|
-
async _loadFromIndexHit(hit) {
|
|
3420
|
-
const { objectId, documentId, spaceKey: spaceKeyInIndex } = objectPointerCodec3.decode(hit.id);
|
|
3421
|
-
const handle = await this._automergeHost.loadDoc(Context3.default(void 0, {
|
|
3422
|
-
F: __dxlog_file11,
|
|
3423
|
-
L: 604
|
|
3424
|
-
}), documentId);
|
|
3425
|
-
const doc = handle.doc();
|
|
3426
|
-
if (!doc) {
|
|
3427
|
-
return null;
|
|
3428
|
-
}
|
|
3429
|
-
const spaceKey = spaceKeyInIndex ?? DatabaseDirectory3.getSpaceKey(doc);
|
|
3430
|
-
if (!spaceKey) {
|
|
3431
|
-
return null;
|
|
3432
|
-
}
|
|
3433
|
-
const object = DatabaseDirectory3.getInlineObject(doc, objectId);
|
|
3434
|
-
if (!object) {
|
|
3435
|
-
return null;
|
|
3436
|
-
}
|
|
3437
|
-
return {
|
|
3438
|
-
objectId,
|
|
3439
|
-
documentId,
|
|
3440
|
-
spaceId: await createIdFromSpaceKey(PublicKey3.from(spaceKey)),
|
|
3441
|
-
doc: object
|
|
3442
|
-
};
|
|
3443
|
-
}
|
|
3444
|
-
async _loadFromDXN(dxn, { sourceSpaceId }) {
|
|
3445
|
-
const echoDxn = dxn.asEchoDXN();
|
|
3446
|
-
if (!echoDxn) {
|
|
3447
|
-
log9.warn("unable to resolve DXN", {
|
|
3448
|
-
dxn
|
|
3449
|
-
}, {
|
|
3450
|
-
F: __dxlog_file11,
|
|
3451
|
-
L: 631,
|
|
3452
|
-
S: this,
|
|
3453
|
-
C: (f, a) => f(...a)
|
|
3454
|
-
});
|
|
3455
|
-
return null;
|
|
3456
|
-
}
|
|
3457
|
-
const spaceId = echoDxn.spaceId ?? sourceSpaceId;
|
|
3458
|
-
const spaceRoot = this._spaceStateManager.getRootBySpaceId(spaceId);
|
|
3459
|
-
if (!spaceRoot) {
|
|
3460
|
-
log9.warn("no space state found for", {
|
|
3461
|
-
spaceId
|
|
3462
|
-
}, {
|
|
3463
|
-
F: __dxlog_file11,
|
|
3464
|
-
L: 639,
|
|
3465
|
-
S: this,
|
|
3466
|
-
C: (f, a) => f(...a)
|
|
3467
|
-
});
|
|
3468
|
-
return null;
|
|
3469
|
-
}
|
|
3470
|
-
const dbDirectory = spaceRoot.doc();
|
|
3471
|
-
if (!dbDirectory) {
|
|
3472
|
-
log9.warn("no space state found for", {
|
|
3473
|
-
spaceId
|
|
3474
|
-
}, {
|
|
3475
|
-
F: __dxlog_file11,
|
|
3476
|
-
L: 644,
|
|
3477
|
-
S: this,
|
|
3478
|
-
C: (f, a) => f(...a)
|
|
3479
|
-
});
|
|
3480
|
-
return null;
|
|
3481
|
-
}
|
|
3482
|
-
const inlineObject = DatabaseDirectory3.getInlineObject(dbDirectory, echoDxn.echoId);
|
|
3483
|
-
if (inlineObject) {
|
|
3484
|
-
return {
|
|
3485
|
-
objectId: echoDxn.echoId,
|
|
3486
|
-
documentId: spaceRoot.documentId,
|
|
3487
|
-
spaceId,
|
|
3488
|
-
doc: inlineObject
|
|
3489
|
-
};
|
|
3490
|
-
}
|
|
3491
|
-
const link = DatabaseDirectory3.getLink(dbDirectory, echoDxn.echoId);
|
|
3492
|
-
if (!link) {
|
|
3493
|
-
return null;
|
|
3494
|
-
}
|
|
3495
|
-
const handle = await this._automergeHost.loadDoc(Context3.default(void 0, {
|
|
3496
|
-
F: __dxlog_file11,
|
|
3497
|
-
L: 663
|
|
3498
|
-
}), link);
|
|
3499
|
-
const doc = handle.doc();
|
|
3500
|
-
if (!doc) {
|
|
3501
|
-
return null;
|
|
3502
|
-
}
|
|
3503
|
-
const object = DatabaseDirectory3.getInlineObject(doc, echoDxn.echoId);
|
|
3504
|
-
if (!object) {
|
|
3505
|
-
return null;
|
|
3506
|
-
}
|
|
3507
|
-
return {
|
|
3508
|
-
objectId: echoDxn.echoId,
|
|
3509
|
-
documentId: handle.documentId,
|
|
3510
|
-
spaceId,
|
|
3511
|
-
doc: object
|
|
3512
|
-
};
|
|
3513
|
-
}
|
|
3514
|
-
};
|
|
3515
|
-
|
|
3516
|
-
// packages/core/echo/echo-pipeline/src/db-host/query-service.ts
|
|
3517
|
-
function _ts_decorate5(decorators, target, key, desc) {
|
|
3518
|
-
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3519
|
-
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
3520
|
-
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
3521
|
-
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
3522
|
-
}
|
|
3523
|
-
var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/query-service.ts";
|
|
3524
|
-
var QueryServiceImpl = class extends Resource7 {
|
|
3525
|
-
// TODO(burdon): OK for options, but not params. Pass separately and type readonly here.
|
|
3526
|
-
constructor(_params) {
|
|
3527
|
-
super(), this._params = _params, this._queries = /* @__PURE__ */ new Set();
|
|
3528
|
-
trace4.diagnostic({
|
|
3529
|
-
id: "active-queries",
|
|
3530
|
-
name: "Active Queries",
|
|
3531
|
-
fetch: () => {
|
|
3532
|
-
return Array.from(this._queries).map((query) => {
|
|
3533
|
-
return {
|
|
3534
|
-
query: JSON.stringify(query.executor.query),
|
|
3535
|
-
plan: JSON.stringify(query.executor.plan),
|
|
3536
|
-
trace: JSON.stringify(query.executor.trace)
|
|
3537
|
-
};
|
|
3538
|
-
});
|
|
3539
|
-
}
|
|
3540
|
-
});
|
|
3541
|
-
}
|
|
3542
|
-
async _open() {
|
|
3543
|
-
this._params.indexer.updated.on(this._ctx, () => this.invalidateQueries());
|
|
3544
|
-
this._updateQueries = new DeferredTask(this._ctx, this._executeQueries.bind(this));
|
|
3545
|
-
}
|
|
3546
|
-
async _close() {
|
|
3547
|
-
await this._updateQueries.join();
|
|
3548
|
-
await Promise.all(Array.from(this._queries).map((query) => query.close()));
|
|
3549
|
-
}
|
|
3550
|
-
async setConfig(config) {
|
|
3551
|
-
await this._params.indexer.setConfig(config);
|
|
3552
|
-
}
|
|
3553
|
-
execQuery(request) {
|
|
3554
|
-
return new Stream2(({ next, close, ctx }) => {
|
|
3555
|
-
const queryEntry = this._createQuery(ctx, request, next, close, close);
|
|
3556
|
-
scheduleMicroTask(ctx, async () => {
|
|
3557
|
-
await queryEntry.executor.open();
|
|
3558
|
-
queryEntry.open = true;
|
|
3559
|
-
this._updateQueries.schedule();
|
|
3560
|
-
});
|
|
3561
|
-
return queryEntry.close;
|
|
3562
|
-
});
|
|
3563
|
-
}
|
|
3564
|
-
/**
|
|
3565
|
-
* Re-index all loaded documents.
|
|
3566
|
-
*/
|
|
3567
|
-
async reindex() {
|
|
3568
|
-
log10("Reindexing all documents...", void 0, {
|
|
3569
|
-
F: __dxlog_file12,
|
|
3570
|
-
L: 113,
|
|
3571
|
-
S: this,
|
|
3572
|
-
C: (f, a) => f(...a)
|
|
3573
|
-
});
|
|
3574
|
-
const iterator = createDocumentsIterator(this._params.automergeHost);
|
|
3575
|
-
const ids = /* @__PURE__ */ new Map();
|
|
3576
|
-
for await (const documents of iterator()) {
|
|
3577
|
-
for (const { id, heads } of documents) {
|
|
3578
|
-
ids.set(id, heads);
|
|
3579
|
-
}
|
|
3580
|
-
if (ids.size % 100 === 0) {
|
|
3581
|
-
log10("Collected documents...", {
|
|
3582
|
-
count: ids.size
|
|
3583
|
-
}, {
|
|
3584
|
-
F: __dxlog_file12,
|
|
3585
|
-
L: 121,
|
|
3586
|
-
S: this,
|
|
3587
|
-
C: (f, a) => f(...a)
|
|
3588
|
-
});
|
|
3589
|
-
}
|
|
3590
|
-
}
|
|
3591
|
-
log10("Marking all documents as dirty...", {
|
|
3592
|
-
count: ids.size
|
|
3593
|
-
}, {
|
|
3594
|
-
F: __dxlog_file12,
|
|
3595
|
-
L: 125,
|
|
3596
|
-
S: this,
|
|
3597
|
-
C: (f, a) => f(...a)
|
|
3598
|
-
});
|
|
3599
|
-
await this._params.indexer.reindex(ids);
|
|
3600
|
-
}
|
|
3601
|
-
/**
|
|
3602
|
-
* Schedule re-execution of all queries.
|
|
3603
|
-
*/
|
|
3604
|
-
invalidateQueries() {
|
|
3605
|
-
for (const query of this._queries) {
|
|
3606
|
-
query.dirty = true;
|
|
3607
|
-
}
|
|
3608
|
-
this._updateQueries.schedule();
|
|
3609
|
-
}
|
|
3610
|
-
_createQuery(ctx, request, onResults, onError, onClose) {
|
|
3611
|
-
const parsedQuery = QueryAST.Query.pipe(Schema.decodeUnknownSync)(JSON.parse(request.query));
|
|
3612
|
-
const queryEntry = {
|
|
3613
|
-
executor: new QueryExecutor({
|
|
3614
|
-
indexer: this._params.indexer,
|
|
3615
|
-
automergeHost: this._params.automergeHost,
|
|
3616
|
-
queryId: request.queryId ?? raise(new Error("query id required")),
|
|
3617
|
-
query: parsedQuery,
|
|
3618
|
-
reactivity: request.reactivity,
|
|
3619
|
-
spaceStateManager: this._params.spaceStateManager
|
|
3620
|
-
}),
|
|
3621
|
-
dirty: true,
|
|
3622
|
-
open: false,
|
|
3623
|
-
firstResult: true,
|
|
3624
|
-
sendResults: (results) => {
|
|
3625
|
-
if (ctx.disposed) {
|
|
3626
|
-
return;
|
|
3627
|
-
}
|
|
3628
|
-
onResults({
|
|
3629
|
-
queryId: request.queryId,
|
|
3630
|
-
results
|
|
3631
|
-
});
|
|
3632
|
-
},
|
|
3633
|
-
onError,
|
|
3634
|
-
close: async () => {
|
|
3635
|
-
onClose();
|
|
3636
|
-
await queryEntry.executor.close();
|
|
3637
|
-
this._queries.delete(queryEntry);
|
|
3638
|
-
}
|
|
3639
|
-
};
|
|
3640
|
-
this._queries.add(queryEntry);
|
|
3641
|
-
return queryEntry;
|
|
3642
|
-
}
|
|
3643
|
-
async _executeQueries() {
|
|
3644
|
-
const begin = performance.now();
|
|
3645
|
-
let count = 0;
|
|
3646
|
-
await Promise.all(Array.from(this._queries).map(async (query) => {
|
|
3647
|
-
if (!query.dirty || !query.open) {
|
|
3648
|
-
return;
|
|
3649
|
-
}
|
|
3650
|
-
count++;
|
|
3651
|
-
try {
|
|
3652
|
-
const { changed } = await query.executor.execQuery();
|
|
3653
|
-
query.dirty = false;
|
|
3654
|
-
if (changed || query.firstResult) {
|
|
3655
|
-
query.firstResult = false;
|
|
3656
|
-
query.sendResults(query.executor.getResults());
|
|
3657
|
-
}
|
|
3658
|
-
} catch (err) {
|
|
3659
|
-
log10.catch(err, void 0, {
|
|
3660
|
-
F: __dxlog_file12,
|
|
3661
|
-
L: 196,
|
|
3662
|
-
S: this,
|
|
3663
|
-
C: (f, a) => f(...a)
|
|
3664
|
-
});
|
|
3665
|
-
}
|
|
3666
|
-
}));
|
|
3667
|
-
log10.verbose("executed queries", {
|
|
3668
|
-
count,
|
|
3669
|
-
duration: performance.now() - begin
|
|
3670
|
-
}, {
|
|
3671
|
-
F: __dxlog_file12,
|
|
3672
|
-
L: 200,
|
|
3673
|
-
S: this,
|
|
3674
|
-
C: (f, a) => f(...a)
|
|
3675
|
-
});
|
|
3676
|
-
}
|
|
3677
|
-
};
|
|
3678
|
-
_ts_decorate5([
|
|
3679
|
-
synchronized2
|
|
3680
|
-
], QueryServiceImpl.prototype, "_close", null);
|
|
3681
|
-
_ts_decorate5([
|
|
3682
|
-
trace4.span({
|
|
3683
|
-
showInBrowserTimeline: true
|
|
3684
|
-
})
|
|
3685
|
-
], QueryServiceImpl.prototype, "_executeQueries", null);
|
|
3686
|
-
QueryServiceImpl = _ts_decorate5([
|
|
3687
|
-
trace4.resource()
|
|
3688
|
-
], QueryServiceImpl);
|
|
3689
|
-
var createDocumentsIterator = (automergeHost) => (
|
|
3690
|
-
/**
|
|
3691
|
-
* Recursively get all object data blobs from loaded documents from Automerge Repo.
|
|
3692
|
-
*/
|
|
3693
|
-
// TODO(mykola): Unload automerge handles after usage.
|
|
3694
|
-
async function* getAllDocuments() {
|
|
3695
|
-
const visited = /* @__PURE__ */ new Set();
|
|
3696
|
-
async function* getObjectsFromHandle(handle) {
|
|
3697
|
-
if (visited.has(handle.documentId) || !handle.isReady()) {
|
|
3698
|
-
return;
|
|
3699
|
-
}
|
|
3700
|
-
const doc = handle.doc();
|
|
3701
|
-
const spaceKey = DatabaseDirectory4.getSpaceKey(doc) ?? void 0;
|
|
3702
|
-
if (doc.objects) {
|
|
3703
|
-
yield Object.entries(doc.objects).map(([objectId, object]) => {
|
|
3704
|
-
return {
|
|
3705
|
-
id: objectPointerCodec4.encode({
|
|
3706
|
-
documentId: handle.documentId,
|
|
3707
|
-
objectId,
|
|
3708
|
-
spaceKey
|
|
3709
|
-
}),
|
|
3710
|
-
object,
|
|
3711
|
-
heads: getHeads3(doc)
|
|
3712
|
-
};
|
|
3713
|
-
});
|
|
3714
|
-
}
|
|
3715
|
-
if (doc.links) {
|
|
3716
|
-
for (const id of Object.values(doc.links)) {
|
|
3717
|
-
const urlString = id.toString();
|
|
3718
|
-
if (visited.has(urlString)) {
|
|
3719
|
-
continue;
|
|
3720
|
-
}
|
|
3721
|
-
const linkHandle = await automergeHost.loadDoc(Context4.default(void 0, {
|
|
3722
|
-
F: __dxlog_file12,
|
|
3723
|
-
L: 240
|
|
3724
|
-
}), urlString);
|
|
3725
|
-
for await (const result of getObjectsFromHandle(linkHandle)) {
|
|
3726
|
-
yield result;
|
|
3727
|
-
}
|
|
3728
|
-
}
|
|
3729
|
-
}
|
|
3730
|
-
visited.add(handle.documentId);
|
|
3731
|
-
}
|
|
3732
|
-
for (const handle of Object.values(automergeHost.repo.handles)) {
|
|
3733
|
-
if (visited.has(handle.documentId)) {
|
|
3734
|
-
continue;
|
|
3735
|
-
}
|
|
3736
|
-
for await (const result of getObjectsFromHandle(handle)) {
|
|
3737
|
-
yield result;
|
|
3738
|
-
}
|
|
3739
|
-
visited.add(handle.documentId);
|
|
3740
|
-
}
|
|
3741
|
-
}
|
|
3742
|
-
);
|
|
3743
|
-
|
|
3744
|
-
// packages/core/echo/echo-pipeline/src/db-host/space-state-manager.ts
|
|
3745
|
-
import { interpretAsDocumentId as interpretAsDocumentId3 } from "@automerge/automerge-repo";
|
|
3746
|
-
import isEqual from "lodash.isequal";
|
|
3747
|
-
import { Event as Event3, UpdateScheduler as UpdateScheduler3 } from "@dxos/async";
|
|
3748
|
-
import { Resource as Resource8, Context as Context5, LifecycleState as LifecycleState4 } from "@dxos/context";
|
|
3749
|
-
import { invariant as invariant12 } from "@dxos/invariant";
|
|
3750
|
-
|
|
3751
|
-
// packages/core/echo/echo-pipeline/src/db-host/database-root.ts
|
|
3752
|
-
import { interpretAsDocumentId as interpretAsDocumentId2 } from "@automerge/automerge-repo";
|
|
3753
|
-
import { DatabaseDirectory as DatabaseDirectory5, SpaceDocVersion as SpaceDocVersion2 } from "@dxos/echo-protocol";
|
|
3754
|
-
import { invariant as invariant11 } from "@dxos/invariant";
|
|
3755
|
-
|
|
3756
|
-
// packages/core/echo/echo-pipeline/src/db-host/automerge-metrics.ts
|
|
3757
|
-
import * as A4 from "@automerge/automerge";
|
|
3758
|
-
import { log as log11 } from "@dxos/log";
|
|
3759
|
-
var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/automerge-metrics.ts";
|
|
3760
|
-
var measureDocMetrics = (doc) => {
|
|
3761
|
-
const snapshot = A4.save(doc);
|
|
3762
|
-
const start = Date.now();
|
|
3763
|
-
const temp = A4.load(snapshot);
|
|
3764
|
-
const end = Date.now();
|
|
3765
|
-
A4.free(temp);
|
|
3766
|
-
const getAllChangesStart = Date.now();
|
|
3767
|
-
const mutationCount = A4.getAllChanges(doc).length;
|
|
3768
|
-
const getAllChangesEnd = Date.now();
|
|
3769
|
-
if (getAllChangesEnd - getAllChangesStart > 300) {
|
|
3770
|
-
log11.warn("getAllChanges took too long", {
|
|
3771
|
-
elapsed: getAllChangesEnd - getAllChangesStart
|
|
3772
|
-
}, {
|
|
3773
|
-
F: __dxlog_file13,
|
|
3774
|
-
L: 31,
|
|
3775
|
-
S: void 0,
|
|
3776
|
-
C: (f, a) => f(...a)
|
|
3777
|
-
});
|
|
3778
|
-
}
|
|
3779
|
-
return {
|
|
3780
|
-
compressedByteSize: snapshot.byteLength,
|
|
3781
|
-
loadTime: end - start,
|
|
3782
|
-
mutationCount
|
|
3783
|
-
};
|
|
3784
|
-
};
|
|
3785
|
-
|
|
3786
|
-
// packages/core/echo/echo-pipeline/src/db-host/database-root.ts
|
|
3787
|
-
var __dxlog_file14 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/database-root.ts";
|
|
3788
|
-
var DatabaseRoot = class {
|
|
3789
|
-
static mapLinks(doc, mapping) {
|
|
3790
|
-
doc.change((d) => {
|
|
3791
|
-
if (!d.links) {
|
|
3792
|
-
return;
|
|
3793
|
-
}
|
|
3794
|
-
for (const [key, value] of Object.entries(d.links)) {
|
|
3795
|
-
const documentId = interpretAsDocumentId2(value.toString());
|
|
3796
|
-
if (mapping[documentId]) {
|
|
3797
|
-
d.links[key] = `automerge:${mapping[documentId]}`;
|
|
3798
|
-
}
|
|
3799
|
-
}
|
|
3800
|
-
});
|
|
3801
|
-
}
|
|
3802
|
-
constructor(_rootHandle) {
|
|
3803
|
-
this._rootHandle = _rootHandle;
|
|
3804
|
-
}
|
|
3805
|
-
get documentId() {
|
|
3806
|
-
return this._rootHandle.documentId;
|
|
3807
|
-
}
|
|
3808
|
-
get url() {
|
|
3809
|
-
return this._rootHandle.url;
|
|
3810
|
-
}
|
|
3811
|
-
get isLoaded() {
|
|
3812
|
-
return this._rootHandle.isReady();
|
|
3813
|
-
}
|
|
3814
|
-
get handle() {
|
|
3815
|
-
return this._rootHandle;
|
|
3816
|
-
}
|
|
3817
|
-
doc() {
|
|
3818
|
-
return this._rootHandle.isReady() ? this._rootHandle.doc() : null;
|
|
3819
|
-
}
|
|
3820
|
-
getVersion() {
|
|
3821
|
-
const doc = this.doc();
|
|
3822
|
-
if (!doc) {
|
|
3823
|
-
return null;
|
|
3824
|
-
}
|
|
3825
|
-
return doc.version ?? SpaceDocVersion2.LEGACY;
|
|
3826
|
-
}
|
|
3827
|
-
getSpaceKey() {
|
|
3828
|
-
const doc = this.doc();
|
|
3829
|
-
if (!doc) {
|
|
3830
|
-
return null;
|
|
3831
|
-
}
|
|
3832
|
-
return DatabaseDirectory5.getSpaceKey(doc);
|
|
3833
|
-
}
|
|
3834
|
-
getInlineObjectCount() {
|
|
3835
|
-
const doc = this.doc();
|
|
3836
|
-
if (!doc) {
|
|
3837
|
-
return null;
|
|
3838
|
-
}
|
|
3839
|
-
return Object.keys(doc.objects ?? {}).length;
|
|
3840
|
-
}
|
|
3841
|
-
getLinkedObjectCount() {
|
|
3842
|
-
const doc = this.doc();
|
|
3843
|
-
if (!doc) {
|
|
3844
|
-
return null;
|
|
3845
|
-
}
|
|
3846
|
-
return Object.keys(doc.links ?? {}).length;
|
|
3847
|
-
}
|
|
3848
|
-
getAllLinkedDocuments() {
|
|
3849
|
-
const doc = this.doc();
|
|
3850
|
-
invariant11(doc, void 0, {
|
|
3851
|
-
F: __dxlog_file14,
|
|
3852
|
-
L: 88,
|
|
3853
|
-
S: this,
|
|
3854
|
-
A: [
|
|
3855
|
-
"doc",
|
|
3856
|
-
""
|
|
3857
|
-
]
|
|
3858
|
-
});
|
|
3859
|
-
return Object.values(doc.links ?? {}).map((s) => s.toString());
|
|
3860
|
-
}
|
|
3861
|
-
measureMetrics() {
|
|
3862
|
-
const doc = this.doc();
|
|
3863
|
-
if (!doc) {
|
|
3864
|
-
return null;
|
|
3865
|
-
}
|
|
3866
|
-
return measureDocMetrics(doc);
|
|
3867
|
-
}
|
|
3868
|
-
};
|
|
3869
|
-
|
|
3870
|
-
// packages/core/echo/echo-pipeline/src/db-host/space-state-manager.ts
|
|
3871
|
-
var __dxlog_file15 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/space-state-manager.ts";
|
|
3872
|
-
var SpaceStateManager = class extends Resource8 {
|
|
3873
|
-
constructor() {
|
|
3874
|
-
super(...arguments);
|
|
3875
|
-
this._roots = /* @__PURE__ */ new Map();
|
|
3876
|
-
this._rootBySpace = /* @__PURE__ */ new Map();
|
|
3877
|
-
this._perRootContext = /* @__PURE__ */ new Map();
|
|
3878
|
-
this._lastSpaceDocumentList = /* @__PURE__ */ new Map();
|
|
3879
|
-
this.spaceDocumentListUpdated = new Event3();
|
|
3880
|
-
}
|
|
3881
|
-
async _close(ctx) {
|
|
3882
|
-
for (const [_, rootCtx] of this._perRootContext) {
|
|
3883
|
-
await rootCtx.dispose();
|
|
3884
|
-
}
|
|
3885
|
-
this._roots.clear();
|
|
3886
|
-
}
|
|
3887
|
-
get roots() {
|
|
3888
|
-
return this._roots;
|
|
3889
|
-
}
|
|
3890
|
-
getRootByDocumentId(documentId) {
|
|
3891
|
-
return this._roots.get(documentId);
|
|
3892
|
-
}
|
|
3893
|
-
getSpaceRootDocumentId(spaceId) {
|
|
3894
|
-
return this._rootBySpace.get(spaceId);
|
|
3895
|
-
}
|
|
3896
|
-
getRootBySpaceId(spaceId) {
|
|
3897
|
-
invariant12(this._lifecycleState === LifecycleState4.OPEN, void 0, {
|
|
3898
|
-
F: __dxlog_file15,
|
|
3899
|
-
L: 44,
|
|
3900
|
-
S: this,
|
|
3901
|
-
A: [
|
|
3902
|
-
"this._lifecycleState === LifecycleState.OPEN",
|
|
3903
|
-
""
|
|
3904
|
-
]
|
|
3905
|
-
});
|
|
3906
|
-
const documentId = this._rootBySpace.get(spaceId);
|
|
3907
|
-
if (!documentId) {
|
|
3908
|
-
return void 0;
|
|
3909
|
-
}
|
|
3910
|
-
return this._roots.get(documentId);
|
|
3911
|
-
}
|
|
3912
|
-
async assignRootToSpace(spaceId, handle) {
|
|
3913
|
-
let root;
|
|
3914
|
-
if (this._roots.has(handle.documentId)) {
|
|
3915
|
-
root = this._roots.get(handle.documentId);
|
|
3916
|
-
} else {
|
|
3917
|
-
root = new DatabaseRoot(handle);
|
|
3918
|
-
this._roots.set(handle.documentId, root);
|
|
3919
|
-
}
|
|
3920
|
-
if (this._rootBySpace.get(spaceId) === root.handle.documentId) {
|
|
3921
|
-
return root;
|
|
3922
|
-
}
|
|
3923
|
-
const prevRootId = this._rootBySpace.get(spaceId);
|
|
3924
|
-
if (prevRootId) {
|
|
3925
|
-
void this._perRootContext.get(prevRootId)?.dispose();
|
|
3926
|
-
this._perRootContext.delete(prevRootId);
|
|
3927
|
-
}
|
|
3928
|
-
this._rootBySpace.set(spaceId, root.handle.documentId);
|
|
3929
|
-
const ctx = new Context5(void 0, {
|
|
3930
|
-
F: __dxlog_file15,
|
|
3931
|
-
L: 72
|
|
3932
|
-
});
|
|
3933
|
-
this._perRootContext.set(root.handle.documentId, ctx);
|
|
3934
|
-
await root.handle.whenReady();
|
|
3935
|
-
const documentListCheckScheduler = new UpdateScheduler3(ctx, async () => {
|
|
3936
|
-
const documentIds = [
|
|
3937
|
-
root.documentId,
|
|
3938
|
-
...root.getAllLinkedDocuments().map((url) => interpretAsDocumentId3(url))
|
|
3939
|
-
];
|
|
3940
|
-
if (!isEqual(documentIds, this._lastSpaceDocumentList.get(spaceId))) {
|
|
3941
|
-
this._lastSpaceDocumentList.set(spaceId, documentIds);
|
|
3942
|
-
this.spaceDocumentListUpdated.emit(new SpaceDocumentListUpdatedEvent(spaceId, root.documentId, prevRootId, documentIds));
|
|
3943
|
-
}
|
|
3944
|
-
}, {
|
|
3945
|
-
maxFrequency: 50
|
|
3946
|
-
});
|
|
3947
|
-
const triggerCheckOnChange = () => documentListCheckScheduler.trigger();
|
|
3948
|
-
root.handle.addListener("change", triggerCheckOnChange);
|
|
3949
|
-
ctx.onDispose(() => root.handle.removeListener("change", triggerCheckOnChange));
|
|
3950
|
-
documentListCheckScheduler.trigger();
|
|
3951
|
-
return root;
|
|
3952
|
-
}
|
|
3953
|
-
};
|
|
3954
|
-
var SpaceDocumentListUpdatedEvent = class {
|
|
3955
|
-
constructor(spaceId, spaceRootId, previousRootId, documentIds) {
|
|
3956
|
-
this.spaceId = spaceId;
|
|
3957
|
-
this.spaceRootId = spaceRootId;
|
|
3958
|
-
this.previousRootId = previousRootId;
|
|
3959
|
-
this.documentIds = documentIds;
|
|
3960
|
-
}
|
|
3961
|
-
};
|
|
3962
|
-
|
|
3963
|
-
// packages/core/echo/echo-pipeline/src/db-host/echo-host.ts
|
|
3964
|
-
var __dxlog_file16 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/echo-host.ts";
|
|
3965
|
-
var DEFAULT_INDEXING_CONFIG = {
|
|
3966
|
-
// TODO(dmaretskyi): Disabled by default since embedding generation is expensive.
|
|
3967
|
-
fullText: false,
|
|
3968
|
-
vector: false
|
|
3969
|
-
};
|
|
3970
|
-
var EchoHost = class extends Resource9 {
|
|
3971
|
-
constructor({ kv, indexing = {}, peerIdProvider, getSpaceKeyByRootDocumentId }) {
|
|
3972
|
-
super();
|
|
3973
|
-
this._spaceStateManager = new SpaceStateManager();
|
|
3974
|
-
const indexingConfig = {
|
|
3975
|
-
...DEFAULT_INDEXING_CONFIG,
|
|
3976
|
-
...indexing
|
|
3977
|
-
};
|
|
3978
|
-
this._indexMetadataStore = new IndexMetadataStore({
|
|
3979
|
-
db: kv.sublevel("index-metadata")
|
|
3980
|
-
});
|
|
3981
|
-
this._echoDataMonitor = new EchoDataMonitor();
|
|
3982
|
-
this._automergeHost = new AutomergeHost({
|
|
3983
|
-
db: kv,
|
|
3984
|
-
dataMonitor: this._echoDataMonitor,
|
|
3985
|
-
indexMetadataStore: this._indexMetadataStore,
|
|
3986
|
-
peerIdProvider,
|
|
3987
|
-
getSpaceKeyByRootDocumentId
|
|
3988
|
-
});
|
|
3989
|
-
this._indexer = new Indexer({
|
|
3990
|
-
db: kv,
|
|
3991
|
-
indexStore: new IndexStore({
|
|
3992
|
-
db: kv.sublevel("index-storage")
|
|
3993
|
-
}),
|
|
3994
|
-
metadataStore: this._indexMetadataStore,
|
|
3995
|
-
loadDocuments: createSelectedDocumentsIterator(this._automergeHost),
|
|
3996
|
-
indexCooldownTime: false ? 0 : void 0
|
|
3997
|
-
});
|
|
3998
|
-
void this._indexer.setConfig({
|
|
3999
|
-
enabled: true,
|
|
4000
|
-
indexes: [
|
|
4001
|
-
//
|
|
4002
|
-
{
|
|
4003
|
-
kind: IndexKind.Kind.SCHEMA_MATCH
|
|
4004
|
-
},
|
|
4005
|
-
{
|
|
4006
|
-
kind: IndexKind.Kind.GRAPH
|
|
4007
|
-
},
|
|
4008
|
-
...indexingConfig.fullText ? [
|
|
4009
|
-
{
|
|
4010
|
-
kind: IndexKind.Kind.FULL_TEXT
|
|
4011
|
-
}
|
|
4012
|
-
] : [],
|
|
4013
|
-
...indexingConfig.vector ? [
|
|
4014
|
-
{
|
|
4015
|
-
kind: IndexKind.Kind.VECTOR
|
|
4016
|
-
}
|
|
4017
|
-
] : []
|
|
4018
|
-
]
|
|
4019
|
-
});
|
|
4020
|
-
this._queryService = new QueryServiceImpl({
|
|
4021
|
-
automergeHost: this._automergeHost,
|
|
4022
|
-
indexer: this._indexer,
|
|
4023
|
-
spaceStateManager: this._spaceStateManager
|
|
4024
|
-
});
|
|
4025
|
-
this._dataService = new DataServiceImpl({
|
|
4026
|
-
automergeHost: this._automergeHost,
|
|
4027
|
-
spaceStateManager: this._spaceStateManager,
|
|
4028
|
-
updateIndexes: async () => {
|
|
4029
|
-
await this._indexer.updateIndexes();
|
|
4030
|
-
}
|
|
4031
|
-
});
|
|
4032
|
-
trace5.diagnostic({
|
|
4033
|
-
id: "echo-stats",
|
|
4034
|
-
name: "Echo Stats",
|
|
4035
|
-
fetch: async () => {
|
|
4036
|
-
return {
|
|
4037
|
-
dataStats: this._echoDataMonitor.computeStats(),
|
|
4038
|
-
loadedDocsCount: this._automergeHost.loadedDocsCount
|
|
4039
|
-
};
|
|
4040
|
-
}
|
|
4041
|
-
});
|
|
4042
|
-
trace5.diagnostic({
|
|
4043
|
-
id: "database-roots",
|
|
4044
|
-
name: "Database Roots",
|
|
4045
|
-
fetch: async () => {
|
|
4046
|
-
return Array.from(this._spaceStateManager.roots.values()).map((root) => ({
|
|
4047
|
-
url: root.url,
|
|
4048
|
-
isLoaded: root.isLoaded,
|
|
4049
|
-
spaceKey: root.getSpaceKey(),
|
|
4050
|
-
inlineObjects: root.getInlineObjectCount(),
|
|
4051
|
-
linkedObjects: root.getLinkedObjectCount()
|
|
4052
|
-
}));
|
|
4053
|
-
}
|
|
4054
|
-
});
|
|
4055
|
-
trace5.diagnostic({
|
|
4056
|
-
id: "database-root-metrics",
|
|
4057
|
-
name: "Database Roots (with metrics)",
|
|
4058
|
-
fetch: async () => {
|
|
4059
|
-
return Array.from(this._spaceStateManager.roots.values()).map((root) => ({
|
|
4060
|
-
url: root.url,
|
|
4061
|
-
isLoaded: root.isLoaded,
|
|
4062
|
-
spaceKey: root.getSpaceKey(),
|
|
4063
|
-
inlineObjects: root.getInlineObjectCount(),
|
|
4064
|
-
linkedObjects: root.getLinkedObjectCount(),
|
|
4065
|
-
...root.measureMetrics() ?? {}
|
|
4066
|
-
}));
|
|
4067
|
-
}
|
|
4068
|
-
});
|
|
4069
|
-
}
|
|
4070
|
-
get queryService() {
|
|
4071
|
-
return this._queryService;
|
|
4072
|
-
}
|
|
4073
|
-
get dataService() {
|
|
4074
|
-
return this._dataService;
|
|
4075
|
-
}
|
|
4076
|
-
/**
|
|
4077
|
-
* @deprecated To be abstracted away.
|
|
4078
|
-
*/
|
|
4079
|
-
get automergeRepo() {
|
|
4080
|
-
return this._automergeHost.repo;
|
|
4081
|
-
}
|
|
4082
|
-
get roots() {
|
|
4083
|
-
return this._spaceStateManager.roots;
|
|
4084
|
-
}
|
|
4085
|
-
async _open(ctx) {
|
|
4086
|
-
await this._automergeHost.open();
|
|
4087
|
-
await this._indexer.open(ctx);
|
|
4088
|
-
await this._queryService.open(ctx);
|
|
4089
|
-
await this._spaceStateManager.open(ctx);
|
|
4090
|
-
this._spaceStateManager.spaceDocumentListUpdated.on(this._ctx, (e) => {
|
|
4091
|
-
if (e.previousRootId) {
|
|
4092
|
-
void this._automergeHost.clearLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.previousRootId));
|
|
4093
|
-
}
|
|
4094
|
-
void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId), e.documentIds);
|
|
4095
|
-
void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.spaceRootId), e.documentIds);
|
|
4096
|
-
});
|
|
4097
|
-
this._automergeHost.documentsSaved.on(this._ctx, () => {
|
|
4098
|
-
this._queryService.invalidateQueries();
|
|
4099
|
-
});
|
|
4100
|
-
}
|
|
4101
|
-
async _close(ctx) {
|
|
4102
|
-
await this._queryService.close(ctx);
|
|
4103
|
-
await this._spaceStateManager.close(ctx);
|
|
4104
|
-
await this._indexer.close(ctx);
|
|
4105
|
-
await this._automergeHost.close();
|
|
4106
|
-
}
|
|
4107
|
-
/**
|
|
4108
|
-
* Flush all pending writes to the underlying storage.
|
|
4109
|
-
*/
|
|
4110
|
-
async flush() {
|
|
4111
|
-
await this._automergeHost.repo.flush();
|
|
4112
|
-
}
|
|
4113
|
-
/**
|
|
4114
|
-
* Perform any pending index updates.
|
|
4115
|
-
*/
|
|
4116
|
-
async updateIndexes() {
|
|
4117
|
-
await this._indexer.updateIndexes();
|
|
4118
|
-
}
|
|
4119
|
-
/**
|
|
4120
|
-
* Loads the document handle from the repo and waits for it to be ready.
|
|
4121
|
-
*/
|
|
4122
|
-
async loadDoc(ctx, documentId, opts) {
|
|
4123
|
-
return await this._automergeHost.loadDoc(ctx, documentId, opts);
|
|
4124
|
-
}
|
|
4125
|
-
async exportDoc(ctx, id) {
|
|
4126
|
-
return await this._automergeHost.exportDoc(ctx, id);
|
|
4127
|
-
}
|
|
4128
|
-
/**
|
|
4129
|
-
* Create new persisted document.
|
|
4130
|
-
*/
|
|
4131
|
-
createDoc(initialValue, opts) {
|
|
4132
|
-
return this._automergeHost.createDoc(initialValue, opts);
|
|
4133
|
-
}
|
|
4134
|
-
/**
|
|
4135
|
-
* Create new space root.
|
|
4136
|
-
*/
|
|
4137
|
-
async createSpaceRoot(spaceKey) {
|
|
4138
|
-
invariant13(this._lifecycleState === LifecycleState5.OPEN, void 0, {
|
|
4139
|
-
F: __dxlog_file16,
|
|
4140
|
-
L: 255,
|
|
4141
|
-
S: this,
|
|
4142
|
-
A: [
|
|
4143
|
-
"this._lifecycleState === LifecycleState.OPEN",
|
|
4144
|
-
""
|
|
4145
|
-
]
|
|
4146
|
-
});
|
|
4147
|
-
const spaceId = await createIdFromSpaceKey2(spaceKey);
|
|
4148
|
-
const automergeRoot = this._automergeHost.createDoc({
|
|
4149
|
-
version: SpaceDocVersion3.CURRENT,
|
|
4150
|
-
access: {
|
|
4151
|
-
spaceKey: spaceKey.toHex()
|
|
4152
|
-
},
|
|
4153
|
-
// Better to initialize them right away to avoid merge conflicts and data loss that can occur if those maps get created on the fly.
|
|
4154
|
-
objects: {},
|
|
4155
|
-
links: {}
|
|
4156
|
-
});
|
|
4157
|
-
await this._automergeHost.flush({
|
|
4158
|
-
documentIds: [
|
|
4159
|
-
automergeRoot.documentId
|
|
4160
|
-
]
|
|
4161
|
-
});
|
|
4162
|
-
return await this.openSpaceRoot(spaceId, automergeRoot.url);
|
|
4163
|
-
}
|
|
4164
|
-
// TODO(dmaretskyi): Change to document id.
|
|
4165
|
-
async openSpaceRoot(spaceId, automergeUrl) {
|
|
4166
|
-
invariant13(this._lifecycleState === LifecycleState5.OPEN, void 0, {
|
|
4167
|
-
F: __dxlog_file16,
|
|
4168
|
-
L: 274,
|
|
4169
|
-
S: this,
|
|
4170
|
-
A: [
|
|
4171
|
-
"this._lifecycleState === LifecycleState.OPEN",
|
|
4172
|
-
""
|
|
4173
|
-
]
|
|
4174
|
-
});
|
|
4175
|
-
const handle = await this._automergeHost.repo.find(automergeUrl, FIND_PARAMS);
|
|
4176
|
-
await handle.whenReady();
|
|
4177
|
-
return this._spaceStateManager.assignRootToSpace(spaceId, handle);
|
|
4178
|
-
}
|
|
4179
|
-
// TODO(dmaretskyi): Change to document id.
|
|
4180
|
-
async closeSpaceRoot(automergeUrl) {
|
|
4181
|
-
todo();
|
|
4182
|
-
}
|
|
4183
|
-
/**
|
|
4184
|
-
* Install data replicator.
|
|
4185
|
-
*/
|
|
4186
|
-
async addReplicator(replicator) {
|
|
4187
|
-
await this._automergeHost.addReplicator(replicator);
|
|
4188
|
-
}
|
|
4189
|
-
/**
|
|
4190
|
-
* Remove data replicator.
|
|
4191
|
-
*/
|
|
4192
|
-
async removeReplicator(replicator) {
|
|
4193
|
-
await this._automergeHost.removeReplicator(replicator);
|
|
4194
|
-
}
|
|
4195
|
-
};
|
|
4196
|
-
|
|
4197
|
-
// packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts
|
|
4198
|
-
import { cbor as cbor2 } from "@automerge/automerge-repo";
|
|
4199
|
-
import { Mutex, scheduleTask as scheduleTask2, scheduleMicroTask as scheduleMicroTask2 } from "@dxos/async";
|
|
4200
|
-
import { Context as Context6, Resource as Resource11 } from "@dxos/context";
|
|
4201
|
-
import { randomUUID } from "@dxos/crypto";
|
|
4202
|
-
import { invariant as invariant14 } from "@dxos/invariant";
|
|
4203
|
-
import { log as log13 } from "@dxos/log";
|
|
4204
|
-
import { EdgeService } from "@dxos/protocols";
|
|
4205
|
-
import { buf } from "@dxos/protocols/buf";
|
|
4206
|
-
import { MessageSchema as RouterMessageSchema } from "@dxos/protocols/buf/dxos/edge/messenger_pb";
|
|
4207
|
-
import { bufferToArray as bufferToArray2 } from "@dxos/util";
|
|
4208
|
-
|
|
4209
|
-
// packages/core/echo/echo-pipeline/src/edge/inflight-request-limiter.ts
|
|
4210
|
-
import { Trigger as Trigger2 } from "@dxos/async";
|
|
4211
|
-
import { Resource as Resource10 } from "@dxos/context";
|
|
4212
|
-
import { log as log12 } from "@dxos/log";
|
|
4213
|
-
var __dxlog_file17 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/inflight-request-limiter.ts";
|
|
4214
|
-
var InflightRequestLimiter = class extends Resource10 {
|
|
4215
|
-
constructor(_config) {
|
|
4216
|
-
super(), this._config = _config, this._inflightRequestBalance = 0, this._requestBarrier = new Trigger2();
|
|
4217
|
-
}
|
|
4218
|
-
async _open() {
|
|
4219
|
-
this._inflightRequestBalance = 0;
|
|
4220
|
-
this._requestBarrier.reset();
|
|
4221
|
-
this._requestBarrier.wake();
|
|
4222
|
-
}
|
|
4223
|
-
async _close() {
|
|
4224
|
-
this._inflightRequestBalance = 0;
|
|
4225
|
-
this._requestBarrier.throw(new Error("Rate limiter closed."));
|
|
4226
|
-
clearTimeout(this._resetBalanceTimeout);
|
|
4227
|
-
}
|
|
4228
|
-
async rateLimit(message) {
|
|
4229
|
-
if (message.type !== "sync") {
|
|
4230
|
-
return;
|
|
4231
|
-
}
|
|
4232
|
-
while (this._inflightRequestBalance >= this._config.maxInflightRequests) {
|
|
4233
|
-
await this._requestBarrier.wait();
|
|
4234
|
-
}
|
|
4235
|
-
this._inflightRequestBalance++;
|
|
4236
|
-
if (this._inflightRequestBalance === this._config.maxInflightRequests) {
|
|
4237
|
-
this._requestBarrier.reset();
|
|
4238
|
-
this._resetBalanceTimeout = setTimeout(() => {
|
|
4239
|
-
log12.warn("Request balance has not changed during specified timeout, resetting request limiter.", void 0, {
|
|
4240
|
-
F: __dxlog_file17,
|
|
4241
|
-
L: 52,
|
|
4242
|
-
S: this,
|
|
4243
|
-
C: (f, a) => f(...a)
|
|
4244
|
-
});
|
|
4245
|
-
this._inflightRequestBalance = 0;
|
|
4246
|
-
this._requestBarrier.wake();
|
|
4247
|
-
}, this._config.resetBalanceTimeoutMs);
|
|
4248
|
-
}
|
|
4249
|
-
}
|
|
4250
|
-
handleResponse(message) {
|
|
4251
|
-
if (message.type !== "sync") {
|
|
4252
|
-
return;
|
|
4253
|
-
}
|
|
4254
|
-
this._inflightRequestBalance--;
|
|
4255
|
-
if (this._inflightRequestBalance + 1 === this._config.maxInflightRequests) {
|
|
4256
|
-
this._requestBarrier.wake();
|
|
4257
|
-
clearInterval(this._resetBalanceTimeout);
|
|
4258
|
-
}
|
|
4259
|
-
}
|
|
4260
|
-
};
|
|
4261
|
-
|
|
4262
|
-
// packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts
|
|
4263
|
-
function _ts_add_disposable_resource(env, value, async) {
|
|
4264
|
-
if (value !== null && value !== void 0) {
|
|
4265
|
-
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4266
|
-
var dispose, inner;
|
|
4267
|
-
if (async) {
|
|
4268
|
-
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
4269
|
-
dispose = value[Symbol.asyncDispose];
|
|
4270
|
-
}
|
|
4271
|
-
if (dispose === void 0) {
|
|
4272
|
-
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
4273
|
-
dispose = value[Symbol.dispose];
|
|
4274
|
-
if (async) inner = dispose;
|
|
4275
|
-
}
|
|
4276
|
-
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
4277
|
-
if (inner) dispose = function() {
|
|
4278
|
-
try {
|
|
4279
|
-
inner.call(this);
|
|
4280
|
-
} catch (e) {
|
|
4281
|
-
return Promise.reject(e);
|
|
4282
|
-
}
|
|
4283
|
-
};
|
|
4284
|
-
env.stack.push({
|
|
4285
|
-
value,
|
|
4286
|
-
dispose,
|
|
4287
|
-
async
|
|
4288
|
-
});
|
|
4289
|
-
} else if (async) {
|
|
4290
|
-
env.stack.push({
|
|
4291
|
-
async: true
|
|
4292
|
-
});
|
|
4293
|
-
}
|
|
4294
|
-
return value;
|
|
4295
|
-
}
|
|
4296
|
-
function _ts_dispose_resources(env) {
|
|
4297
|
-
var _SuppressedError = typeof SuppressedError === "function" ? SuppressedError : function(error, suppressed, message) {
|
|
4298
|
-
var e = new Error(message);
|
|
4299
|
-
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
4300
|
-
};
|
|
4301
|
-
return (_ts_dispose_resources = function _ts_dispose_resources2(env2) {
|
|
4302
|
-
function fail(e) {
|
|
4303
|
-
env2.error = env2.hasError ? new _SuppressedError(e, env2.error, "An error was suppressed during disposal.") : e;
|
|
4304
|
-
env2.hasError = true;
|
|
4305
|
-
}
|
|
4306
|
-
var r, s = 0;
|
|
4307
|
-
function next() {
|
|
4308
|
-
while (r = env2.stack.pop()) {
|
|
4309
|
-
try {
|
|
4310
|
-
if (!r.async && s === 1) return s = 0, env2.stack.push(r), Promise.resolve().then(next);
|
|
4311
|
-
if (r.dispose) {
|
|
4312
|
-
var result = r.dispose.call(r.value);
|
|
4313
|
-
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) {
|
|
4314
|
-
fail(e);
|
|
4315
|
-
return next();
|
|
4316
|
-
});
|
|
4317
|
-
} else s |= 1;
|
|
4318
|
-
} catch (e) {
|
|
4319
|
-
fail(e);
|
|
4320
|
-
}
|
|
4321
|
-
}
|
|
4322
|
-
if (s === 1) return env2.hasError ? Promise.reject(env2.error) : Promise.resolve();
|
|
4323
|
-
if (env2.hasError) throw env2.error;
|
|
4324
|
-
}
|
|
4325
|
-
return next();
|
|
4326
|
-
})(env);
|
|
4327
|
-
}
|
|
4328
|
-
var __dxlog_file18 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/edge/echo-edge-replicator.ts";
|
|
4329
|
-
var INITIAL_RESTART_DELAY = 500;
|
|
4330
|
-
var RESTART_DELAY_JITTER = 250;
|
|
4331
|
-
var MAX_RESTART_DELAY = 5e3;
|
|
4332
|
-
var EchoEdgeReplicator = class {
|
|
4333
|
-
constructor({ edgeConnection, disableSharePolicy }) {
|
|
4334
|
-
this._mutex = new Mutex();
|
|
4335
|
-
this._ctx = void 0;
|
|
4336
|
-
this._context = null;
|
|
4337
|
-
this._connectedSpaces = /* @__PURE__ */ new Set();
|
|
4338
|
-
this._connections = /* @__PURE__ */ new Map();
|
|
4339
|
-
this._sharePolicyEnabled = true;
|
|
4340
|
-
this._edgeConnection = edgeConnection;
|
|
4341
|
-
this._sharePolicyEnabled = !disableSharePolicy;
|
|
4342
|
-
}
|
|
4343
|
-
async connect(context) {
|
|
4344
|
-
log13("connecting...", {
|
|
4345
|
-
peerId: context.peerId,
|
|
4346
|
-
connectedSpaces: this._connectedSpaces.size
|
|
4347
|
-
}, {
|
|
4348
|
-
F: __dxlog_file18,
|
|
4349
|
-
L: 61,
|
|
4350
|
-
S: this,
|
|
4351
|
-
C: (f, a) => f(...a)
|
|
4352
|
-
});
|
|
4353
|
-
this._context = context;
|
|
4354
|
-
this._ctx = Context6.default(void 0, {
|
|
4355
|
-
F: __dxlog_file18,
|
|
4356
|
-
L: 63
|
|
4357
|
-
});
|
|
4358
|
-
this._ctx.onDispose(this._edgeConnection.onReconnected(() => {
|
|
4359
|
-
this._ctx && scheduleMicroTask2(this._ctx, () => this._handleReconnect());
|
|
4360
|
-
}));
|
|
4361
|
-
}
|
|
4362
|
-
async _handleReconnect() {
|
|
4363
|
-
const env = {
|
|
4364
|
-
stack: [],
|
|
4365
|
-
error: void 0,
|
|
4366
|
-
hasError: false
|
|
4367
|
-
};
|
|
4368
|
-
try {
|
|
4369
|
-
const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
|
|
4370
|
-
const spaces = [
|
|
4371
|
-
...this._connectedSpaces
|
|
4372
|
-
];
|
|
4373
|
-
for (const connection of this._connections.values()) {
|
|
4374
|
-
await connection.close();
|
|
4375
|
-
}
|
|
4376
|
-
this._connections.clear();
|
|
4377
|
-
if (this._context !== null) {
|
|
4378
|
-
for (const spaceId of spaces) {
|
|
4379
|
-
await this._openConnection(spaceId);
|
|
4380
|
-
}
|
|
4381
|
-
}
|
|
4382
|
-
} catch (e) {
|
|
4383
|
-
env.error = e;
|
|
4384
|
-
env.hasError = true;
|
|
4385
|
-
} finally {
|
|
4386
|
-
_ts_dispose_resources(env);
|
|
4387
|
-
}
|
|
4388
|
-
}
|
|
4389
|
-
async disconnect() {
|
|
4390
|
-
const env = {
|
|
4391
|
-
stack: [],
|
|
4392
|
-
error: void 0,
|
|
4393
|
-
hasError: false
|
|
4394
|
-
};
|
|
4395
|
-
try {
|
|
4396
|
-
const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
|
|
4397
|
-
await this._ctx?.dispose();
|
|
4398
|
-
for (const connection of this._connections.values()) {
|
|
4399
|
-
await connection.close();
|
|
4400
|
-
}
|
|
4401
|
-
this._connections.clear();
|
|
4402
|
-
} catch (e) {
|
|
4403
|
-
env.error = e;
|
|
4404
|
-
env.hasError = true;
|
|
4405
|
-
} finally {
|
|
4406
|
-
_ts_dispose_resources(env);
|
|
4407
|
-
}
|
|
4408
|
-
}
|
|
4409
|
-
async connectToSpace(spaceId) {
|
|
4410
|
-
const env = {
|
|
4411
|
-
stack: [],
|
|
4412
|
-
error: void 0,
|
|
4413
|
-
hasError: false
|
|
4414
|
-
};
|
|
4415
|
-
try {
|
|
4416
|
-
const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
|
|
4417
|
-
if (this._connectedSpaces.has(spaceId)) {
|
|
4418
|
-
return;
|
|
4419
|
-
}
|
|
4420
|
-
this._connectedSpaces.add(spaceId);
|
|
4421
|
-
if (this._context !== null) {
|
|
4422
|
-
await this._openConnection(spaceId);
|
|
4423
|
-
}
|
|
4424
|
-
} catch (e) {
|
|
4425
|
-
env.error = e;
|
|
4426
|
-
env.hasError = true;
|
|
4427
|
-
} finally {
|
|
4428
|
-
_ts_dispose_resources(env);
|
|
4429
|
-
}
|
|
4430
|
-
}
|
|
4431
|
-
async disconnectFromSpace(spaceId) {
|
|
4432
|
-
const env = {
|
|
4433
|
-
stack: [],
|
|
4434
|
-
error: void 0,
|
|
4435
|
-
hasError: false
|
|
4436
|
-
};
|
|
4437
|
-
try {
|
|
4438
|
-
const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
|
|
4439
|
-
this._connectedSpaces.delete(spaceId);
|
|
4440
|
-
const connection = this._connections.get(spaceId);
|
|
4441
|
-
if (connection) {
|
|
4442
|
-
await connection.close();
|
|
4443
|
-
this._connections.delete(spaceId);
|
|
4444
|
-
}
|
|
4445
|
-
} catch (e) {
|
|
4446
|
-
env.error = e;
|
|
4447
|
-
env.hasError = true;
|
|
4448
|
-
} finally {
|
|
4449
|
-
_ts_dispose_resources(env);
|
|
4450
|
-
}
|
|
4451
|
-
}
|
|
4452
|
-
async _openConnection(spaceId, reconnects = 0) {
|
|
4453
|
-
invariant14(this._context, void 0, {
|
|
4454
|
-
F: __dxlog_file18,
|
|
4455
|
-
L: 124,
|
|
4456
|
-
S: this,
|
|
4457
|
-
A: [
|
|
4458
|
-
"this._context",
|
|
4459
|
-
""
|
|
4460
|
-
]
|
|
4461
|
-
});
|
|
4462
|
-
invariant14(!this._connections.has(spaceId), void 0, {
|
|
4463
|
-
F: __dxlog_file18,
|
|
4464
|
-
L: 125,
|
|
4465
|
-
S: this,
|
|
4466
|
-
A: [
|
|
4467
|
-
"!this._connections.has(spaceId)",
|
|
4468
|
-
""
|
|
4469
|
-
]
|
|
4470
|
-
});
|
|
4471
|
-
let restartScheduled = false;
|
|
4472
|
-
const connection = new EdgeReplicatorConnection({
|
|
4473
|
-
edgeConnection: this._edgeConnection,
|
|
4474
|
-
spaceId,
|
|
4475
|
-
context: this._context,
|
|
4476
|
-
sharedPolicyEnabled: this._sharePolicyEnabled,
|
|
4477
|
-
onRemoteConnected: async () => {
|
|
4478
|
-
this._context?.onConnectionOpen(connection);
|
|
4479
|
-
},
|
|
4480
|
-
onRemoteDisconnected: async () => {
|
|
4481
|
-
this._context?.onConnectionClosed(connection);
|
|
4482
|
-
},
|
|
4483
|
-
onRestartRequested: async () => {
|
|
4484
|
-
if (!this._ctx || restartScheduled) {
|
|
4485
|
-
return;
|
|
4486
|
-
}
|
|
4487
|
-
const restartDelay = Math.min(MAX_RESTART_DELAY, INITIAL_RESTART_DELAY * reconnects) + Math.random() * RESTART_DELAY_JITTER;
|
|
4488
|
-
log13("connection restart scheduled", {
|
|
4489
|
-
spaceId,
|
|
4490
|
-
reconnects,
|
|
4491
|
-
restartDelay
|
|
4492
|
-
}, {
|
|
4493
|
-
F: __dxlog_file18,
|
|
4494
|
-
L: 148,
|
|
4495
|
-
S: this,
|
|
4496
|
-
C: (f, a) => f(...a)
|
|
4497
|
-
});
|
|
4498
|
-
restartScheduled = true;
|
|
4499
|
-
scheduleTask2(this._ctx, async () => {
|
|
4500
|
-
const env = {
|
|
4501
|
-
stack: [],
|
|
4502
|
-
error: void 0,
|
|
4503
|
-
hasError: false
|
|
4504
|
-
};
|
|
4505
|
-
try {
|
|
4506
|
-
const _guard = _ts_add_disposable_resource(env, await this._mutex.acquire(), false);
|
|
4507
|
-
if (this._connections.get(spaceId) !== connection) {
|
|
4508
|
-
return;
|
|
4509
|
-
}
|
|
4510
|
-
const ctx = this._ctx;
|
|
4511
|
-
await connection.close();
|
|
4512
|
-
this._connections.delete(spaceId);
|
|
4513
|
-
if (ctx?.disposed) {
|
|
4514
|
-
return;
|
|
4515
|
-
}
|
|
4516
|
-
await this._openConnection(spaceId, reconnects + 1);
|
|
4517
|
-
} catch (e) {
|
|
4518
|
-
env.error = e;
|
|
4519
|
-
env.hasError = true;
|
|
4520
|
-
} finally {
|
|
4521
|
-
_ts_dispose_resources(env);
|
|
4522
|
-
}
|
|
4523
|
-
}, restartDelay);
|
|
4524
|
-
}
|
|
4525
|
-
});
|
|
4526
|
-
this._connections.set(spaceId, connection);
|
|
4527
|
-
await connection.open();
|
|
4528
|
-
}
|
|
4529
|
-
};
|
|
4530
|
-
var MAX_INFLIGHT_REQUESTS = 5;
|
|
4531
|
-
var MAX_RATE_LIMIT_WAIT_TIME_MS = 3e3;
|
|
4532
|
-
var EdgeReplicatorConnection = class extends Resource11 {
|
|
4533
|
-
constructor({ edgeConnection, spaceId, context, sharedPolicyEnabled, onRemoteConnected, onRemoteDisconnected, onRestartRequested }) {
|
|
4534
|
-
super();
|
|
4535
|
-
this._remotePeerId = null;
|
|
4536
|
-
this._requestLimiter = new InflightRequestLimiter({
|
|
4537
|
-
maxInflightRequests: MAX_INFLIGHT_REQUESTS,
|
|
4538
|
-
resetBalanceTimeoutMs: MAX_RATE_LIMIT_WAIT_TIME_MS
|
|
4539
|
-
});
|
|
4540
|
-
this._edgeConnection = edgeConnection;
|
|
4541
|
-
this._spaceId = spaceId;
|
|
4542
|
-
this._context = context;
|
|
4543
|
-
this._remotePeerId = `${EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}-${randomUUID()}`;
|
|
4544
|
-
this._targetServiceId = `${EdgeService.AUTOMERGE_REPLICATOR}:${spaceId}`;
|
|
4545
|
-
this._sharedPolicyEnabled = sharedPolicyEnabled;
|
|
4546
|
-
this._onRemoteConnected = onRemoteConnected;
|
|
4547
|
-
this._onRemoteDisconnected = onRemoteDisconnected;
|
|
4548
|
-
this._onRestartRequested = onRestartRequested;
|
|
4549
|
-
this.readable = new ReadableStream({
|
|
4550
|
-
start: (controller) => {
|
|
4551
|
-
this._readableStreamController = controller;
|
|
4552
|
-
}
|
|
4553
|
-
});
|
|
4554
|
-
this.writable = new WritableStream({
|
|
4555
|
-
write: async (message, controller) => {
|
|
4556
|
-
await this._requestLimiter.rateLimit(message);
|
|
4557
|
-
await this._sendMessage(message);
|
|
4558
|
-
}
|
|
4559
|
-
});
|
|
4560
|
-
}
|
|
4561
|
-
async _open(ctx) {
|
|
4562
|
-
log13("opening...", void 0, {
|
|
4563
|
-
F: __dxlog_file18,
|
|
4564
|
-
L: 251,
|
|
4565
|
-
S: this,
|
|
4566
|
-
C: (f, a) => f(...a)
|
|
4567
|
-
});
|
|
4568
|
-
await this._requestLimiter.open();
|
|
4569
|
-
this._ctx.onDispose(this._edgeConnection.onMessage((msg) => {
|
|
4570
|
-
this._onMessage(msg);
|
|
4571
|
-
}));
|
|
4572
|
-
await this._onRemoteConnected();
|
|
4573
|
-
}
|
|
4574
|
-
async _close() {
|
|
4575
|
-
log13("closing...", void 0, {
|
|
4576
|
-
F: __dxlog_file18,
|
|
4577
|
-
L: 266,
|
|
4578
|
-
S: this,
|
|
4579
|
-
C: (f, a) => f(...a)
|
|
4580
|
-
});
|
|
4581
|
-
this._readableStreamController.close();
|
|
4582
|
-
await this._requestLimiter.close();
|
|
4583
|
-
await this._onRemoteDisconnected();
|
|
4584
|
-
}
|
|
4585
|
-
get peerId() {
|
|
4586
|
-
invariant14(this._remotePeerId, "Not connected", {
|
|
4587
|
-
F: __dxlog_file18,
|
|
4588
|
-
L: 275,
|
|
4589
|
-
S: this,
|
|
4590
|
-
A: [
|
|
4591
|
-
"this._remotePeerId",
|
|
4592
|
-
"'Not connected'"
|
|
4593
|
-
]
|
|
4594
|
-
});
|
|
4595
|
-
return this._remotePeerId;
|
|
4596
|
-
}
|
|
4597
|
-
async shouldAdvertise(params) {
|
|
4598
|
-
if (!this._sharedPolicyEnabled) {
|
|
4599
|
-
return true;
|
|
4600
|
-
}
|
|
4601
|
-
const spaceId = await this._context.getContainingSpaceIdForDocument(params.documentId);
|
|
4602
|
-
if (!spaceId) {
|
|
4603
|
-
const remoteDocumentExists = await this._context.isDocumentInRemoteCollection({
|
|
4604
|
-
documentId: params.documentId,
|
|
4605
|
-
peerId: this._remotePeerId
|
|
4606
|
-
});
|
|
4607
|
-
log13.verbose("document not found locally for share policy check", {
|
|
4608
|
-
documentId: params.documentId,
|
|
4609
|
-
acceptDocument: remoteDocumentExists,
|
|
4610
|
-
remoteId: this._remotePeerId
|
|
4611
|
-
}, {
|
|
4612
|
-
F: __dxlog_file18,
|
|
4613
|
-
L: 290,
|
|
4614
|
-
S: this,
|
|
4615
|
-
C: (f, a) => f(...a)
|
|
4616
|
-
});
|
|
4617
|
-
return remoteDocumentExists;
|
|
4618
|
-
}
|
|
4619
|
-
return spaceId === this._spaceId;
|
|
4620
|
-
}
|
|
4621
|
-
shouldSyncCollection(params) {
|
|
4622
|
-
if (!this._sharedPolicyEnabled) {
|
|
4623
|
-
return true;
|
|
4624
|
-
}
|
|
4625
|
-
const spaceId = getSpaceIdFromCollectionId(params.collectionId);
|
|
4626
|
-
return spaceId === this._spaceId && params.collectionId.split(":").length === 3;
|
|
4627
|
-
}
|
|
4628
|
-
_onMessage(message) {
|
|
4629
|
-
if (message.serviceId !== this._targetServiceId) {
|
|
4630
|
-
return;
|
|
4631
|
-
}
|
|
4632
|
-
const payload = cbor2.decode(message.payload.value);
|
|
4633
|
-
log13.verbose("received", {
|
|
4634
|
-
type: payload.type,
|
|
4635
|
-
documentId: payload.type === "sync" && payload.documentId,
|
|
4636
|
-
remoteId: this._remotePeerId
|
|
4637
|
-
}, {
|
|
4638
|
-
F: __dxlog_file18,
|
|
4639
|
-
L: 319,
|
|
4640
|
-
S: this,
|
|
4641
|
-
C: (f, a) => f(...a)
|
|
4642
|
-
});
|
|
4643
|
-
payload.senderId = this._remotePeerId;
|
|
4644
|
-
this._processMessage(payload);
|
|
4645
|
-
}
|
|
4646
|
-
_processMessage(message) {
|
|
4647
|
-
if (isForbiddenErrorMessage(message)) {
|
|
4648
|
-
this._onRestartRequested();
|
|
4649
|
-
return;
|
|
4650
|
-
}
|
|
4651
|
-
this._requestLimiter.handleResponse(message);
|
|
4652
|
-
this._readableStreamController.enqueue(message);
|
|
4653
|
-
}
|
|
4654
|
-
async _sendMessage(message) {
|
|
4655
|
-
message.targetId = this._targetServiceId;
|
|
4656
|
-
log13.verbose("sending...", {
|
|
4657
|
-
type: message.type,
|
|
4658
|
-
documentId: message.type === "sync" && message.documentId,
|
|
4659
|
-
remoteId: this._remotePeerId
|
|
4660
|
-
}, {
|
|
4661
|
-
F: __dxlog_file18,
|
|
4662
|
-
L: 348,
|
|
4663
|
-
S: this,
|
|
4664
|
-
C: (f, a) => f(...a)
|
|
4665
|
-
});
|
|
4666
|
-
const encoded = cbor2.encode(message);
|
|
4667
|
-
await this._edgeConnection.send(buf.create(RouterMessageSchema, {
|
|
4668
|
-
serviceId: this._targetServiceId,
|
|
4669
|
-
source: {
|
|
4670
|
-
identityKey: this._edgeConnection.identityKey,
|
|
4671
|
-
peerKey: this._edgeConnection.peerKey
|
|
4672
|
-
},
|
|
4673
|
-
payload: {
|
|
4674
|
-
value: bufferToArray2(encoded)
|
|
4675
|
-
}
|
|
4676
|
-
}));
|
|
4677
|
-
}
|
|
4678
|
-
};
|
|
4679
|
-
var isForbiddenErrorMessage = (message) => message.type === "error" && message.message === "Forbidden";
|
|
4680
|
-
|
|
4681
|
-
// packages/core/echo/echo-pipeline/src/util.ts
|
|
4682
|
-
import { decodeReference, ObjectStructure as ObjectStructure2 } from "@dxos/echo-protocol";
|
|
4683
|
-
var findInlineObjectOfType = (spaceDoc, typename) => {
|
|
4684
|
-
for (const id in spaceDoc.objects ?? {}) {
|
|
4685
|
-
const obj = spaceDoc.objects[id];
|
|
4686
|
-
const objType = ObjectStructure2.getTypeReference(obj);
|
|
4687
|
-
if (objType && decodeReference(objType).objectId === typename) {
|
|
4688
|
-
return [
|
|
4689
|
-
id,
|
|
4690
|
-
obj
|
|
4691
|
-
];
|
|
4692
|
-
}
|
|
4693
|
-
}
|
|
4694
|
-
return void 0;
|
|
4695
|
-
};
|
|
4696
|
-
export {
|
|
4697
|
-
AuthExtension,
|
|
4698
|
-
AuthStatus,
|
|
4699
|
-
AutomergeHost,
|
|
4700
|
-
CredentialRetrieverExtension,
|
|
4701
|
-
CredentialServerExtension,
|
|
4702
|
-
DataServiceImpl,
|
|
4703
|
-
DatabaseRoot,
|
|
4704
|
-
DocumentsSynchronizer,
|
|
4705
|
-
EchoDataMonitor,
|
|
4706
|
-
EchoEdgeReplicator,
|
|
4707
|
-
EchoHost,
|
|
4708
|
-
ExecutionTrace,
|
|
4709
|
-
FIND_PARAMS,
|
|
4710
|
-
LevelDBStorageAdapter,
|
|
4711
|
-
MOCK_AUTH_PROVIDER,
|
|
4712
|
-
MOCK_AUTH_VERIFIER,
|
|
4713
|
-
MeshEchoReplicator,
|
|
4714
|
-
MetadataStore,
|
|
4715
|
-
Pipeline,
|
|
4716
|
-
QueryExecutor,
|
|
4717
|
-
QueryPlan,
|
|
4718
|
-
QueryPlanner,
|
|
4719
|
-
QueryServiceImpl,
|
|
4720
|
-
Space,
|
|
4721
|
-
SpaceDocumentListUpdatedEvent,
|
|
4722
|
-
SpaceManager,
|
|
4723
|
-
SpaceProtocol,
|
|
4724
|
-
SpaceProtocolSession,
|
|
4725
|
-
SpaceStateManager,
|
|
4726
|
-
TimeframeClock,
|
|
4727
|
-
codec,
|
|
4728
|
-
createIdFromSpaceKey,
|
|
4729
|
-
createMappedFeedWriter,
|
|
4730
|
-
deriveCollectionIdFromSpaceId,
|
|
4731
|
-
diffCollectionState,
|
|
4732
|
-
encodingOptions,
|
|
4733
|
-
filterMatchObject,
|
|
4734
|
-
filterMatchValue,
|
|
4735
|
-
findInlineObjectOfType,
|
|
4736
|
-
getSpaceIdFromCollectionId,
|
|
4737
|
-
hasInvitationExpired,
|
|
4738
|
-
mapFeedIndexesToTimeframe,
|
|
4739
|
-
mapTimeframeToFeedIndexes,
|
|
4740
|
-
startAfter,
|
|
4741
|
-
valueEncoding
|
|
4742
|
-
};
|
|
4743
|
-
//# sourceMappingURL=index.mjs.map
|