@dxos/echo-pipeline 0.8.3 → 0.8.4-main.28f8d3d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-35I6ERLG.mjs → chunk-2543T5DX.mjs} +224 -145
- package/dist/lib/browser/chunk-2543T5DX.mjs.map +7 -0
- package/dist/lib/browser/{chunk-TQJTKNMS.mjs → chunk-VUXUDIPM.mjs} +2 -2
- package/dist/lib/{node/chunk-HOPOFWAL.cjs.map → browser/chunk-VUXUDIPM.mjs.map} +3 -3
- package/dist/lib/browser/filter/index.mjs +1 -1
- package/dist/lib/browser/index.mjs +419 -242
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +42 -17
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node-esm/{chunk-RVK35BS7.mjs → chunk-PGZYXNYE.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-RVK35BS7.mjs.map → chunk-PGZYXNYE.mjs.map} +2 -2
- package/dist/lib/node-esm/{chunk-5BHLPT24.mjs → chunk-UQI6R3TD.mjs} +224 -145
- package/dist/lib/node-esm/chunk-UQI6R3TD.mjs.map +7 -0
- package/dist/lib/node-esm/filter/index.mjs +1 -1
- package/dist/lib/node-esm/index.mjs +419 -242
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/testing/index.mjs +42 -17
- package/dist/lib/node-esm/testing/index.mjs.map +3 -3
- package/dist/types/src/automerge/automerge-host.d.ts +1 -1
- package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -1
- package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts +1 -1
- package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/echo-replicator.d.ts +1 -1
- package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
- package/dist/types/src/automerge/index.d.ts +1 -1
- package/dist/types/src/automerge/index.d.ts.map +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
- package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
- package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts +2 -2
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/database-root.d.ts.map +1 -1
- package/dist/types/src/db-host/documents-synchronizer.d.ts +2 -2
- package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
- package/dist/types/src/db-host/echo-host.d.ts +2 -2
- package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
- package/dist/types/src/db-host/query-service.d.ts +1 -1
- package/dist/types/src/db-host/query-service.d.ts.map +1 -1
- package/dist/types/src/db-host/space-state-manager.d.ts +1 -1
- package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
- package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
- package/dist/types/src/filter/filter-match.d.ts +1 -1
- package/dist/types/src/filter/filter-match.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/query/errors.d.ts +19 -6
- package/dist/types/src/query/errors.d.ts.map +1 -1
- package/dist/types/src/query/query-executor.d.ts +1 -1
- package/dist/types/src/query/query-executor.d.ts.map +1 -1
- package/dist/types/src/space/admission-discovery-extension.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +1 -1
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts +1 -1
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +1 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts +2 -2
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/util.d.ts +1 -1
- package/dist/types/src/util.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +41 -38
- package/src/automerge/automerge-host.test.ts +3 -2
- package/src/automerge/automerge-host.ts +20 -10
- package/src/automerge/automerge-repo.test.ts +67 -16
- package/src/automerge/collection-synchronizer.test.ts +2 -2
- package/src/automerge/collection-synchronizer.ts +2 -2
- package/src/automerge/echo-data-monitor.ts +1 -1
- package/src/automerge/echo-network-adapter.test.ts +3 -3
- package/src/automerge/echo-network-adapter.ts +8 -6
- package/src/automerge/echo-replicator.ts +2 -1
- package/src/automerge/index.ts +1 -1
- package/src/automerge/leveldb-storage-adapter.ts +1 -1
- package/src/automerge/mesh-echo-replicator.ts +2 -1
- package/src/automerge/storage-adapter.test.ts +1 -1
- package/src/common/space-id.ts +1 -1
- package/src/db-host/data-service.ts +8 -7
- package/src/db-host/database-root.ts +2 -2
- package/src/db-host/documents-synchronizer.test.ts +1 -1
- package/src/db-host/documents-synchronizer.ts +39 -26
- package/src/db-host/echo-host.ts +13 -12
- package/src/db-host/query-service.ts +8 -1
- package/src/db-host/space-state-manager.ts +2 -2
- package/src/edge/echo-edge-replicator.test.ts +3 -2
- package/src/edge/echo-edge-replicator.ts +36 -15
- package/src/filter/filter-match.test.ts +2 -2
- package/src/filter/filter-match.ts +1 -1
- package/src/metadata/metadata-store.ts +3 -3
- package/src/pipeline/pipeline-stress.test.ts +4 -2
- package/src/pipeline/pipeline.test.ts +3 -2
- package/src/pipeline/pipeline.ts +8 -5
- package/src/query/query-executor.ts +7 -7
- package/src/query/query-planner.test.ts +2 -1
- package/src/space/admission-discovery-extension.ts +2 -2
- package/src/space/control-pipeline.test.ts +4 -3
- package/src/space/control-pipeline.ts +4 -4
- package/src/space/space-manager.browser.test.ts +1 -1
- package/src/space/space-manager.ts +5 -4
- package/src/space/space-protocol.browser.test.ts +2 -2
- package/src/space/space-protocol.test.ts +3 -2
- package/src/space/space-protocol.ts +6 -3
- package/src/space/space.test.ts +1 -1
- package/src/space/space.ts +3 -2
- package/src/testing/test-agent-builder.ts +4 -3
- package/src/util.ts +1 -1
- package/dist/lib/browser/chunk-35I6ERLG.mjs.map +0 -7
- package/dist/lib/browser/chunk-TQJTKNMS.mjs.map +0 -7
- package/dist/lib/node/chunk-HOPOFWAL.cjs +0 -147
- package/dist/lib/node/chunk-JXX6LF5U.cjs +0 -2084
- package/dist/lib/node/chunk-JXX6LF5U.cjs.map +0 -7
- package/dist/lib/node/chunk-Q7SFCCGT.cjs +0 -33
- package/dist/lib/node/chunk-Q7SFCCGT.cjs.map +0 -7
- package/dist/lib/node/filter/index.cjs +0 -32
- package/dist/lib/node/filter/index.cjs.map +0 -7
- package/dist/lib/node/index.cjs +0 -4699
- package/dist/lib/node/index.cjs.map +0 -7
- package/dist/lib/node/meta.json +0 -1
- package/dist/lib/node/testing/index.cjs +0 -753
- package/dist/lib/node/testing/index.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-5BHLPT24.mjs.map +0 -7
|
@@ -3,17 +3,15 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { next as A, type Heads } from '@automerge/automerge';
|
|
6
|
-
import { type
|
|
6
|
+
import { type DocHandle, type DocumentId, type Repo } from '@automerge/automerge-repo';
|
|
7
7
|
|
|
8
|
-
import { UpdateScheduler } from '@dxos/async';
|
|
9
|
-
import { Resource } from '@dxos/context';
|
|
8
|
+
import { UpdateScheduler, sleep } from '@dxos/async';
|
|
9
|
+
import { LifecycleState, Resource, cancelWithContext } from '@dxos/context';
|
|
10
10
|
import { type DatabaseDirectory } from '@dxos/echo-protocol';
|
|
11
11
|
import { invariant } from '@dxos/invariant';
|
|
12
12
|
import { log } from '@dxos/log';
|
|
13
13
|
import { type BatchedDocumentUpdates, type DocumentUpdate } from '@dxos/protocols/proto/dxos/echo/service';
|
|
14
14
|
|
|
15
|
-
import { FIND_PARAMS } from '../automerge';
|
|
16
|
-
|
|
17
15
|
const MAX_UPDATE_FREQ = 10; // [updates/sec]
|
|
18
16
|
|
|
19
17
|
export type DocumentsSynchronizerParams = {
|
|
@@ -27,6 +25,9 @@ interface DocSyncState {
|
|
|
27
25
|
clearSubscriptions?: () => void;
|
|
28
26
|
}
|
|
29
27
|
|
|
28
|
+
const WRAP_AROUND_RETRY_LIMIT = 3;
|
|
29
|
+
const WRAP_AROUND_RETRY_INITIAL_DELAY = 100; // [ms]
|
|
30
|
+
|
|
30
31
|
/**
|
|
31
32
|
* Manages a connection and replication between worker's Automerge Repo and the client's Repo.
|
|
32
33
|
*/
|
|
@@ -47,8 +48,12 @@ export class DocumentsSynchronizer extends Resource {
|
|
|
47
48
|
super();
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
addDocuments(
|
|
51
|
-
|
|
51
|
+
addDocuments(
|
|
52
|
+
documentIds: DocumentId[],
|
|
53
|
+
retryCounter = 0,
|
|
54
|
+
wrapAroundRetryDelay = WRAP_AROUND_RETRY_INITIAL_DELAY,
|
|
55
|
+
): void {
|
|
56
|
+
if (retryCounter > WRAP_AROUND_RETRY_LIMIT) {
|
|
52
57
|
log.warn('Failed to load document, retry limit reached', { documentIds });
|
|
53
58
|
return;
|
|
54
59
|
}
|
|
@@ -64,7 +69,9 @@ export class DocumentsSynchronizer extends Resource {
|
|
|
64
69
|
})
|
|
65
70
|
.catch((error) => {
|
|
66
71
|
log.warn('Failed to load document, wraparound', { documentId, error });
|
|
67
|
-
this.
|
|
72
|
+
void cancelWithContext(this._ctx, sleep(wrapAroundRetryDelay)).then(() =>
|
|
73
|
+
this.addDocuments([documentId], retryCounter + 1, wrapAroundRetryDelay * 2),
|
|
74
|
+
);
|
|
68
75
|
});
|
|
69
76
|
}
|
|
70
77
|
}
|
|
@@ -90,17 +97,13 @@ export class DocumentsSynchronizer extends Resource {
|
|
|
90
97
|
|
|
91
98
|
async update(updates: DocumentUpdate[]): Promise<void> {
|
|
92
99
|
for (const { documentId, mutation, isNew } of updates) {
|
|
93
|
-
|
|
94
|
-
const doc = await this._params.repo.find<DatabaseDirectory>(documentId as DocumentId, FIND_PARAMS);
|
|
95
|
-
doc.update((doc) => A.loadIncremental(doc, mutation));
|
|
96
|
-
this._startSync(doc);
|
|
97
|
-
} else {
|
|
98
|
-
this._writeMutation(documentId as DocumentId, mutation);
|
|
99
|
-
}
|
|
100
|
+
this._writeMutation(documentId as DocumentId, mutation, isNew);
|
|
100
101
|
}
|
|
102
|
+
// TODO(mykola): This should not be required.
|
|
103
|
+
await this._params.repo.flush(updates.map(({ documentId }) => documentId as DocumentId));
|
|
101
104
|
}
|
|
102
105
|
|
|
103
|
-
private _startSync(doc: DocHandle<DatabaseDirectory>)
|
|
106
|
+
private _startSync(doc: DocHandle<DatabaseDirectory>) {
|
|
104
107
|
if (this._syncStates.has(doc.documentId)) {
|
|
105
108
|
log('Document already being synced', { documentId: doc.documentId });
|
|
106
109
|
return;
|
|
@@ -109,6 +112,7 @@ export class DocumentsSynchronizer extends Resource {
|
|
|
109
112
|
const syncState: DocSyncState = { handle: doc };
|
|
110
113
|
this._subscribeForChanges(syncState);
|
|
111
114
|
this._syncStates.set(doc.documentId, syncState);
|
|
115
|
+
return syncState;
|
|
112
116
|
}
|
|
113
117
|
|
|
114
118
|
_subscribeForChanges(syncState: DocSyncState): void {
|
|
@@ -157,16 +161,25 @@ export class DocumentsSynchronizer extends Resource {
|
|
|
157
161
|
return mutation;
|
|
158
162
|
}
|
|
159
163
|
|
|
160
|
-
private _writeMutation(documentId: DocumentId, mutation: Uint8Array): void {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
164
|
+
private _writeMutation(documentId: DocumentId, mutation: Uint8Array, isNew?: boolean): void {
|
|
165
|
+
if (this._lifecycleState === LifecycleState.CLOSED) {
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
if (isNew) {
|
|
169
|
+
const newHandle = this._params.repo.import<DatabaseDirectory>(mutation, { docId: documentId });
|
|
170
|
+
const syncState = this._startSync(newHandle);
|
|
171
|
+
syncState!.lastSentHead = A.getHeads(newHandle.doc());
|
|
172
|
+
} else {
|
|
173
|
+
const syncState = this._syncStates.get(documentId);
|
|
174
|
+
invariant(syncState, 'Sync state for document not found');
|
|
175
|
+
const headsBefore = A.getHeads(syncState.handle.doc());
|
|
176
|
+
// This will update corresponding handle in the repo.
|
|
177
|
+
this._params.repo.import(mutation, { docId: documentId });
|
|
178
|
+
|
|
179
|
+
if (A.equals(headsBefore, syncState!.lastSentHead)) {
|
|
180
|
+
// No new mutations were discovered on network, so we do not need to send updates from worker to client.
|
|
181
|
+
syncState!.lastSentHead = A.getHeads(syncState.handle.doc());
|
|
168
182
|
}
|
|
169
|
-
|
|
170
|
-
});
|
|
183
|
+
}
|
|
171
184
|
}
|
|
172
185
|
}
|
package/src/db-host/echo-host.ts
CHANGED
|
@@ -10,9 +10,9 @@ import {
|
|
|
10
10
|
type Repo,
|
|
11
11
|
} from '@automerge/automerge-repo';
|
|
12
12
|
|
|
13
|
-
import { LifecycleState, Resource
|
|
13
|
+
import { type Context, LifecycleState, Resource } from '@dxos/context';
|
|
14
14
|
import { todo } from '@dxos/debug';
|
|
15
|
-
import {
|
|
15
|
+
import { type DatabaseDirectory, SpaceDocVersion, createIdFromSpaceKey } from '@dxos/echo-protocol';
|
|
16
16
|
import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
|
|
17
17
|
import { invariant } from '@dxos/invariant';
|
|
18
18
|
import { type PublicKey, type SpaceId } from '@dxos/keys';
|
|
@@ -20,24 +20,25 @@ import { type LevelDB } from '@dxos/kv-store';
|
|
|
20
20
|
import { IndexKind } from '@dxos/protocols/proto/dxos/echo/indexing';
|
|
21
21
|
import { trace } from '@dxos/tracing';
|
|
22
22
|
|
|
23
|
-
import { DataServiceImpl } from './data-service';
|
|
24
|
-
import { type DatabaseRoot } from './database-root';
|
|
25
|
-
import { createSelectedDocumentsIterator } from './documents-iterator';
|
|
26
|
-
import { QueryServiceImpl } from './query-service';
|
|
27
|
-
import { SpaceStateManager } from './space-state-manager';
|
|
28
23
|
import {
|
|
29
24
|
AutomergeHost,
|
|
30
|
-
FIND_PARAMS,
|
|
31
|
-
EchoDataMonitor,
|
|
32
|
-
deriveCollectionIdFromSpaceId,
|
|
33
|
-
type LoadDocOptions,
|
|
34
25
|
type CreateDocOptions,
|
|
35
|
-
|
|
26
|
+
EchoDataMonitor,
|
|
36
27
|
type EchoDataStats,
|
|
28
|
+
type EchoReplicator,
|
|
29
|
+
FIND_PARAMS,
|
|
30
|
+
type LoadDocOptions,
|
|
37
31
|
type PeerIdProvider,
|
|
38
32
|
type RootDocumentSpaceKeyProvider,
|
|
33
|
+
deriveCollectionIdFromSpaceId,
|
|
39
34
|
} from '../automerge';
|
|
40
35
|
|
|
36
|
+
import { DataServiceImpl } from './data-service';
|
|
37
|
+
import { type DatabaseRoot } from './database-root';
|
|
38
|
+
import { createSelectedDocumentsIterator } from './documents-iterator';
|
|
39
|
+
import { QueryServiceImpl } from './query-service';
|
|
40
|
+
import { SpaceStateManager } from './space-state-manager';
|
|
41
|
+
|
|
41
42
|
export interface EchoHostIndexingConfig {
|
|
42
43
|
/**
|
|
43
44
|
* @default true
|
|
@@ -23,10 +23,11 @@ import {
|
|
|
23
23
|
} from '@dxos/protocols/proto/dxos/echo/query';
|
|
24
24
|
import { trace } from '@dxos/tracing';
|
|
25
25
|
|
|
26
|
-
import type { SpaceStateManager } from './space-state-manager';
|
|
27
26
|
import { type AutomergeHost } from '../automerge';
|
|
28
27
|
import { QueryExecutor } from '../query';
|
|
29
28
|
|
|
29
|
+
import type { SpaceStateManager } from './space-state-manager';
|
|
30
|
+
|
|
30
31
|
export type QueryServiceParams = {
|
|
31
32
|
indexer: Indexer;
|
|
32
33
|
automergeHost: AutomergeHost;
|
|
@@ -96,6 +97,12 @@ export class QueryServiceImpl extends Resource implements QueryService {
|
|
|
96
97
|
|
|
97
98
|
execQuery(request: QueryRequest): Stream<QueryResponse> {
|
|
98
99
|
return new Stream<QueryResponse>(({ next, close, ctx }) => {
|
|
100
|
+
if (this._params.indexer.config?.enabled !== true) {
|
|
101
|
+
log.error('indexer is disabled', { config: this._params.indexer.config });
|
|
102
|
+
close();
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
|
|
99
106
|
const queryEntry = this._createQuery(ctx, request, next, close, close);
|
|
100
107
|
scheduleMicroTask(ctx, async () => {
|
|
101
108
|
await queryEntry.executor.open();
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { type DocHandle, type DocumentId, interpretAsDocumentId } from '@automerge/automerge-repo';
|
|
6
6
|
import isEqual from 'lodash.isequal';
|
|
7
7
|
|
|
8
8
|
import { Event, UpdateScheduler } from '@dxos/async';
|
|
9
|
-
import {
|
|
9
|
+
import { Context, LifecycleState, Resource } from '@dxos/context';
|
|
10
10
|
import { type DatabaseDirectory } from '@dxos/echo-protocol';
|
|
11
11
|
import { invariant } from '@dxos/invariant';
|
|
12
12
|
import { type SpaceId } from '@dxos/keys';
|
|
@@ -7,7 +7,7 @@ import { getRandomPort } from 'get-port-please';
|
|
|
7
7
|
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
8
8
|
|
|
9
9
|
import { Event } from '@dxos/async';
|
|
10
|
-
import {
|
|
10
|
+
import { EdgeClient, MessageSchema, createEphemeralEdgeIdentity } from '@dxos/edge-client';
|
|
11
11
|
import { createTestEdgeWsServer } from '@dxos/edge-client/testing';
|
|
12
12
|
import { PublicKey, SpaceId } from '@dxos/keys';
|
|
13
13
|
import { EdgeService } from '@dxos/protocols';
|
|
@@ -16,9 +16,10 @@ import { createBuf } from '@dxos/protocols/buf';
|
|
|
16
16
|
import type { Peer } from '@dxos/protocols/proto/dxos/edge/messenger';
|
|
17
17
|
import { openAndClose } from '@dxos/test-utils';
|
|
18
18
|
|
|
19
|
-
import { EchoEdgeReplicator } from './echo-edge-replicator';
|
|
20
19
|
import type { EchoReplicatorContext, ReplicatorConnection } from '../automerge';
|
|
21
20
|
|
|
21
|
+
import { EchoEdgeReplicator } from './echo-edge-replicator';
|
|
22
|
+
|
|
22
23
|
describe('EchoEdgeReplicator', () => {
|
|
23
24
|
test('reconnects', async () => {
|
|
24
25
|
const { client, server } = await createClientServer();
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { cbor } from '@automerge/automerge-repo';
|
|
6
6
|
|
|
7
|
-
import { Mutex,
|
|
7
|
+
import { Mutex, scheduleMicroTask, scheduleTask } from '@dxos/async';
|
|
8
8
|
import { Context, Resource } from '@dxos/context';
|
|
9
9
|
import { randomUUID } from '@dxos/crypto';
|
|
10
10
|
import type { CollectionId } from '@dxos/echo-protocol';
|
|
@@ -12,7 +12,7 @@ import { type EdgeConnection } from '@dxos/edge-client';
|
|
|
12
12
|
import { invariant } from '@dxos/invariant';
|
|
13
13
|
import type { SpaceId } from '@dxos/keys';
|
|
14
14
|
import { log } from '@dxos/log';
|
|
15
|
-
import {
|
|
15
|
+
import { type AutomergeProtocolMessage, EdgeService, type PeerId } from '@dxos/protocols';
|
|
16
16
|
import { buf } from '@dxos/protocols/buf';
|
|
17
17
|
import {
|
|
18
18
|
type Message as RouterMessage,
|
|
@@ -20,16 +20,17 @@ import {
|
|
|
20
20
|
} from '@dxos/protocols/buf/dxos/edge/messenger_pb';
|
|
21
21
|
import { bufferToArray } from '@dxos/util';
|
|
22
22
|
|
|
23
|
-
import { InflightRequestLimiter } from './inflight-request-limiter';
|
|
24
23
|
import {
|
|
25
|
-
getSpaceIdFromCollectionId,
|
|
26
24
|
type EchoReplicator,
|
|
27
25
|
type EchoReplicatorContext,
|
|
28
26
|
type ReplicatorConnection,
|
|
29
27
|
type ShouldAdvertiseParams,
|
|
30
28
|
type ShouldSyncCollectionParams,
|
|
29
|
+
getSpaceIdFromCollectionId,
|
|
31
30
|
} from '../automerge';
|
|
32
31
|
|
|
32
|
+
import { InflightRequestLimiter } from './inflight-request-limiter';
|
|
33
|
+
|
|
33
34
|
/**
|
|
34
35
|
* Delay before restarting the connection after receiving a forbidden error.
|
|
35
36
|
*/
|
|
@@ -132,9 +133,11 @@ export class EchoEdgeReplicator implements EchoReplicator {
|
|
|
132
133
|
context: this._context,
|
|
133
134
|
sharedPolicyEnabled: this._sharePolicyEnabled,
|
|
134
135
|
onRemoteConnected: async () => {
|
|
136
|
+
log.trace('dxos.echo.edge.replicator.onRemoteConnected', { spaceId });
|
|
135
137
|
this._context?.onConnectionOpen(connection);
|
|
136
138
|
},
|
|
137
139
|
onRemoteDisconnected: async () => {
|
|
140
|
+
log.trace('dxos.echo.edge.replicator.onRemoteDisconnected', { spaceId });
|
|
138
141
|
this._context?.onConnectionClosed(connection);
|
|
139
142
|
},
|
|
140
143
|
onRestartRequested: async () => {
|
|
@@ -162,6 +165,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
|
|
|
162
165
|
if (ctx?.disposed) {
|
|
163
166
|
return;
|
|
164
167
|
}
|
|
168
|
+
log.trace('dxos.echo.edge.replicator.restart', { spaceId, reconnects, restartDelay });
|
|
165
169
|
await this._openConnection(spaceId, reconnects + 1);
|
|
166
170
|
},
|
|
167
171
|
restartDelay,
|
|
@@ -252,13 +256,26 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
252
256
|
|
|
253
257
|
await this._requestLimiter.open();
|
|
254
258
|
|
|
255
|
-
// TODO: handle reconnects
|
|
256
259
|
this._ctx.onDispose(
|
|
257
260
|
this._edgeConnection.onMessage((msg: RouterMessage) => {
|
|
258
261
|
this._onMessage(msg);
|
|
259
262
|
}),
|
|
260
263
|
);
|
|
261
264
|
|
|
265
|
+
let firstReconnect = true;
|
|
266
|
+
this._ctx.onDispose(
|
|
267
|
+
// NOTE: This will fire immediately if the connection is already open.
|
|
268
|
+
this._edgeConnection.onReconnected(async () => {
|
|
269
|
+
if (firstReconnect) {
|
|
270
|
+
log.verbose('first reconnect skipped');
|
|
271
|
+
firstReconnect = false;
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
this._onRestartRequested();
|
|
276
|
+
}),
|
|
277
|
+
);
|
|
278
|
+
|
|
262
279
|
await this._onRemoteConnected();
|
|
263
280
|
}
|
|
264
281
|
|
|
@@ -353,16 +370,20 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
|
|
|
353
370
|
|
|
354
371
|
const encoded = cbor.encode(message);
|
|
355
372
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
373
|
+
try {
|
|
374
|
+
await this._edgeConnection.send(
|
|
375
|
+
buf.create(RouterMessageSchema, {
|
|
376
|
+
serviceId: this._targetServiceId,
|
|
377
|
+
source: {
|
|
378
|
+
identityKey: this._edgeConnection.identityKey,
|
|
379
|
+
peerKey: this._edgeConnection.peerKey,
|
|
380
|
+
},
|
|
381
|
+
payload: { value: bufferToArray(encoded) },
|
|
382
|
+
}),
|
|
383
|
+
);
|
|
384
|
+
} catch (err) {
|
|
385
|
+
log.error('failed to send message', { err });
|
|
386
|
+
}
|
|
366
387
|
}
|
|
367
388
|
}
|
|
368
389
|
|
|
@@ -6,10 +6,10 @@ import { describe, expect, test } from 'vitest';
|
|
|
6
6
|
|
|
7
7
|
import { Filter } from '@dxos/echo';
|
|
8
8
|
import { ObjectStructure } from '@dxos/echo-protocol';
|
|
9
|
-
import {
|
|
9
|
+
import { EXPANDO_TYPENAME, Expando, Ref } from '@dxos/echo-schema';
|
|
10
10
|
import { DXN, ObjectId, SpaceId } from '@dxos/keys';
|
|
11
11
|
|
|
12
|
-
import {
|
|
12
|
+
import { type MatchedObject, filterMatchObject } from './filter-match';
|
|
13
13
|
|
|
14
14
|
describe('filterMatch', () => {
|
|
15
15
|
test('properties', () => {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { type ObjectStructure, type QueryAST, decodeReference, isEncodedReference } from '@dxos/echo-protocol';
|
|
6
6
|
import { EXPANDO_TYPENAME } from '@dxos/echo-schema';
|
|
7
7
|
import { DXN, type ObjectId, type SpaceId } from '@dxos/keys';
|
|
8
8
|
|
|
@@ -16,11 +16,11 @@ import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/servic
|
|
|
16
16
|
import {
|
|
17
17
|
type ControlPipelineSnapshot,
|
|
18
18
|
type EchoMetadata,
|
|
19
|
-
type
|
|
19
|
+
type EdgeReplicationSetting,
|
|
20
20
|
type IdentityRecord,
|
|
21
|
-
type SpaceCache,
|
|
22
21
|
type LargeSpaceMetadata,
|
|
23
|
-
type
|
|
22
|
+
type SpaceCache,
|
|
23
|
+
type SpaceMetadata,
|
|
24
24
|
} from '@dxos/protocols/proto/dxos/echo/metadata';
|
|
25
25
|
import { type Directory, type File } from '@dxos/random-access-storage';
|
|
26
26
|
import { type Timeframe } from '@dxos/timeframe';
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
// Copyright 2022 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import * as fc from 'fast-check';
|
|
6
5
|
import { inspect } from 'node:util';
|
|
6
|
+
|
|
7
|
+
import * as fc from 'fast-check';
|
|
7
8
|
import { describe, expect, test } from 'vitest';
|
|
8
9
|
|
|
9
10
|
import { asyncTimeout } from '@dxos/async';
|
|
@@ -15,9 +16,10 @@ import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
|
|
|
15
16
|
import { Timeframe } from '@dxos/timeframe';
|
|
16
17
|
import { range } from '@dxos/util';
|
|
17
18
|
|
|
18
|
-
import { Pipeline } from './pipeline';
|
|
19
19
|
import { TestFeedBuilder } from '../testing';
|
|
20
20
|
|
|
21
|
+
import { Pipeline } from './pipeline';
|
|
22
|
+
|
|
21
23
|
const NUM_AGENTS = 2;
|
|
22
24
|
const NUM_MESSAGES = 10;
|
|
23
25
|
|
|
@@ -2,16 +2,17 @@
|
|
|
2
2
|
// Copyright 2022 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { describe, expect,
|
|
5
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
6
6
|
|
|
7
7
|
import { Event, sleep } from '@dxos/async';
|
|
8
8
|
import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
|
|
9
9
|
import { Timeframe } from '@dxos/timeframe';
|
|
10
10
|
import { range } from '@dxos/util';
|
|
11
11
|
|
|
12
|
-
import { Pipeline } from './pipeline';
|
|
13
12
|
import { TestFeedBuilder } from '../testing';
|
|
14
13
|
|
|
14
|
+
import { Pipeline } from './pipeline';
|
|
15
|
+
|
|
15
16
|
const TEST_MESSAGE: FeedMessage = {
|
|
16
17
|
timeframe: new Timeframe(),
|
|
17
18
|
payload: {},
|
package/src/pipeline/pipeline.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2022 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { Event, sleepWithContext, synchronized
|
|
5
|
+
import { Event, Trigger, sleepWithContext, synchronized } from '@dxos/async';
|
|
6
6
|
import { Context, rejectOnDispose } from '@dxos/context';
|
|
7
7
|
import { failUndefined } from '@dxos/debug';
|
|
8
8
|
import { FeedSetIterator, type FeedWrapper, type FeedWriter } from '@dxos/feed-store';
|
|
@@ -14,10 +14,11 @@ import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
|
|
|
14
14
|
import { Timeframe } from '@dxos/timeframe';
|
|
15
15
|
import { ComplexMap } from '@dxos/util';
|
|
16
16
|
|
|
17
|
-
import { createMessageSelector } from './message-selector';
|
|
18
|
-
import { mapFeedIndexesToTimeframe, startAfter, TimeframeClock } from './timeframe-clock';
|
|
19
17
|
import { createMappedFeedWriter } from '../common';
|
|
20
18
|
|
|
19
|
+
import { createMessageSelector } from './message-selector';
|
|
20
|
+
import { TimeframeClock, mapFeedIndexesToTimeframe, startAfter } from './timeframe-clock';
|
|
21
|
+
|
|
21
22
|
export type WaitUntilReachedTargetParams = {
|
|
22
23
|
/**
|
|
23
24
|
* For cancellation.
|
|
@@ -40,8 +41,10 @@ export class PipelineState {
|
|
|
40
41
|
*/
|
|
41
42
|
_ctx = new Context();
|
|
42
43
|
|
|
43
|
-
// TODO(dmaretskyi): Remove?.
|
|
44
|
-
public
|
|
44
|
+
// TODO(dmaretskyi): Remove?. Avoid accessing `_timeframeClock` before constructor initialization.
|
|
45
|
+
public get timeframeUpdate() {
|
|
46
|
+
return this._timeframeClock.update;
|
|
47
|
+
}
|
|
45
48
|
|
|
46
49
|
public readonly stalled = new Event();
|
|
47
50
|
|
|
@@ -6,7 +6,7 @@ import type { AutomergeUrl, DocumentId } from '@automerge/automerge-repo';
|
|
|
6
6
|
import { Match } from 'effect';
|
|
7
7
|
|
|
8
8
|
import { Context, ContextDisposedError, LifecycleState, Resource } from '@dxos/context';
|
|
9
|
-
import { DatabaseDirectory,
|
|
9
|
+
import { DatabaseDirectory, ObjectStructure, type QueryAST, isEncodedReference } from '@dxos/echo-protocol';
|
|
10
10
|
import { EscapedPropPath, type FindResult, type Indexer } from '@dxos/indexing';
|
|
11
11
|
import { invariant } from '@dxos/invariant';
|
|
12
12
|
import { DXN, type ObjectId, PublicKey, type SpaceId } from '@dxos/keys';
|
|
@@ -15,13 +15,14 @@ import { objectPointerCodec } from '@dxos/protocols';
|
|
|
15
15
|
import { type QueryReactivity, type QueryResult } from '@dxos/protocols/proto/dxos/echo/query';
|
|
16
16
|
import { getDeep, isNonNullable } from '@dxos/util';
|
|
17
17
|
|
|
18
|
-
import type { QueryPlan } from './plan';
|
|
19
|
-
import { QueryPlanner } from './query-planner';
|
|
20
18
|
import type { AutomergeHost } from '../automerge';
|
|
21
19
|
import { createIdFromSpaceKey } from '../common';
|
|
22
20
|
import type { SpaceStateManager } from '../db-host';
|
|
23
21
|
import { filterMatchObject } from '../filter';
|
|
24
22
|
|
|
23
|
+
import type { QueryPlan } from './plan';
|
|
24
|
+
import { QueryPlanner } from './query-planner';
|
|
25
|
+
|
|
25
26
|
type QueryExecutorOptions = {
|
|
26
27
|
indexer: Indexer;
|
|
27
28
|
automergeHost: AutomergeHost;
|
|
@@ -189,7 +190,6 @@ export class QueryExecutor extends Resource {
|
|
|
189
190
|
);
|
|
190
191
|
|
|
191
192
|
if (TRACE_QUERY_EXECUTION) {
|
|
192
|
-
// eslint-disable-next-line no-console
|
|
193
193
|
console.log(ExecutionTrace.format(trace));
|
|
194
194
|
}
|
|
195
195
|
|
|
@@ -383,7 +383,7 @@ export class QueryExecutor extends Resource {
|
|
|
383
383
|
workingSet: QueryItem[],
|
|
384
384
|
): Promise<StepExecutionResult> {
|
|
385
385
|
if (workingSet.length === 6) {
|
|
386
|
-
log
|
|
386
|
+
log('filter deleted step', { step, workingSet });
|
|
387
387
|
}
|
|
388
388
|
|
|
389
389
|
const expected = step.mode === 'only-deleted';
|
|
@@ -428,7 +428,7 @@ export class QueryExecutor extends Resource {
|
|
|
428
428
|
}
|
|
429
429
|
: null;
|
|
430
430
|
} catch {
|
|
431
|
-
log.warn('
|
|
431
|
+
log.warn('invalid reference', { ref: ref['/'] });
|
|
432
432
|
return null;
|
|
433
433
|
}
|
|
434
434
|
});
|
|
@@ -491,7 +491,7 @@ export class QueryExecutor extends Resource {
|
|
|
491
491
|
spaceId: item.spaceId,
|
|
492
492
|
};
|
|
493
493
|
} catch {
|
|
494
|
-
log.warn('
|
|
494
|
+
log.warn('invalid reference', { ref: ref['/'] });
|
|
495
495
|
return null;
|
|
496
496
|
}
|
|
497
497
|
})
|
|
@@ -8,9 +8,10 @@ import { Filter, Query } from '@dxos/echo';
|
|
|
8
8
|
import { type QueryAST } from '@dxos/echo-protocol';
|
|
9
9
|
import { SpaceId } from '@dxos/keys';
|
|
10
10
|
|
|
11
|
-
import { QueryPlanner } from './query-planner';
|
|
12
11
|
import { TestSchema } from '../testing';
|
|
13
12
|
|
|
13
|
+
import { QueryPlanner } from './query-planner';
|
|
14
|
+
|
|
14
15
|
describe('QueryPlanner', () => {
|
|
15
16
|
const planner = new QueryPlanner();
|
|
16
17
|
|
|
@@ -2,15 +2,15 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { type Trigger, scheduleTask } from '@dxos/async';
|
|
6
6
|
import { Context } from '@dxos/context';
|
|
7
7
|
import { ProtocolError } from '@dxos/protocols';
|
|
8
8
|
import { schema } from '@dxos/protocols/proto';
|
|
9
9
|
import { type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
|
|
10
10
|
import {
|
|
11
11
|
type AdmissionDiscoveryService,
|
|
12
|
-
type GetAdmissionCredentialResponse,
|
|
13
12
|
type GetAdmissionCredentialRequest,
|
|
13
|
+
type GetAdmissionCredentialResponse,
|
|
14
14
|
} from '@dxos/protocols/proto/dxos/mesh/teleport';
|
|
15
15
|
import { type ExtensionContext, RpcExtension } from '@dxos/teleport';
|
|
16
16
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2022 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { describe, expect,
|
|
5
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
6
6
|
|
|
7
7
|
import { CredentialGenerator, createCredential } from '@dxos/credentials';
|
|
8
8
|
import { FeedFactory, FeedStore } from '@dxos/feed-store';
|
|
@@ -11,13 +11,14 @@ import { type PublicKey } from '@dxos/keys';
|
|
|
11
11
|
import { log } from '@dxos/log';
|
|
12
12
|
import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
|
|
13
13
|
import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
|
|
14
|
-
import {
|
|
14
|
+
import { StorageType, createStorage } from '@dxos/random-access-storage';
|
|
15
15
|
import { Timeframe } from '@dxos/timeframe';
|
|
16
16
|
|
|
17
|
-
import { ControlPipeline } from './control-pipeline';
|
|
18
17
|
import { valueEncoding } from '../common';
|
|
19
18
|
import { MetadataStore } from '../metadata';
|
|
20
19
|
|
|
20
|
+
import { ControlPipeline } from './control-pipeline';
|
|
21
|
+
|
|
21
22
|
describe('space/control-pipeline', () => {
|
|
22
23
|
test('admits feeds', async () => {
|
|
23
24
|
const keyring = new Keyring();
|
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
import { DeferredTask, sleepWithContext, trackLeaks } from '@dxos/async';
|
|
6
6
|
import { Context } from '@dxos/context';
|
|
7
7
|
import {
|
|
8
|
-
SpaceStateMachine,
|
|
9
|
-
type SpaceState,
|
|
10
|
-
type MemberInfo,
|
|
11
|
-
type FeedInfo,
|
|
12
8
|
type DelegateInvitationCredential,
|
|
9
|
+
type FeedInfo,
|
|
10
|
+
type MemberInfo,
|
|
11
|
+
type SpaceState,
|
|
12
|
+
SpaceStateMachine,
|
|
13
13
|
} from '@dxos/credentials';
|
|
14
14
|
import { type FeedWrapper } from '@dxos/feed-store';
|
|
15
15
|
import { PublicKey } from '@dxos/keys';
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
import { type AutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
|
|
6
6
|
|
|
7
|
-
import { synchronized, trackLeaks
|
|
8
|
-
import {
|
|
7
|
+
import { Trigger, synchronized, trackLeaks } from '@dxos/async';
|
|
8
|
+
import { type DelegateInvitationCredential, type MemberInfo, getCredentialAssertion } from '@dxos/credentials';
|
|
9
9
|
import { failUndefined } from '@dxos/debug';
|
|
10
10
|
import { type FeedStore } from '@dxos/feed-store';
|
|
11
11
|
import { PublicKey } from '@dxos/keys';
|
|
@@ -19,11 +19,12 @@ import { type Teleport } from '@dxos/teleport';
|
|
|
19
19
|
import { type BlobStore } from '@dxos/teleport-extension-object-sync';
|
|
20
20
|
import { ComplexMap } from '@dxos/util';
|
|
21
21
|
|
|
22
|
+
import { createIdFromSpaceKey } from '../common/space-id';
|
|
23
|
+
import { type MetadataStore } from '../metadata';
|
|
24
|
+
|
|
22
25
|
import { CredentialRetrieverExtension } from './admission-discovery-extension';
|
|
23
26
|
import { Space } from './space';
|
|
24
27
|
import { SpaceProtocol, type SwarmIdentity } from './space-protocol';
|
|
25
|
-
import { createIdFromSpaceKey } from '../common/space-id';
|
|
26
|
-
import { type MetadataStore } from '../metadata';
|
|
27
28
|
|
|
28
29
|
export type SpaceManagerParams = {
|
|
29
30
|
feedStore: FeedStore<FeedMessage>;
|
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
// Copyright 2022 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { describe, expect,
|
|
5
|
+
import { describe, expect, onTestFinished, test } from 'vitest';
|
|
6
6
|
|
|
7
7
|
import { Keyring } from '@dxos/keyring';
|
|
8
8
|
import { PublicKey } from '@dxos/keys';
|
|
9
9
|
import { createStorage } from '@dxos/random-access-storage';
|
|
10
10
|
import { Timeframe } from '@dxos/timeframe';
|
|
11
11
|
|
|
12
|
-
import {
|
|
12
|
+
import { TestAgentBuilder, TestFeedBuilder, WebsocketNetworkManagerProvider } from '../testing';
|
|
13
13
|
|
|
14
14
|
// TODO(burdon): Config.
|
|
15
15
|
// Signal server will be started by the setup script.
|