@dxos/echo-db 2.33.5-dev.cf9f6681 → 2.33.5-dev.fa6b779b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/echo.d.ts.map +1 -1
- package/dist/src/echo.js +1 -1
- package/dist/src/echo.js.map +1 -1
- package/dist/src/echo.test.js +0 -4
- package/dist/src/echo.test.js.map +1 -1
- package/dist/src/halo/halo-factory.d.ts +4 -4
- package/dist/src/halo/halo-factory.d.ts.map +1 -1
- package/dist/src/halo/halo-factory.js +3 -2
- package/dist/src/halo/halo-factory.js.map +1 -1
- package/dist/src/halo/halo-party.d.ts +4 -3
- package/dist/src/halo/halo-party.d.ts.map +1 -1
- package/dist/src/halo/halo-party.js +8 -4
- package/dist/src/halo/halo-party.js.map +1 -1
- package/dist/src/halo/halo.d.ts +2 -2
- package/dist/src/halo/halo.d.ts.map +1 -1
- package/dist/src/halo/party-opener.d.ts.map +1 -1
- package/dist/src/halo/party-opener.js +3 -1
- package/dist/src/halo/party-opener.js.map +1 -1
- package/dist/src/invitations/greeting-initiator.d.ts +2 -2
- package/dist/src/invitations/greeting-initiator.d.ts.map +1 -1
- package/dist/src/invitations/greeting-initiator.js +2 -1
- package/dist/src/invitations/greeting-initiator.js.map +1 -1
- package/dist/src/packlets/database/data-mirror.test.js +2 -2
- package/dist/src/packlets/database/data-mirror.test.js.map +1 -1
- package/dist/src/packlets/database/database-backend.d.ts +4 -5
- package/dist/src/packlets/database/database-backend.d.ts.map +1 -1
- package/dist/src/packlets/database/database-backend.js +5 -6
- package/dist/src/packlets/database/database-backend.js.map +1 -1
- package/dist/src/packlets/database/item-demuxer.d.ts +2 -2
- package/dist/src/packlets/database/item-demuxer.d.ts.map +1 -1
- package/dist/src/packlets/database/item-demuxer.js +2 -3
- package/dist/src/packlets/database/item-demuxer.js.map +1 -1
- package/dist/src/packlets/database/item-demuxer.test.js +13 -9
- package/dist/src/packlets/database/item-demuxer.test.js.map +1 -1
- package/dist/src/packlets/database/testing.d.ts.map +1 -1
- package/dist/src/packlets/database/testing.js +3 -4
- package/dist/src/packlets/database/testing.js.map +1 -1
- package/dist/src/packlets/database/timeframe-clock.d.ts +1 -0
- package/dist/src/packlets/database/timeframe-clock.d.ts.map +1 -1
- package/dist/src/packlets/database/timeframe-clock.js +3 -0
- package/dist/src/packlets/database/timeframe-clock.js.map +1 -1
- package/dist/src/parties/data-party.d.ts +6 -5
- package/dist/src/parties/data-party.d.ts.map +1 -1
- package/dist/src/parties/data-party.js +26 -5
- package/dist/src/parties/data-party.js.map +1 -1
- package/dist/src/parties/data-party.test.js +24 -5
- package/dist/src/parties/data-party.test.js.map +1 -1
- package/dist/src/parties/party-factory.d.ts +5 -4
- package/dist/src/parties/party-factory.d.ts.map +1 -1
- package/dist/src/parties/party-factory.js +13 -9
- package/dist/src/parties/party-factory.js.map +1 -1
- package/dist/src/parties/party-manager.d.ts +3 -4
- package/dist/src/parties/party-manager.d.ts.map +1 -1
- package/dist/src/parties/party-manager.js +8 -7
- package/dist/src/parties/party-manager.js.map +1 -1
- package/dist/src/parties/party-manager.test.js +3 -6
- package/dist/src/parties/party-manager.test.js.map +1 -1
- package/dist/src/pipeline/{pipeline.d.ts → feed-muxer.d.ts} +8 -11
- package/dist/src/pipeline/feed-muxer.d.ts.map +1 -0
- package/dist/src/pipeline/{pipeline.js → feed-muxer.js} +33 -40
- package/dist/src/pipeline/feed-muxer.js.map +1 -0
- package/dist/src/pipeline/feed-muxer.test.d.ts +2 -0
- package/dist/src/pipeline/feed-muxer.test.d.ts.map +1 -0
- package/dist/src/pipeline/{pipeline.test.js → feed-muxer.test.js} +16 -13
- package/dist/src/pipeline/feed-muxer.test.js.map +1 -0
- package/dist/src/pipeline/index.d.ts +2 -2
- package/dist/src/pipeline/index.d.ts.map +1 -1
- package/dist/src/pipeline/index.js +2 -2
- package/dist/src/pipeline/index.js.map +1 -1
- package/dist/src/pipeline/metadata-store.d.ts +2 -1
- package/dist/src/pipeline/metadata-store.d.ts.map +1 -1
- package/dist/src/pipeline/metadata-store.js +6 -0
- package/dist/src/pipeline/metadata-store.js.map +1 -1
- package/dist/src/pipeline/party-feed-provider.d.ts +2 -3
- package/dist/src/pipeline/party-feed-provider.d.ts.map +1 -1
- package/dist/src/pipeline/party-feed-provider.js +2 -17
- package/dist/src/pipeline/party-feed-provider.js.map +1 -1
- package/dist/src/pipeline/{party-core.d.ts → party-pipeline.d.ts} +23 -9
- package/dist/src/pipeline/party-pipeline.d.ts.map +1 -0
- package/dist/src/pipeline/{party-core.js → party-pipeline.js} +26 -25
- package/dist/src/pipeline/party-pipeline.js.map +1 -0
- package/dist/src/pipeline/party-pipeline.test.d.ts +2 -0
- package/dist/src/pipeline/party-pipeline.test.d.ts.map +1 -0
- package/dist/src/pipeline/{party-core.test.js → party-pipeline.test.js} +37 -21
- package/dist/src/pipeline/party-pipeline.test.js.map +1 -0
- package/dist/src/snapshots/snapshot-generator.d.ts +2 -2
- package/dist/src/snapshots/snapshot-generator.d.ts.map +1 -1
- package/dist/src/snapshots/snapshot-generator.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +17 -17
- package/src/echo.test.ts +0 -4
- package/src/echo.ts +1 -0
- package/src/halo/halo-factory.ts +6 -6
- package/src/halo/halo-party.ts +10 -8
- package/src/halo/halo.ts +2 -2
- package/src/halo/party-opener.ts +3 -1
- package/src/invitations/greeting-initiator.ts +5 -4
- package/src/packlets/database/data-mirror.test.ts +2 -2
- package/src/packlets/database/database-backend.ts +7 -6
- package/src/packlets/database/item-demuxer.test.ts +20 -18
- package/src/packlets/database/item-demuxer.ts +5 -4
- package/src/packlets/database/testing.ts +3 -6
- package/src/packlets/database/timeframe-clock.ts +4 -0
- package/src/parties/data-party.test.ts +39 -8
- package/src/parties/data-party.ts +31 -9
- package/src/parties/party-factory.ts +17 -15
- package/src/parties/party-manager.test.ts +4 -5
- package/src/parties/party-manager.ts +9 -6
- package/src/pipeline/{pipeline.test.ts → feed-muxer.test.ts} +19 -14
- package/src/pipeline/{pipeline.ts → feed-muxer.ts} +38 -51
- package/src/pipeline/index.ts +2 -2
- package/src/pipeline/metadata-store.ts +7 -1
- package/src/pipeline/party-feed-provider.ts +3 -16
- package/src/pipeline/{party-core.test.ts → party-pipeline.test.ts} +35 -14
- package/src/pipeline/{party-core.ts → party-pipeline.ts} +53 -24
- package/src/snapshots/snapshot-generator.ts +2 -2
- package/dist/src/pipeline/party-core.d.ts.map +0 -1
- package/dist/src/pipeline/party-core.js.map +0 -1
- package/dist/src/pipeline/party-core.test.d.ts +0 -2
- package/dist/src/pipeline/party-core.test.d.ts.map +0 -1
- package/dist/src/pipeline/party-core.test.js.map +0 -1
- package/dist/src/pipeline/pipeline.d.ts.map +0 -1
- package/dist/src/pipeline/pipeline.js.map +0 -1
- package/dist/src/pipeline/pipeline.test.d.ts +0 -2
- package/dist/src/pipeline/pipeline.test.d.ts.map +0 -1
- package/dist/src/pipeline/pipeline.test.js.map +0 -1
package/src/halo/halo-party.ts
CHANGED
|
@@ -14,7 +14,7 @@ import { NetworkManager } from '@dxos/network-manager';
|
|
|
14
14
|
|
|
15
15
|
import { InvitationAuthenticator, InvitationDescriptor, InvitationFactory, InvitationOptions } from '../invitations';
|
|
16
16
|
import { PARTY_ITEM_TYPE } from '../parties';
|
|
17
|
-
import { PartyFeedProvider, PartyProtocolFactory,
|
|
17
|
+
import { PartyFeedProvider, PartyProtocolFactory, PartyPipeline, PipelineOptions } from '../pipeline';
|
|
18
18
|
import { createAuthenticator, createAuthPlugin, createCredentialsProvider, createHaloRecoveryPlugin } from '../protocol';
|
|
19
19
|
import { CredentialsSigner } from '../protocol/credentials-signer';
|
|
20
20
|
import { createReplicatorPlugin } from '../protocol/replicator-plugin';
|
|
@@ -41,7 +41,7 @@ export interface JoinedParty {
|
|
|
41
41
|
export class HaloParty {
|
|
42
42
|
public readonly update = new Event<void>();
|
|
43
43
|
|
|
44
|
-
private readonly _partyCore:
|
|
44
|
+
private readonly _partyCore: PartyPipeline;
|
|
45
45
|
private _invitationManager?: InvitationFactory;
|
|
46
46
|
private _protocol?: PartyProtocolFactory;
|
|
47
47
|
|
|
@@ -54,17 +54,16 @@ export class HaloParty {
|
|
|
54
54
|
private readonly _feedProvider: PartyFeedProvider,
|
|
55
55
|
private readonly _credentialsSigner: CredentialsSigner,
|
|
56
56
|
private readonly _networkManager: NetworkManager,
|
|
57
|
-
private readonly
|
|
58
|
-
_initialTimeframe: Timeframe | undefined,
|
|
59
|
-
_options:
|
|
57
|
+
private readonly _feedHints: PublicKey[] = [],
|
|
58
|
+
private readonly _initialTimeframe: Timeframe | undefined,
|
|
59
|
+
_options: PipelineOptions
|
|
60
60
|
) {
|
|
61
|
-
this._partyCore = new
|
|
61
|
+
this._partyCore = new PartyPipeline(
|
|
62
62
|
_credentialsSigner.getIdentityKey().publicKey,
|
|
63
63
|
_feedProvider,
|
|
64
64
|
modelFactory,
|
|
65
65
|
snapshotStore,
|
|
66
66
|
_credentialsSigner.getIdentityKey().publicKey,
|
|
67
|
-
_initialTimeframe,
|
|
68
67
|
_options
|
|
69
68
|
);
|
|
70
69
|
|
|
@@ -145,7 +144,10 @@ export class HaloParty {
|
|
|
145
144
|
return this;
|
|
146
145
|
}
|
|
147
146
|
|
|
148
|
-
await this._partyCore.open(
|
|
147
|
+
await this._partyCore.open({
|
|
148
|
+
feedHints: this._feedHints,
|
|
149
|
+
initialTimeframe: this._initialTimeframe
|
|
150
|
+
});
|
|
149
151
|
|
|
150
152
|
this._invitationManager = new InvitationFactory(
|
|
151
153
|
this._partyCore.processor,
|
package/src/halo/halo.ts
CHANGED
|
@@ -15,7 +15,7 @@ import { NetworkManager } from '@dxos/network-manager';
|
|
|
15
15
|
import { ResultSet } from '../api';
|
|
16
16
|
import { InvitationAuthenticator, InvitationDescriptor, InvitationOptions } from '../invitations';
|
|
17
17
|
import { OpenProgress } from '../parties';
|
|
18
|
-
import { MetadataStore,
|
|
18
|
+
import { MetadataStore, PipelineOptions, PartyFeedProvider } from '../pipeline';
|
|
19
19
|
import { SnapshotStore } from '../snapshots';
|
|
20
20
|
import { Contact } from './contact-manager';
|
|
21
21
|
import { HaloFactory } from './halo-factory';
|
|
@@ -36,7 +36,7 @@ export interface HaloConfiguration {
|
|
|
36
36
|
modelFactory: ModelFactory,
|
|
37
37
|
snapshotStore: SnapshotStore,
|
|
38
38
|
feedProviderFactory: (partyKey: PublicKey) => PartyFeedProvider,
|
|
39
|
-
options:
|
|
39
|
+
options: PipelineOptions
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
/**
|
package/src/halo/party-opener.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import debug from 'debug';
|
|
6
6
|
|
|
7
|
+
import { KeyType } from '@dxos/credentials';
|
|
7
8
|
import { SubscriptionGroup, Unsubscribe } from '@dxos/util';
|
|
8
9
|
|
|
9
10
|
import { PartyManager } from '../parties';
|
|
@@ -25,7 +26,8 @@ export const autoPartyOpener = (preferences: Preferences, partyManager: PartyMan
|
|
|
25
26
|
for (const partyDesc of values) {
|
|
26
27
|
if (!partyManager.parties.some(x => x.key === partyDesc.partyKey)) {
|
|
27
28
|
log(`Auto-opening new Party from HALO: ${partyDesc.partyKey.toHex()} hints=${JSON.stringify(partyDesc.keyHints)}`);
|
|
28
|
-
|
|
29
|
+
const feedHints = partyDesc.keyHints.filter(hint => hint.type === KeyType.FEED).map(hint => hint.publicKey!);
|
|
30
|
+
await partyManager.addParty(partyDesc.partyKey, feedHints);
|
|
29
31
|
}
|
|
30
32
|
}
|
|
31
33
|
}));
|
|
@@ -20,7 +20,8 @@ import {
|
|
|
20
20
|
SecretProvider,
|
|
21
21
|
WithTypeUrl,
|
|
22
22
|
ERR_GREET_CONNECTED_TO_SWARM_TIMEOUT,
|
|
23
|
-
SignedMessage
|
|
23
|
+
SignedMessage,
|
|
24
|
+
NotarizeResponse
|
|
24
25
|
} from '@dxos/credentials';
|
|
25
26
|
import { keyToString, PublicKey } from '@dxos/crypto';
|
|
26
27
|
import { FullyConnectedTopology, NetworkManager } from '@dxos/network-manager';
|
|
@@ -111,7 +112,7 @@ export class GreetingInitiator {
|
|
|
111
112
|
/**
|
|
112
113
|
* Called after connecting to initiate greeting protocol exchange.
|
|
113
114
|
*/
|
|
114
|
-
async redeemInvitation (secretProvider: SecretProvider) {
|
|
115
|
+
async redeemInvitation (secretProvider: SecretProvider): Promise<{ partyKey: PublicKey, hints: PublicKey[] }> {
|
|
115
116
|
assert(this._state === GreetingState.CONNECTED);
|
|
116
117
|
const { swarmKey } = this._invitationDescriptor;
|
|
117
118
|
|
|
@@ -150,7 +151,7 @@ export class GreetingInitiator {
|
|
|
150
151
|
const credentialMessages = await this._getMessagesToNotarize(PublicKey.from(partyKey), nonce);
|
|
151
152
|
|
|
152
153
|
// Send the signed payload to the greeting responder.
|
|
153
|
-
const notarizeResponse = await this._greeterPlugin.send(responderPeerId,
|
|
154
|
+
const notarizeResponse: NotarizeResponse = await this._greeterPlugin.send(responderPeerId,
|
|
154
155
|
createGreetingNotarizeMessage(secret, credentialMessages as WithTypeUrl<Message>[]));
|
|
155
156
|
|
|
156
157
|
//
|
|
@@ -171,7 +172,7 @@ export class GreetingInitiator {
|
|
|
171
172
|
this._state = GreetingState.SUCCEEDED;
|
|
172
173
|
return {
|
|
173
174
|
partyKey,
|
|
174
|
-
hints: notarizeResponse.
|
|
175
|
+
hints: notarizeResponse.feedHints ?? []
|
|
175
176
|
};
|
|
176
177
|
}
|
|
177
178
|
|
|
@@ -26,8 +26,8 @@ describe('DataMirror', () => {
|
|
|
26
26
|
const itemManager = new ItemManager(modelFactory, PublicKey.random(), feed);
|
|
27
27
|
const itemDemuxer = new ItemDemuxer(itemManager, modelFactory, { snapshots: true });
|
|
28
28
|
|
|
29
|
-
const
|
|
30
|
-
feed.written.on(([msg, meta]) =>
|
|
29
|
+
const process = itemDemuxer.open();
|
|
30
|
+
feed.written.on(([msg, meta]) => process({
|
|
31
31
|
data: msg,
|
|
32
32
|
meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe() }
|
|
33
33
|
} as any));
|
|
@@ -10,7 +10,7 @@ import { ModelFactory } from '@dxos/model-factory';
|
|
|
10
10
|
|
|
11
11
|
import { DataMirror } from './data-mirror';
|
|
12
12
|
import { DataServiceHost } from './data-service-host';
|
|
13
|
-
import { ItemDemuxer, ItemDemuxerOptions } from './item-demuxer';
|
|
13
|
+
import { EchoProcessor, ItemDemuxer, ItemDemuxerOptions } from './item-demuxer';
|
|
14
14
|
import { ItemManager } from './item-manager';
|
|
15
15
|
|
|
16
16
|
const log = debug('dxos:echo-db:database-backend');
|
|
@@ -42,12 +42,11 @@ export interface DatabaseBackend {
|
|
|
42
42
|
* Write operations result in mutations being written to the outgoing stream.
|
|
43
43
|
*/
|
|
44
44
|
export class FeedDatabaseBackend implements DatabaseBackend {
|
|
45
|
-
private
|
|
45
|
+
private _echoProcessor!: EchoProcessor;
|
|
46
46
|
private _itemManager!: ItemManager;
|
|
47
47
|
private _itemDemuxer!: ItemDemuxer;
|
|
48
48
|
|
|
49
49
|
constructor (
|
|
50
|
-
private readonly _inboundStream: NodeJS.ReadableStream,
|
|
51
50
|
private readonly _outboundStream: FeedWriter<EchoEnvelope> | undefined,
|
|
52
51
|
private readonly _snapshot?: DatabaseSnapshot,
|
|
53
52
|
private readonly _options: ItemDemuxerOptions = {}
|
|
@@ -56,16 +55,18 @@ export class FeedDatabaseBackend implements DatabaseBackend {
|
|
|
56
55
|
async open (itemManager: ItemManager, modelFactory: ModelFactory) {
|
|
57
56
|
this._itemManager = itemManager;
|
|
58
57
|
this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory, this._options);
|
|
59
|
-
this.
|
|
60
|
-
this._inboundStream.pipe(this._itemDemuxerInboundStream);
|
|
58
|
+
this._echoProcessor = this._itemDemuxer.open();
|
|
61
59
|
|
|
62
60
|
if (this._snapshot) {
|
|
63
61
|
await this._itemDemuxer.restoreFromSnapshot(this._snapshot);
|
|
64
62
|
}
|
|
65
63
|
}
|
|
66
64
|
|
|
65
|
+
get echoProcessor () {
|
|
66
|
+
return this._echoProcessor;
|
|
67
|
+
}
|
|
68
|
+
|
|
67
69
|
async close () {
|
|
68
|
-
this._inboundStream?.unpipe(this._itemDemuxerInboundStream);
|
|
69
70
|
}
|
|
70
71
|
|
|
71
72
|
get isReadOnly (): boolean {
|
|
@@ -9,8 +9,7 @@ import { it as test } from 'mocha';
|
|
|
9
9
|
import { latch } from '@dxos/async';
|
|
10
10
|
import { createId, PublicKey } from '@dxos/crypto';
|
|
11
11
|
import { checkType } from '@dxos/debug';
|
|
12
|
-
import {
|
|
13
|
-
import { createTransform } from '@dxos/feed-store';
|
|
12
|
+
import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
|
|
14
13
|
import { ModelFactory, TestModel } from '@dxos/model-factory';
|
|
15
14
|
import { ObjectModel } from '@dxos/object-model';
|
|
16
15
|
|
|
@@ -32,7 +31,7 @@ describe('Item demuxer', () => {
|
|
|
32
31
|
const itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
|
|
33
32
|
|
|
34
33
|
const inboundStream = itemDemuxer.open();
|
|
35
|
-
feedWriter.written.on(([msg, meta]) => inboundStream
|
|
34
|
+
feedWriter.written.on(([msg, meta]) => inboundStream({
|
|
36
35
|
data: msg,
|
|
37
36
|
meta: { ...meta, memberKey }
|
|
38
37
|
} as any));
|
|
@@ -96,28 +95,31 @@ describe('Item demuxer', () => {
|
|
|
96
95
|
const modelFactory = new ModelFactory()
|
|
97
96
|
.registerModel(ObjectModel);
|
|
98
97
|
|
|
99
|
-
const
|
|
100
|
-
async (message
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
timeframe: new Timeframe()
|
|
106
|
-
},
|
|
107
|
-
data: message
|
|
108
|
-
})
|
|
109
|
-
);
|
|
110
|
-
const itemManager = new ItemManager(modelFactory, PublicKey.random(), createMockFeedWriterFromStream(writeStream));
|
|
98
|
+
const itemManager = new ItemManager(modelFactory, PublicKey.random(), {
|
|
99
|
+
write: async (message) => {
|
|
100
|
+
void processEchoMessage(message);
|
|
101
|
+
return { feedKey: PublicKey.random(), seq: 0 };
|
|
102
|
+
}
|
|
103
|
+
});
|
|
111
104
|
const itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
|
|
112
|
-
|
|
105
|
+
const processor = itemDemuxer.open();
|
|
106
|
+
const processEchoMessage = (message: EchoEnvelope) => processor({
|
|
107
|
+
meta: {
|
|
108
|
+
feedKey: PublicKey.random(),
|
|
109
|
+
memberKey: PublicKey.random(),
|
|
110
|
+
seq: 0,
|
|
111
|
+
timeframe: new Timeframe()
|
|
112
|
+
},
|
|
113
|
+
data: message
|
|
114
|
+
});
|
|
113
115
|
|
|
114
|
-
|
|
116
|
+
void processEchoMessage(checkType<EchoEnvelope>({
|
|
115
117
|
itemId: 'foo',
|
|
116
118
|
genesis: {
|
|
117
119
|
modelType: TestModel.meta.type
|
|
118
120
|
}
|
|
119
121
|
}));
|
|
120
|
-
|
|
122
|
+
void processEchoMessage(checkType<EchoEnvelope>({
|
|
121
123
|
itemId: 'bar',
|
|
122
124
|
genesis: {
|
|
123
125
|
modelType: ObjectModel.meta.type
|
|
@@ -8,7 +8,6 @@ import debug from 'debug';
|
|
|
8
8
|
import { Event } from '@dxos/async';
|
|
9
9
|
import { failUndefined } from '@dxos/debug';
|
|
10
10
|
import { DatabaseSnapshot, IEchoStream, ItemID, ItemSnapshot, LinkSnapshot } from '@dxos/echo-protocol';
|
|
11
|
-
import { createWritable } from '@dxos/feed-store';
|
|
12
11
|
import { Model, ModelFactory, ModelMessage } from '@dxos/model-factory';
|
|
13
12
|
|
|
14
13
|
import { Entity } from './entity';
|
|
@@ -22,6 +21,8 @@ export interface ItemDemuxerOptions {
|
|
|
22
21
|
snapshots?: boolean
|
|
23
22
|
}
|
|
24
23
|
|
|
24
|
+
export type EchoProcessor = (message: IEchoStream) => Promise<void>
|
|
25
|
+
|
|
25
26
|
/**
|
|
26
27
|
* Creates a stream that consumes `IEchoStream` messages and routes them to the associated items.
|
|
27
28
|
* @param itemManager
|
|
@@ -35,7 +36,7 @@ export class ItemDemuxer {
|
|
|
35
36
|
private readonly _options: ItemDemuxerOptions = {}
|
|
36
37
|
) {}
|
|
37
38
|
|
|
38
|
-
open ():
|
|
39
|
+
open (): EchoProcessor {
|
|
39
40
|
this._modelFactory.registered.on(async model => {
|
|
40
41
|
for (const item of this._itemManager.getUninitializedEntities()) {
|
|
41
42
|
if (item._stateManager.modelType === model.meta.type) {
|
|
@@ -46,7 +47,7 @@ export class ItemDemuxer {
|
|
|
46
47
|
|
|
47
48
|
// TODO(burdon): Factor out.
|
|
48
49
|
// TODO(burdon): Should this implement some "back-pressure" (hints) to the PartyProcessor?
|
|
49
|
-
return
|
|
50
|
+
return async (message: IEchoStream) => {
|
|
50
51
|
const { data: { itemId, genesis, itemMutation, mutation, snapshot }, meta } = message;
|
|
51
52
|
assert(itemId);
|
|
52
53
|
|
|
@@ -111,7 +112,7 @@ export class ItemDemuxer {
|
|
|
111
112
|
}
|
|
112
113
|
|
|
113
114
|
this.mutation.emit(message);
|
|
114
|
-
}
|
|
115
|
+
};
|
|
115
116
|
}
|
|
116
117
|
|
|
117
118
|
createSnapshot (): DatabaseSnapshot {
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
// Copyright 2021 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { Readable } from 'stream';
|
|
6
|
-
|
|
7
5
|
import { PublicKey } from '@dxos/crypto';
|
|
8
6
|
import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
|
|
9
7
|
import { ModelFactory } from '@dxos/model-factory';
|
|
@@ -15,12 +13,11 @@ import { FeedDatabaseBackend, RemoteDatabaseBackend } from './database-backend';
|
|
|
15
13
|
|
|
16
14
|
export const createInMemoryDatabase = async (modelFactory: ModelFactory) => {
|
|
17
15
|
const feed = new MockFeedWriter<EchoEnvelope>();
|
|
18
|
-
const
|
|
19
|
-
feed.written.on(([data, meta]) =>
|
|
20
|
-
|
|
16
|
+
const backend = new FeedDatabaseBackend(feed, undefined, { snapshots: true });
|
|
17
|
+
feed.written.on(([data, meta]) => backend.echoProcessor({ data, meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe([[meta.feedKey, meta.seq]]) } }));
|
|
21
18
|
const database = new Database(
|
|
22
19
|
modelFactory,
|
|
23
|
-
|
|
20
|
+
backend,
|
|
24
21
|
PublicKey.random()
|
|
25
22
|
);
|
|
26
23
|
|
|
@@ -28,4 +28,8 @@ export class TimeframeClock {
|
|
|
28
28
|
const gaps = Timeframe.dependencies(timeframe, this._timeframe);
|
|
29
29
|
return !gaps.isEmpty();
|
|
30
30
|
}
|
|
31
|
+
|
|
32
|
+
async waitUntilReached (target: Timeframe) {
|
|
33
|
+
await this.update.waitForCondition(() => Timeframe.dependencies(target, this._timeframe).isEmpty());
|
|
34
|
+
}
|
|
31
35
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import expect from 'expect';
|
|
6
6
|
import { it as test } from 'mocha';
|
|
7
7
|
|
|
8
|
-
import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider,
|
|
8
|
+
import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
|
|
9
9
|
import { PublicKey } from '@dxos/crypto';
|
|
10
10
|
import { codec } from '@dxos/echo-protocol';
|
|
11
11
|
import { FeedStore } from '@dxos/feed-store';
|
|
@@ -22,7 +22,7 @@ import { SnapshotStore } from '../snapshots';
|
|
|
22
22
|
import { DataParty } from './data-party';
|
|
23
23
|
|
|
24
24
|
describe('DataParty', () => {
|
|
25
|
-
const createParty = async (identity: IdentityCredentials, partyKey: PublicKey,
|
|
25
|
+
const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, feedHints: PublicKey[]) => {
|
|
26
26
|
|
|
27
27
|
const storage = createStorage('', StorageType.RAM);
|
|
28
28
|
const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
|
|
@@ -31,17 +31,20 @@ describe('DataParty', () => {
|
|
|
31
31
|
const modelFactory = new ModelFactory().registerModel(ObjectModel);
|
|
32
32
|
const networkManager = new NetworkManager();
|
|
33
33
|
const partyFeedProvider = new PartyFeedProvider(metadataStore, identity.keyring, feedStore, partyKey);
|
|
34
|
+
const writableFeed = await partyFeedProvider.createOrOpenWritableFeed();
|
|
34
35
|
|
|
35
|
-
|
|
36
|
+
const party = new DataParty(
|
|
36
37
|
partyKey,
|
|
37
38
|
modelFactory,
|
|
38
39
|
snapshotStore,
|
|
39
40
|
partyFeedProvider,
|
|
41
|
+
metadataStore,
|
|
40
42
|
identity.createCredentialsSigner(),
|
|
41
43
|
identity.preferences,
|
|
42
|
-
networkManager
|
|
43
|
-
hints
|
|
44
|
+
networkManager
|
|
44
45
|
);
|
|
46
|
+
party._setFeedHints([...feedHints, writableFeed.key]);
|
|
47
|
+
return party;
|
|
45
48
|
};
|
|
46
49
|
|
|
47
50
|
test('open & close', async () => {
|
|
@@ -74,6 +77,33 @@ describe('DataParty', () => {
|
|
|
74
77
|
await party.close();
|
|
75
78
|
});
|
|
76
79
|
|
|
80
|
+
test('data is immediately available after re-opening', async () => {
|
|
81
|
+
const keyring = new Keyring();
|
|
82
|
+
const identity = await createTestIdentityCredentials(keyring);
|
|
83
|
+
const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
|
|
84
|
+
const party = await createParty(identity, partyKey.publicKey, []);
|
|
85
|
+
await party.open();
|
|
86
|
+
|
|
87
|
+
const feed = await party.getWriteFeed();
|
|
88
|
+
await party.credentialsWriter.write(createPartyGenesisMessage(
|
|
89
|
+
keyring,
|
|
90
|
+
partyKey,
|
|
91
|
+
feed.key,
|
|
92
|
+
partyKey
|
|
93
|
+
));
|
|
94
|
+
|
|
95
|
+
for (let i = 0; i < 10; i++) {
|
|
96
|
+
await party.database.createItem({ type: 'test:item' });
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
await party.close();
|
|
100
|
+
await party.open();
|
|
101
|
+
|
|
102
|
+
expect(party.database.select({ type: 'test:item' }).exec().entities).toHaveLength(10);
|
|
103
|
+
|
|
104
|
+
await party.close();
|
|
105
|
+
});
|
|
106
|
+
|
|
77
107
|
test('authenticates its own credentials', async () => {
|
|
78
108
|
const keyring = new Keyring();
|
|
79
109
|
const identity = await createTestIdentityCredentials(keyring);
|
|
@@ -88,6 +118,7 @@ describe('DataParty', () => {
|
|
|
88
118
|
feed.key,
|
|
89
119
|
partyKey
|
|
90
120
|
));
|
|
121
|
+
await party.processor.feedAdded.waitForCount(1);
|
|
91
122
|
|
|
92
123
|
const authenticator = createAuthenticator(party.processor, identity.createCredentialsSigner(), party.credentialsWriter);
|
|
93
124
|
const credentialsProvider = createCredentialsProvider(identity.createCredentialsSigner(), party.key, feed.key);
|
|
@@ -112,6 +143,8 @@ describe('DataParty', () => {
|
|
|
112
143
|
feed.key,
|
|
113
144
|
partyKey
|
|
114
145
|
));
|
|
146
|
+
await party.processor.feedAdded.waitForCount(1);
|
|
147
|
+
|
|
115
148
|
const authenticator = createAuthenticator(party.processor, identityA.createCredentialsSigner(), party.credentialsWriter);
|
|
116
149
|
|
|
117
150
|
const identityB = await deriveTestDeviceCredentials(identityA);
|
|
@@ -145,9 +178,7 @@ describe('DataParty', () => {
|
|
|
145
178
|
));
|
|
146
179
|
|
|
147
180
|
const identityB = await deriveTestDeviceCredentials(identityA);
|
|
148
|
-
const partyB = await createParty(identityB, partyKey.publicKey, [
|
|
149
|
-
{ type: KeyType.FEED, publicKey: feedA.key }
|
|
150
|
-
]);
|
|
181
|
+
const partyB = await createParty(identityB, partyKey.publicKey, [feedA.key]);
|
|
151
182
|
await partyB.open();
|
|
152
183
|
|
|
153
184
|
await partyA.database.createItem({ type: 'test:item-a' });
|
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
import assert from 'assert';
|
|
6
6
|
|
|
7
7
|
import { synchronized, Event } from '@dxos/async';
|
|
8
|
-
import { KeyHint } from '@dxos/credentials';
|
|
9
8
|
import { PublicKey } from '@dxos/crypto';
|
|
10
9
|
import { timed } from '@dxos/debug';
|
|
11
10
|
import { PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
|
|
@@ -18,7 +17,7 @@ import { ResultSet } from '../api';
|
|
|
18
17
|
import { ActivationOptions, PartyPreferences, Preferences } from '../halo';
|
|
19
18
|
import { InvitationFactory } from '../invitations';
|
|
20
19
|
import { Database, Item } from '../packlets/database';
|
|
21
|
-
import { PartyFeedProvider, PartyProtocolFactory,
|
|
20
|
+
import { PartyFeedProvider, PartyProtocolFactory, PartyPipeline, PipelineOptions, MetadataStore } from '../pipeline';
|
|
22
21
|
import { createAuthPlugin, createOfflineInvitationPlugin, createAuthenticator, createCredentialsProvider } from '../protocol';
|
|
23
22
|
import { CredentialsSigner } from '../protocol/credentials-signer';
|
|
24
23
|
import { createReplicatorPlugin } from '../protocol/replicator-plugin';
|
|
@@ -43,31 +42,31 @@ export interface PartyMember {
|
|
|
43
42
|
export class DataParty {
|
|
44
43
|
public readonly update = new Event<void>();
|
|
45
44
|
|
|
46
|
-
private readonly _partyCore:
|
|
45
|
+
private readonly _partyCore: PartyPipeline;
|
|
47
46
|
private readonly _preferences?: PartyPreferences;
|
|
48
47
|
private _invitationManager?: InvitationFactory;
|
|
49
48
|
private _protocol?: PartyProtocolFactory;
|
|
49
|
+
private _feedHints: PublicKey[] = []
|
|
50
50
|
|
|
51
51
|
constructor (
|
|
52
52
|
partyKey: PartyKey,
|
|
53
53
|
modelFactory: ModelFactory,
|
|
54
54
|
snapshotStore: SnapshotStore,
|
|
55
55
|
private readonly _feedProvider: PartyFeedProvider,
|
|
56
|
+
private readonly _metadataStore: MetadataStore,
|
|
56
57
|
private readonly _credentialsSigner: CredentialsSigner,
|
|
57
58
|
// TODO(dmaretskyi): Pull this out to a higher level. Should preferences be part of client API instead?
|
|
58
59
|
private readonly _profilePreferences: Preferences | undefined,
|
|
59
60
|
private readonly _networkManager: NetworkManager,
|
|
60
|
-
private readonly
|
|
61
|
-
|
|
62
|
-
_options: PartyOptions = {}
|
|
61
|
+
private readonly _initialTimeframe?: Timeframe,
|
|
62
|
+
_options: PipelineOptions = {}
|
|
63
63
|
) {
|
|
64
|
-
this._partyCore = new
|
|
64
|
+
this._partyCore = new PartyPipeline(
|
|
65
65
|
partyKey,
|
|
66
66
|
_feedProvider,
|
|
67
67
|
modelFactory,
|
|
68
68
|
snapshotStore,
|
|
69
69
|
this._credentialsSigner.getIdentityKey().publicKey,
|
|
70
|
-
_initialTimeframe,
|
|
71
70
|
_options
|
|
72
71
|
);
|
|
73
72
|
|
|
@@ -142,6 +141,13 @@ export class DataParty {
|
|
|
142
141
|
await this._preferences?.setLastKnownTitle(title);
|
|
143
142
|
}
|
|
144
143
|
|
|
144
|
+
/**
|
|
145
|
+
* @internal
|
|
146
|
+
*/
|
|
147
|
+
_setFeedHints (feedHints: PublicKey[]) {
|
|
148
|
+
this._feedHints = feedHints;
|
|
149
|
+
}
|
|
150
|
+
|
|
145
151
|
/**
|
|
146
152
|
* Opens the pipeline and connects the streams.
|
|
147
153
|
*/
|
|
@@ -152,7 +158,20 @@ export class DataParty {
|
|
|
152
158
|
return this;
|
|
153
159
|
}
|
|
154
160
|
|
|
155
|
-
|
|
161
|
+
// TODO(dmaretskyi): May be undefined in some tests.
|
|
162
|
+
const party = this._metadataStore.getParty(this._partyCore.key);
|
|
163
|
+
|
|
164
|
+
await this._partyCore.open({
|
|
165
|
+
feedHints: this._feedHints,
|
|
166
|
+
initialTimeframe: this._initialTimeframe,
|
|
167
|
+
targetTimeframe: party?.latestTimeframe
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
// Keep updating latest reached timeframe in the metadata.
|
|
171
|
+
// This timeframe will be waited for when opening the party next time.
|
|
172
|
+
this._partyCore.timeframeUpdate.on(timeframe => {
|
|
173
|
+
void this._metadataStore.setTimeframe(this._partyCore.key, timeframe);
|
|
174
|
+
});
|
|
156
175
|
|
|
157
176
|
this._invitationManager = new InvitationFactory(
|
|
158
177
|
this._partyCore.processor,
|
|
@@ -197,6 +216,9 @@ export class DataParty {
|
|
|
197
216
|
return this;
|
|
198
217
|
}
|
|
199
218
|
|
|
219
|
+
// Save the latest reached timeframe.
|
|
220
|
+
await this._metadataStore.setTimeframe(this._partyCore.key, this._partyCore.timeframe);
|
|
221
|
+
|
|
200
222
|
await this._partyCore.close();
|
|
201
223
|
await this._protocol?.stop();
|
|
202
224
|
|
|
@@ -9,7 +9,6 @@ import {
|
|
|
9
9
|
createEnvelopeMessage,
|
|
10
10
|
createFeedAdmitMessage,
|
|
11
11
|
createPartyGenesisMessage,
|
|
12
|
-
KeyHint,
|
|
13
12
|
KeyType,
|
|
14
13
|
SecretProvider,
|
|
15
14
|
wrapMessage
|
|
@@ -26,7 +25,7 @@ import {
|
|
|
26
25
|
GreetingInitiator, InvitationDescriptor, InvitationDescriptorType, OfflineInvitationClaimer
|
|
27
26
|
} from '../invitations';
|
|
28
27
|
import { IdentityNotInitializedError } from '../packlets/errors';
|
|
29
|
-
import { PartyFeedProvider,
|
|
28
|
+
import { MetadataStore, PartyFeedProvider, PipelineOptions } from '../pipeline';
|
|
30
29
|
import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
|
|
31
30
|
import { SnapshotStore } from '../snapshots';
|
|
32
31
|
import { DataParty, PARTY_ITEM_TYPE } from './data-party';
|
|
@@ -43,7 +42,8 @@ export class PartyFactory {
|
|
|
43
42
|
private readonly _modelFactory: ModelFactory,
|
|
44
43
|
private readonly _snapshotStore: SnapshotStore,
|
|
45
44
|
private readonly _feedProviderFactory: (partyKey: PublicKey) => PartyFeedProvider,
|
|
46
|
-
private readonly
|
|
45
|
+
private readonly _metadataStore: MetadataStore,
|
|
46
|
+
private readonly _options: PipelineOptions = {}
|
|
47
47
|
) {}
|
|
48
48
|
|
|
49
49
|
/**
|
|
@@ -57,11 +57,13 @@ export class PartyFactory {
|
|
|
57
57
|
const partyKey = await identity.keyring.createKeyRecord({ type: KeyType.PARTY });
|
|
58
58
|
const party = await this.constructParty(partyKey.publicKey);
|
|
59
59
|
|
|
60
|
+
const writableFeed = await party.getWriteFeed();
|
|
61
|
+
// Hint at the newly created writable feed so that we can start replicating from it.
|
|
62
|
+
party._setFeedHints([writableFeed.key]);
|
|
63
|
+
|
|
60
64
|
// Connect the pipeline.
|
|
61
65
|
await party.open();
|
|
62
66
|
|
|
63
|
-
const writableFeed = await party.getWriteFeed();
|
|
64
|
-
|
|
65
67
|
// PartyGenesis (self-signed by Party).
|
|
66
68
|
await party.credentialsWriter.write(createPartyGenesisMessage(
|
|
67
69
|
identity.keyring,
|
|
@@ -111,12 +113,9 @@ export class PartyFactory {
|
|
|
111
113
|
* @param partyKey
|
|
112
114
|
* @param hints
|
|
113
115
|
*/
|
|
114
|
-
async constructParty (partyKey: PartyKey,
|
|
116
|
+
async constructParty (partyKey: PartyKey, initialTimeframe?: Timeframe) {
|
|
115
117
|
const identity = this._identityProvider() ?? raise(new IdentityNotInitializedError());
|
|
116
118
|
|
|
117
|
-
// TODO(marik-d): Support read-only parties if this feed doesn't exist?
|
|
118
|
-
const feedProvider = this._feedProviderFactory(partyKey);
|
|
119
|
-
|
|
120
119
|
//
|
|
121
120
|
// Create the party.
|
|
122
121
|
//
|
|
@@ -124,11 +123,11 @@ export class PartyFactory {
|
|
|
124
123
|
partyKey,
|
|
125
124
|
this._modelFactory,
|
|
126
125
|
this._snapshotStore,
|
|
127
|
-
|
|
126
|
+
this._feedProviderFactory(partyKey),
|
|
127
|
+
this._metadataStore,
|
|
128
128
|
identity.createCredentialsSigner(),
|
|
129
129
|
identity.preferences,
|
|
130
130
|
this._networkManager,
|
|
131
|
-
hints,
|
|
132
131
|
initialTimeframe,
|
|
133
132
|
this._options
|
|
134
133
|
);
|
|
@@ -140,7 +139,7 @@ export class PartyFactory {
|
|
|
140
139
|
assert(snapshot.partyKey);
|
|
141
140
|
log(`Constructing ${humanize(snapshot.partyKey)} from snapshot at ${JSON.stringify(snapshot.timeframe)}.`);
|
|
142
141
|
|
|
143
|
-
const party = await this.constructParty(PublicKey.from(snapshot.partyKey),
|
|
142
|
+
const party = await this.constructParty(PublicKey.from(snapshot.partyKey), snapshot.timeframe);
|
|
144
143
|
await party.restoreFromSnapshot(snapshot);
|
|
145
144
|
return party;
|
|
146
145
|
}
|
|
@@ -173,7 +172,8 @@ export class PartyFactory {
|
|
|
173
172
|
|
|
174
173
|
await initiator.connect();
|
|
175
174
|
const { partyKey, hints } = await initiator.redeemInvitation(secretProvider);
|
|
176
|
-
const party = await this.constructParty(partyKey
|
|
175
|
+
const party = await this.constructParty(partyKey);
|
|
176
|
+
party._setFeedHints(hints);
|
|
177
177
|
await party.open();
|
|
178
178
|
await initiator.destroy();
|
|
179
179
|
|
|
@@ -198,11 +198,13 @@ export class PartyFactory {
|
|
|
198
198
|
const partyKey = await identity.keyring.createKeyRecord({ type: KeyType.PARTY });
|
|
199
199
|
const party = await this.constructParty(partyKey.publicKey);
|
|
200
200
|
|
|
201
|
+
const writableFeed = await party.getWriteFeed();
|
|
202
|
+
// Hint at the newly created writable feed so that we can start replicating from it.
|
|
203
|
+
party._setFeedHints([writableFeed.key]);
|
|
204
|
+
|
|
201
205
|
// Connect the pipeline.
|
|
202
206
|
await party.open();
|
|
203
207
|
|
|
204
|
-
const writableFeed = await party.getWriteFeed();
|
|
205
|
-
|
|
206
208
|
// PartyGenesis (self-signed by Party).
|
|
207
209
|
await party.credentialsWriter.write(createPartyGenesisMessage(
|
|
208
210
|
identity.keyring,
|
|
@@ -71,6 +71,7 @@ const setup = async () => {
|
|
|
71
71
|
modelFactory,
|
|
72
72
|
snapshotStore,
|
|
73
73
|
feedProviderFactory,
|
|
74
|
+
metadataStore,
|
|
74
75
|
{
|
|
75
76
|
writeLogger: messageLogger('<<<'),
|
|
76
77
|
readLogger: messageLogger('>>>')
|
|
@@ -140,10 +141,7 @@ describe('Party manager', () => {
|
|
|
140
141
|
const feedStream = createWritableFeedStream(feed);
|
|
141
142
|
feedStream.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
|
|
142
143
|
|
|
143
|
-
await partyManager.addParty(partyKey.publicKey, [
|
|
144
|
-
type: KeyType.FEED,
|
|
145
|
-
publicKey: PublicKey.from(feed.key)
|
|
146
|
-
}]);
|
|
144
|
+
await partyManager.addParty(partyKey.publicKey, [PublicKey.from(feed.key)]);
|
|
147
145
|
|
|
148
146
|
await update;
|
|
149
147
|
});
|
|
@@ -164,7 +162,8 @@ describe('Party manager', () => {
|
|
|
164
162
|
networkManager,
|
|
165
163
|
modelFactory,
|
|
166
164
|
snapshotStore,
|
|
167
|
-
feedProviderFactory
|
|
165
|
+
feedProviderFactory,
|
|
166
|
+
metadataStore
|
|
168
167
|
);
|
|
169
168
|
const partyManager = new PartyManager(metadataStore, snapshotStore, () => identity, partyFactory);
|
|
170
169
|
|