@dxos/echo-db 2.33.5-dev.ebc105f7 → 2.33.5-dev.fa6b779b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/echo.d.ts.map +1 -1
- package/dist/src/echo.js +1 -1
- package/dist/src/echo.js.map +1 -1
- package/dist/src/echo.test.js +12 -4
- package/dist/src/echo.test.js.map +1 -1
- package/dist/src/halo/halo-factory.d.ts +4 -4
- package/dist/src/halo/halo-factory.d.ts.map +1 -1
- package/dist/src/halo/halo-factory.js +3 -2
- package/dist/src/halo/halo-factory.js.map +1 -1
- package/dist/src/halo/halo-party.d.ts +4 -3
- package/dist/src/halo/halo-party.d.ts.map +1 -1
- package/dist/src/halo/halo-party.js +8 -4
- package/dist/src/halo/halo-party.js.map +1 -1
- package/dist/src/halo/halo.d.ts +2 -2
- package/dist/src/halo/halo.d.ts.map +1 -1
- package/dist/src/halo/party-opener.d.ts.map +1 -1
- package/dist/src/halo/party-opener.js +3 -1
- package/dist/src/halo/party-opener.js.map +1 -1
- package/dist/src/invitations/greeting-initiator.d.ts +2 -2
- package/dist/src/invitations/greeting-initiator.d.ts.map +1 -1
- package/dist/src/invitations/greeting-initiator.js +2 -1
- package/dist/src/invitations/greeting-initiator.js.map +1 -1
- package/dist/src/packlets/database/data-mirror.test.js +2 -2
- package/dist/src/packlets/database/data-mirror.test.js.map +1 -1
- package/dist/src/packlets/database/database-backend.d.ts +4 -5
- package/dist/src/packlets/database/database-backend.d.ts.map +1 -1
- package/dist/src/packlets/database/database-backend.js +5 -6
- package/dist/src/packlets/database/database-backend.js.map +1 -1
- package/dist/src/packlets/database/item-demuxer.d.ts +2 -2
- package/dist/src/packlets/database/item-demuxer.d.ts.map +1 -1
- package/dist/src/packlets/database/item-demuxer.js +2 -3
- package/dist/src/packlets/database/item-demuxer.js.map +1 -1
- package/dist/src/packlets/database/item-demuxer.test.js +13 -9
- package/dist/src/packlets/database/item-demuxer.test.js.map +1 -1
- package/dist/src/packlets/database/testing.d.ts.map +1 -1
- package/dist/src/packlets/database/testing.js +3 -4
- package/dist/src/packlets/database/testing.js.map +1 -1
- package/dist/src/packlets/database/timeframe-clock.d.ts +1 -0
- package/dist/src/packlets/database/timeframe-clock.d.ts.map +1 -1
- package/dist/src/packlets/database/timeframe-clock.js +3 -0
- package/dist/src/packlets/database/timeframe-clock.js.map +1 -1
- package/dist/src/parties/data-party.d.ts +6 -5
- package/dist/src/parties/data-party.d.ts.map +1 -1
- package/dist/src/parties/data-party.js +26 -5
- package/dist/src/parties/data-party.js.map +1 -1
- package/dist/src/parties/data-party.test.js +24 -5
- package/dist/src/parties/data-party.test.js.map +1 -1
- package/dist/src/parties/party-factory.d.ts +5 -4
- package/dist/src/parties/party-factory.d.ts.map +1 -1
- package/dist/src/parties/party-factory.js +13 -9
- package/dist/src/parties/party-factory.js.map +1 -1
- package/dist/src/parties/party-manager.d.ts +3 -4
- package/dist/src/parties/party-manager.d.ts.map +1 -1
- package/dist/src/parties/party-manager.js +8 -7
- package/dist/src/parties/party-manager.js.map +1 -1
- package/dist/src/parties/party-manager.test.js +3 -6
- package/dist/src/parties/party-manager.test.js.map +1 -1
- package/dist/src/pipeline/{pipeline.d.ts → feed-muxer.d.ts} +8 -11
- package/dist/src/pipeline/feed-muxer.d.ts.map +1 -0
- package/dist/src/pipeline/{pipeline.js → feed-muxer.js} +33 -40
- package/dist/src/pipeline/feed-muxer.js.map +1 -0
- package/dist/src/pipeline/feed-muxer.test.d.ts +2 -0
- package/dist/src/pipeline/feed-muxer.test.d.ts.map +1 -0
- package/dist/src/pipeline/{pipeline.test.js → feed-muxer.test.js} +16 -13
- package/dist/src/pipeline/feed-muxer.test.js.map +1 -0
- package/dist/src/pipeline/index.d.ts +2 -2
- package/dist/src/pipeline/index.d.ts.map +1 -1
- package/dist/src/pipeline/index.js +2 -2
- package/dist/src/pipeline/index.js.map +1 -1
- package/dist/src/pipeline/metadata-store.d.ts +2 -1
- package/dist/src/pipeline/metadata-store.d.ts.map +1 -1
- package/dist/src/pipeline/metadata-store.js +7 -1
- package/dist/src/pipeline/metadata-store.js.map +1 -1
- package/dist/src/pipeline/party-feed-provider.d.ts +2 -3
- package/dist/src/pipeline/party-feed-provider.d.ts.map +1 -1
- package/dist/src/pipeline/party-feed-provider.js +2 -17
- package/dist/src/pipeline/party-feed-provider.js.map +1 -1
- package/dist/src/pipeline/{party-core.d.ts → party-pipeline.d.ts} +23 -9
- package/dist/src/pipeline/party-pipeline.d.ts.map +1 -0
- package/dist/src/pipeline/{party-core.js → party-pipeline.js} +26 -25
- package/dist/src/pipeline/party-pipeline.js.map +1 -0
- package/dist/src/pipeline/party-pipeline.test.d.ts +2 -0
- package/dist/src/pipeline/party-pipeline.test.d.ts.map +1 -0
- package/dist/src/pipeline/{party-core.test.js → party-pipeline.test.js} +37 -21
- package/dist/src/pipeline/party-pipeline.test.js.map +1 -0
- package/dist/src/snapshots/snapshot-generator.d.ts +2 -2
- package/dist/src/snapshots/snapshot-generator.d.ts.map +1 -1
- package/dist/src/snapshots/snapshot-generator.js.map +1 -1
- package/dist/src/snapshots/snapshot-store.js +1 -1
- package/dist/src/snapshots/snapshot-store.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +17 -17
- package/src/echo.test.ts +15 -4
- package/src/echo.ts +1 -0
- package/src/halo/halo-factory.ts +6 -6
- package/src/halo/halo-party.ts +10 -8
- package/src/halo/halo.ts +2 -2
- package/src/halo/party-opener.ts +3 -1
- package/src/invitations/greeting-initiator.ts +5 -4
- package/src/packlets/database/data-mirror.test.ts +2 -2
- package/src/packlets/database/database-backend.ts +7 -6
- package/src/packlets/database/item-demuxer.test.ts +20 -18
- package/src/packlets/database/item-demuxer.ts +5 -4
- package/src/packlets/database/testing.ts +3 -6
- package/src/packlets/database/timeframe-clock.ts +4 -0
- package/src/parties/data-party.test.ts +39 -8
- package/src/parties/data-party.ts +31 -9
- package/src/parties/party-factory.ts +17 -15
- package/src/parties/party-manager.test.ts +4 -5
- package/src/parties/party-manager.ts +9 -6
- package/src/pipeline/{pipeline.test.ts → feed-muxer.test.ts} +19 -14
- package/src/pipeline/{pipeline.ts → feed-muxer.ts} +38 -51
- package/src/pipeline/index.ts +2 -2
- package/src/pipeline/metadata-store.ts +8 -2
- package/src/pipeline/party-feed-provider.ts +3 -16
- package/src/pipeline/{party-core.test.ts → party-pipeline.test.ts} +35 -14
- package/src/pipeline/{party-core.ts → party-pipeline.ts} +53 -24
- package/src/snapshots/snapshot-generator.ts +2 -2
- package/src/snapshots/snapshot-store.ts +1 -1
- package/dist/src/pipeline/party-core.d.ts.map +0 -1
- package/dist/src/pipeline/party-core.js.map +0 -1
- package/dist/src/pipeline/party-core.test.d.ts +0 -2
- package/dist/src/pipeline/party-core.test.d.ts.map +0 -1
- package/dist/src/pipeline/party-core.test.js.map +0 -1
- package/dist/src/pipeline/pipeline.d.ts.map +0 -1
- package/dist/src/pipeline/pipeline.js.map +0 -1
- package/dist/src/pipeline/pipeline.test.d.ts +0 -2
- package/dist/src/pipeline/pipeline.test.d.ts.map +0 -1
- package/dist/src/pipeline/pipeline.test.js.map +0 -1
|
@@ -9,7 +9,6 @@ import {
|
|
|
9
9
|
createEnvelopeMessage,
|
|
10
10
|
createFeedAdmitMessage,
|
|
11
11
|
createPartyGenesisMessage,
|
|
12
|
-
KeyHint,
|
|
13
12
|
KeyType,
|
|
14
13
|
SecretProvider,
|
|
15
14
|
wrapMessage
|
|
@@ -26,7 +25,7 @@ import {
|
|
|
26
25
|
GreetingInitiator, InvitationDescriptor, InvitationDescriptorType, OfflineInvitationClaimer
|
|
27
26
|
} from '../invitations';
|
|
28
27
|
import { IdentityNotInitializedError } from '../packlets/errors';
|
|
29
|
-
import { PartyFeedProvider,
|
|
28
|
+
import { MetadataStore, PartyFeedProvider, PipelineOptions } from '../pipeline';
|
|
30
29
|
import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
|
|
31
30
|
import { SnapshotStore } from '../snapshots';
|
|
32
31
|
import { DataParty, PARTY_ITEM_TYPE } from './data-party';
|
|
@@ -43,7 +42,8 @@ export class PartyFactory {
|
|
|
43
42
|
private readonly _modelFactory: ModelFactory,
|
|
44
43
|
private readonly _snapshotStore: SnapshotStore,
|
|
45
44
|
private readonly _feedProviderFactory: (partyKey: PublicKey) => PartyFeedProvider,
|
|
46
|
-
private readonly
|
|
45
|
+
private readonly _metadataStore: MetadataStore,
|
|
46
|
+
private readonly _options: PipelineOptions = {}
|
|
47
47
|
) {}
|
|
48
48
|
|
|
49
49
|
/**
|
|
@@ -57,11 +57,13 @@ export class PartyFactory {
|
|
|
57
57
|
const partyKey = await identity.keyring.createKeyRecord({ type: KeyType.PARTY });
|
|
58
58
|
const party = await this.constructParty(partyKey.publicKey);
|
|
59
59
|
|
|
60
|
+
const writableFeed = await party.getWriteFeed();
|
|
61
|
+
// Hint at the newly created writable feed so that we can start replicating from it.
|
|
62
|
+
party._setFeedHints([writableFeed.key]);
|
|
63
|
+
|
|
60
64
|
// Connect the pipeline.
|
|
61
65
|
await party.open();
|
|
62
66
|
|
|
63
|
-
const writableFeed = await party.getWriteFeed();
|
|
64
|
-
|
|
65
67
|
// PartyGenesis (self-signed by Party).
|
|
66
68
|
await party.credentialsWriter.write(createPartyGenesisMessage(
|
|
67
69
|
identity.keyring,
|
|
@@ -111,12 +113,9 @@ export class PartyFactory {
|
|
|
111
113
|
* @param partyKey
|
|
112
114
|
* @param hints
|
|
113
115
|
*/
|
|
114
|
-
async constructParty (partyKey: PartyKey,
|
|
116
|
+
async constructParty (partyKey: PartyKey, initialTimeframe?: Timeframe) {
|
|
115
117
|
const identity = this._identityProvider() ?? raise(new IdentityNotInitializedError());
|
|
116
118
|
|
|
117
|
-
// TODO(marik-d): Support read-only parties if this feed doesn't exist?
|
|
118
|
-
const feedProvider = this._feedProviderFactory(partyKey);
|
|
119
|
-
|
|
120
119
|
//
|
|
121
120
|
// Create the party.
|
|
122
121
|
//
|
|
@@ -124,11 +123,11 @@ export class PartyFactory {
|
|
|
124
123
|
partyKey,
|
|
125
124
|
this._modelFactory,
|
|
126
125
|
this._snapshotStore,
|
|
127
|
-
|
|
126
|
+
this._feedProviderFactory(partyKey),
|
|
127
|
+
this._metadataStore,
|
|
128
128
|
identity.createCredentialsSigner(),
|
|
129
129
|
identity.preferences,
|
|
130
130
|
this._networkManager,
|
|
131
|
-
hints,
|
|
132
131
|
initialTimeframe,
|
|
133
132
|
this._options
|
|
134
133
|
);
|
|
@@ -140,7 +139,7 @@ export class PartyFactory {
|
|
|
140
139
|
assert(snapshot.partyKey);
|
|
141
140
|
log(`Constructing ${humanize(snapshot.partyKey)} from snapshot at ${JSON.stringify(snapshot.timeframe)}.`);
|
|
142
141
|
|
|
143
|
-
const party = await this.constructParty(PublicKey.from(snapshot.partyKey),
|
|
142
|
+
const party = await this.constructParty(PublicKey.from(snapshot.partyKey), snapshot.timeframe);
|
|
144
143
|
await party.restoreFromSnapshot(snapshot);
|
|
145
144
|
return party;
|
|
146
145
|
}
|
|
@@ -173,7 +172,8 @@ export class PartyFactory {
|
|
|
173
172
|
|
|
174
173
|
await initiator.connect();
|
|
175
174
|
const { partyKey, hints } = await initiator.redeemInvitation(secretProvider);
|
|
176
|
-
const party = await this.constructParty(partyKey
|
|
175
|
+
const party = await this.constructParty(partyKey);
|
|
176
|
+
party._setFeedHints(hints);
|
|
177
177
|
await party.open();
|
|
178
178
|
await initiator.destroy();
|
|
179
179
|
|
|
@@ -198,11 +198,13 @@ export class PartyFactory {
|
|
|
198
198
|
const partyKey = await identity.keyring.createKeyRecord({ type: KeyType.PARTY });
|
|
199
199
|
const party = await this.constructParty(partyKey.publicKey);
|
|
200
200
|
|
|
201
|
+
const writableFeed = await party.getWriteFeed();
|
|
202
|
+
// Hint at the newly created writable feed so that we can start replicating from it.
|
|
203
|
+
party._setFeedHints([writableFeed.key]);
|
|
204
|
+
|
|
201
205
|
// Connect the pipeline.
|
|
202
206
|
await party.open();
|
|
203
207
|
|
|
204
|
-
const writableFeed = await party.getWriteFeed();
|
|
205
|
-
|
|
206
208
|
// PartyGenesis (self-signed by Party).
|
|
207
209
|
await party.credentialsWriter.write(createPartyGenesisMessage(
|
|
208
210
|
identity.keyring,
|
|
@@ -71,6 +71,7 @@ const setup = async () => {
|
|
|
71
71
|
modelFactory,
|
|
72
72
|
snapshotStore,
|
|
73
73
|
feedProviderFactory,
|
|
74
|
+
metadataStore,
|
|
74
75
|
{
|
|
75
76
|
writeLogger: messageLogger('<<<'),
|
|
76
77
|
readLogger: messageLogger('>>>')
|
|
@@ -140,10 +141,7 @@ describe('Party manager', () => {
|
|
|
140
141
|
const feedStream = createWritableFeedStream(feed);
|
|
141
142
|
feedStream.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
|
|
142
143
|
|
|
143
|
-
await partyManager.addParty(partyKey.publicKey, [
|
|
144
|
-
type: KeyType.FEED,
|
|
145
|
-
publicKey: PublicKey.from(feed.key)
|
|
146
|
-
}]);
|
|
144
|
+
await partyManager.addParty(partyKey.publicKey, [PublicKey.from(feed.key)]);
|
|
147
145
|
|
|
148
146
|
await update;
|
|
149
147
|
});
|
|
@@ -164,7 +162,8 @@ describe('Party manager', () => {
|
|
|
164
162
|
networkManager,
|
|
165
163
|
modelFactory,
|
|
166
164
|
snapshotStore,
|
|
167
|
-
feedProviderFactory
|
|
165
|
+
feedProviderFactory,
|
|
166
|
+
metadataStore
|
|
168
167
|
);
|
|
169
168
|
const partyManager = new PartyManager(metadataStore, snapshotStore, () => identity, partyFactory);
|
|
170
169
|
|
|
@@ -9,7 +9,7 @@ import unionWith from 'lodash.unionwith';
|
|
|
9
9
|
import { Event, synchronized } from '@dxos/async';
|
|
10
10
|
import { KeyHint, KeyType, SecretProvider } from '@dxos/credentials';
|
|
11
11
|
import { PublicKey } from '@dxos/crypto';
|
|
12
|
-
import { timed } from '@dxos/debug';
|
|
12
|
+
import { failUndefined, timed } from '@dxos/debug';
|
|
13
13
|
import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
|
|
14
14
|
import { ComplexMap, boolGuard } from '@dxos/util';
|
|
15
15
|
|
|
@@ -94,9 +94,13 @@ export class PartyManager {
|
|
|
94
94
|
const partyKey = partyKeys[i];
|
|
95
95
|
if (!this._parties.has(partyKey)) {
|
|
96
96
|
const snapshot = await this._snapshotStore.load(partyKey);
|
|
97
|
+
|
|
98
|
+
const metadata = this._metadataStore.getParty(partyKey) ?? failUndefined();
|
|
99
|
+
|
|
97
100
|
const party = snapshot
|
|
98
101
|
? await this._partyFactory.constructPartyFromSnapshot(snapshot)
|
|
99
102
|
: await this._partyFactory.constructParty(partyKey);
|
|
103
|
+
party._setFeedHints(metadata.feedKeys ?? []);
|
|
100
104
|
|
|
101
105
|
const isActive = identity?.preferences?.isPartyActive(partyKey) ?? true;
|
|
102
106
|
if (isActive) {
|
|
@@ -157,11 +161,9 @@ export class PartyManager {
|
|
|
157
161
|
|
|
158
162
|
/**
|
|
159
163
|
* Construct a party object and start replicating with the remote peer that created that party.
|
|
160
|
-
* @param partyKey
|
|
161
|
-
* @param hints
|
|
162
164
|
*/
|
|
163
165
|
@synchronized
|
|
164
|
-
async addParty (partyKey: PartyKey,
|
|
166
|
+
async addParty (partyKey: PartyKey, feedHints: PublicKey[] = []) {
|
|
165
167
|
assert(this._open, 'PartyManager is not open.');
|
|
166
168
|
|
|
167
169
|
/*
|
|
@@ -174,8 +176,9 @@ export class PartyManager {
|
|
|
174
176
|
return this._parties.get(partyKey);
|
|
175
177
|
}
|
|
176
178
|
|
|
177
|
-
log(`Adding party partyKey=${partyKey.toHex()} hints=${
|
|
178
|
-
const party = await this._partyFactory.constructParty(partyKey
|
|
179
|
+
log(`Adding party partyKey=${partyKey.toHex()} hints=${feedHints.length}`);
|
|
180
|
+
const party = await this._partyFactory.constructParty(partyKey);
|
|
181
|
+
party._setFeedHints(feedHints);
|
|
179
182
|
await party.open();
|
|
180
183
|
await this._metadataStore.addParty(party.key);
|
|
181
184
|
this._setParty(party);
|
|
@@ -10,19 +10,19 @@ import { waitForCondition, latch } from '@dxos/async';
|
|
|
10
10
|
import { createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
|
|
11
11
|
import { createId, createKeyPair, PublicKey } from '@dxos/crypto';
|
|
12
12
|
import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream, Timeframe } from '@dxos/echo-protocol';
|
|
13
|
-
import { FeedStore, createWritableFeedStream
|
|
13
|
+
import { FeedStore, createWritableFeedStream } from '@dxos/feed-store';
|
|
14
14
|
import { createSetPropertyMutation } from '@dxos/model-factory';
|
|
15
15
|
import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
|
|
16
16
|
import { jsonReplacer } from '@dxos/util';
|
|
17
17
|
|
|
18
18
|
import { TimeframeClock } from '../packlets/database';
|
|
19
|
+
import { FeedMuxer } from './feed-muxer';
|
|
19
20
|
import { PartyProcessor } from './party-processor';
|
|
20
|
-
import { Pipeline } from './pipeline';
|
|
21
21
|
|
|
22
22
|
const log = debug('dxos:echo:pipeline:test');
|
|
23
23
|
|
|
24
24
|
// TODO(burdon): Test read-only.
|
|
25
|
-
describe('
|
|
25
|
+
describe('FeedMuxer', () => {
|
|
26
26
|
test('streams', async () => {
|
|
27
27
|
const storage = createStorage('', StorageType.RAM);
|
|
28
28
|
const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
|
|
@@ -56,9 +56,6 @@ describe('pipeline', () => {
|
|
|
56
56
|
seq: 0
|
|
57
57
|
}
|
|
58
58
|
});
|
|
59
|
-
const pipeline = new Pipeline(partyProcessor, feedReadStream, new TimeframeClock());
|
|
60
|
-
const [readStream] = await pipeline.open();
|
|
61
|
-
expect(readStream).toBeTruthy();
|
|
62
59
|
|
|
63
60
|
//
|
|
64
61
|
// Pipeline consumer.
|
|
@@ -66,10 +63,14 @@ describe('pipeline', () => {
|
|
|
66
63
|
//
|
|
67
64
|
const numMessages = 5;
|
|
68
65
|
const [counter, updateCounter] = latch(numMessages);
|
|
69
|
-
|
|
66
|
+
const echoProcessor = async (message: IEchoStream) => {
|
|
70
67
|
log('Processed:', JSON.stringify(message, jsonReplacer, 2));
|
|
71
68
|
updateCounter();
|
|
72
|
-
}
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const pipeline = new FeedMuxer(partyProcessor, feedReadStream, new TimeframeClock());
|
|
72
|
+
pipeline.setEchoProcessor(echoProcessor);
|
|
73
|
+
await pipeline.open();
|
|
73
74
|
|
|
74
75
|
//
|
|
75
76
|
// Write directly to feed store.
|
|
@@ -105,17 +106,21 @@ describe('pipeline', () => {
|
|
|
105
106
|
});
|
|
106
107
|
|
|
107
108
|
const partyProcessor = new PartyProcessor(partyKey.publicKey);
|
|
108
|
-
|
|
109
|
+
|
|
110
|
+
const echoMessages: IEchoStream[] = [];
|
|
111
|
+
const echoProcessor = async (msg: IEchoStream) => {
|
|
112
|
+
echoMessages.push(msg);
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const pipeline = new FeedMuxer(
|
|
109
116
|
partyProcessor,
|
|
110
117
|
feedReadStream,
|
|
111
118
|
new TimeframeClock(),
|
|
112
119
|
createFeedWriter(feed)
|
|
113
120
|
);
|
|
121
|
+
pipeline.setEchoProcessor(echoProcessor);
|
|
114
122
|
await pipeline.open();
|
|
115
123
|
|
|
116
|
-
const writable = new WritableArray();
|
|
117
|
-
pipeline.inboundEchoStream!.pipe(writable);
|
|
118
|
-
|
|
119
124
|
await pipeline.outboundHaloStream!.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
|
|
120
125
|
await waitForCondition(() => !partyProcessor.genesisRequired);
|
|
121
126
|
|
|
@@ -126,10 +131,10 @@ describe('pipeline', () => {
|
|
|
126
131
|
}
|
|
127
132
|
});
|
|
128
133
|
|
|
129
|
-
await waitForCondition(() =>
|
|
134
|
+
await waitForCondition(() => echoMessages.length === 1);
|
|
130
135
|
|
|
131
136
|
expect(partyProcessor.genesisRequired).toEqual(false);
|
|
132
|
-
expect((
|
|
137
|
+
expect((echoMessages[0] as any).data).toEqual({
|
|
133
138
|
itemId: '123',
|
|
134
139
|
genesis: {
|
|
135
140
|
itemType: 'foo'
|
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
|
|
5
5
|
import assert from 'assert';
|
|
6
6
|
import debug from 'debug';
|
|
7
|
-
import { Readable } from 'stream';
|
|
8
7
|
|
|
9
8
|
import { Event } from '@dxos/async';
|
|
10
9
|
import { Message as HaloMessage } from '@dxos/credentials';
|
|
@@ -13,10 +12,9 @@ import { checkType } from '@dxos/debug';
|
|
|
13
12
|
import {
|
|
14
13
|
createFeedMeta, EchoEnvelope, FeedMessage, FeedStoreIterator, FeedWriter, IEchoStream, mapFeedWriter, Timeframe
|
|
15
14
|
} from '@dxos/echo-protocol';
|
|
16
|
-
import { createReadable } from '@dxos/feed-store';
|
|
17
15
|
import { jsonReplacer } from '@dxos/util';
|
|
18
16
|
|
|
19
|
-
import { TimeframeClock } from '../packlets/database';
|
|
17
|
+
import { EchoProcessor, TimeframeClock } from '../packlets/database';
|
|
20
18
|
import { CredentialProcessor, PartyStateProvider } from './party-processor';
|
|
21
19
|
|
|
22
20
|
interface Options {
|
|
@@ -28,14 +26,12 @@ const log = debug('dxos:echo-db:pipeline');
|
|
|
28
26
|
|
|
29
27
|
/**
|
|
30
28
|
* Manages the inbound and outbound message streams for an individual party.
|
|
29
|
+
* Reads messages from individual feeds and splits them into ECHO and HALO streams.
|
|
31
30
|
*/
|
|
32
|
-
export class
|
|
31
|
+
export class FeedMuxer {
|
|
33
32
|
private readonly _errors = new Event<Error>();
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
* Messages to be consumed from the pipeline (e.g., mutations to model).
|
|
37
|
-
*/
|
|
38
|
-
private _inboundEchoStream: Readable | undefined;
|
|
34
|
+
private _isOpen = false;
|
|
39
35
|
|
|
40
36
|
/**
|
|
41
37
|
* Messages to write into pipeline (e.g., mutations from model).
|
|
@@ -47,6 +43,8 @@ export class Pipeline {
|
|
|
47
43
|
*/
|
|
48
44
|
private _outboundHaloStream: FeedWriter<HaloMessage> | undefined;
|
|
49
45
|
|
|
46
|
+
private _echoProcessor: EchoProcessor | undefined
|
|
47
|
+
|
|
50
48
|
/**
|
|
51
49
|
* @param _partyProcessor Processes HALO messages to update party state.
|
|
52
50
|
* @param _feedStorIterator Inbound messages from the feed store.
|
|
@@ -60,20 +58,32 @@ export class Pipeline {
|
|
|
60
58
|
private readonly _timeframeClock: TimeframeClock,
|
|
61
59
|
private readonly _feedWriter?: FeedWriter<FeedMessage>,
|
|
62
60
|
private readonly _options: Options = {}
|
|
63
|
-
) {
|
|
61
|
+
) {
|
|
62
|
+
if (this._feedWriter) {
|
|
63
|
+
const loggingWriter = mapFeedWriter<FeedMessage, FeedMessage>(async msg => {
|
|
64
|
+
this._options.writeLogger?.(msg);
|
|
65
|
+
return msg;
|
|
66
|
+
}, this._feedWriter);
|
|
67
|
+
|
|
68
|
+
this._outboundEchoStream = mapFeedWriter<EchoEnvelope, FeedMessage>(async message => ({
|
|
69
|
+
timeframe: this._timeframeClock.timeframe,
|
|
70
|
+
echo: message
|
|
71
|
+
}), loggingWriter);
|
|
72
|
+
this._outboundHaloStream = mapFeedWriter<HaloMessage, FeedMessage>(async message => ({
|
|
73
|
+
timeframe: this._timeframeClock.timeframe,
|
|
74
|
+
halo: message
|
|
75
|
+
}), loggingWriter);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
64
78
|
|
|
65
79
|
get isOpen () {
|
|
66
|
-
return this.
|
|
80
|
+
return this._isOpen;
|
|
67
81
|
}
|
|
68
82
|
|
|
69
83
|
get readOnly () {
|
|
70
84
|
return this._outboundEchoStream === undefined;
|
|
71
85
|
}
|
|
72
86
|
|
|
73
|
-
get inboundEchoStream () {
|
|
74
|
-
return this._inboundEchoStream;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
87
|
get outboundEchoStream () {
|
|
78
88
|
return this._outboundEchoStream;
|
|
79
89
|
}
|
|
@@ -86,6 +96,10 @@ export class Pipeline {
|
|
|
86
96
|
return this._errors;
|
|
87
97
|
}
|
|
88
98
|
|
|
99
|
+
setEchoProcessor (processor: EchoProcessor) {
|
|
100
|
+
this._echoProcessor = processor;
|
|
101
|
+
}
|
|
102
|
+
|
|
89
103
|
/**
|
|
90
104
|
* Create inbound and outbound pipielines.
|
|
91
105
|
* https://nodejs.org/api/stream.html#stream_stream_pipeline_source_transforms_destination_callback
|
|
@@ -96,10 +110,8 @@ export class Pipeline {
|
|
|
96
110
|
* Transform(dxos.echo.IEchoEnvelope => dxos.IFeedMessage): update clock
|
|
97
111
|
* Feed
|
|
98
112
|
*/
|
|
99
|
-
async open (): Promise<
|
|
100
|
-
const { readLogger
|
|
101
|
-
|
|
102
|
-
this._inboundEchoStream = createReadable();
|
|
113
|
+
async open (): Promise<FeedWriter<EchoEnvelope> | undefined> {
|
|
114
|
+
const { readLogger } = this._options;
|
|
103
115
|
|
|
104
116
|
// This will exit cleanly once FeedStoreIterator is closed.
|
|
105
117
|
setImmediate(async () => {
|
|
@@ -109,8 +121,6 @@ export class Pipeline {
|
|
|
109
121
|
try {
|
|
110
122
|
const { data: message } = block;
|
|
111
123
|
|
|
112
|
-
this._timeframeClock.updateTimeframe(PublicKey.from(block.key), block.seq);
|
|
113
|
-
|
|
114
124
|
//
|
|
115
125
|
// HALO
|
|
116
126
|
//
|
|
@@ -133,8 +143,8 @@ export class Pipeline {
|
|
|
133
143
|
// Validate messge.
|
|
134
144
|
const { itemId } = message.echo;
|
|
135
145
|
if (itemId) {
|
|
136
|
-
assert(this.
|
|
137
|
-
this.
|
|
146
|
+
assert(this._echoProcessor);
|
|
147
|
+
await this._echoProcessor(checkType<IEchoStream>({
|
|
138
148
|
meta: {
|
|
139
149
|
seq: block.seq,
|
|
140
150
|
feedKey: block.key,
|
|
@@ -150,6 +160,8 @@ export class Pipeline {
|
|
|
150
160
|
// TODO(burdon): Can we throw and have the pipeline log (without breaking the stream)?
|
|
151
161
|
log(`Skipping invalid message: ${JSON.stringify(message, jsonReplacer)}`);
|
|
152
162
|
}
|
|
163
|
+
|
|
164
|
+
this._timeframeClock.updateTimeframe(PublicKey.from(block.key), block.seq);
|
|
153
165
|
} catch (err: any) {
|
|
154
166
|
console.error('Error in message processing.');
|
|
155
167
|
console.error(err);
|
|
@@ -157,30 +169,7 @@ export class Pipeline {
|
|
|
157
169
|
}
|
|
158
170
|
});
|
|
159
171
|
|
|
160
|
-
|
|
161
|
-
// Processes outbound messages (piped to the feed).
|
|
162
|
-
// Sets the current timeframe.
|
|
163
|
-
//
|
|
164
|
-
if (this._feedWriter) {
|
|
165
|
-
const loggingWriter = mapFeedWriter<FeedMessage, FeedMessage>(async msg => {
|
|
166
|
-
writeLogger?.(msg);
|
|
167
|
-
return msg;
|
|
168
|
-
}, this._feedWriter);
|
|
169
|
-
|
|
170
|
-
this._outboundEchoStream = mapFeedWriter<EchoEnvelope, FeedMessage>(async message => ({
|
|
171
|
-
timeframe: this._timeframeClock.timeframe,
|
|
172
|
-
echo: message
|
|
173
|
-
}), loggingWriter);
|
|
174
|
-
this._outboundHaloStream = mapFeedWriter<HaloMessage, FeedMessage>(async message => ({
|
|
175
|
-
timeframe: this._timeframeClock.timeframe,
|
|
176
|
-
halo: message
|
|
177
|
-
}), loggingWriter);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
return [
|
|
181
|
-
this._inboundEchoStream,
|
|
182
|
-
this._outboundEchoStream
|
|
183
|
-
];
|
|
172
|
+
return this._outboundEchoStream;
|
|
184
173
|
}
|
|
185
174
|
|
|
186
175
|
/**
|
|
@@ -190,11 +179,9 @@ export class Pipeline {
|
|
|
190
179
|
async close () {
|
|
191
180
|
await this._feedStorIterator.close();
|
|
192
181
|
|
|
193
|
-
if (this._inboundEchoStream) {
|
|
194
|
-
this._inboundEchoStream.destroy();
|
|
195
|
-
this._inboundEchoStream = undefined;
|
|
196
|
-
}
|
|
197
|
-
|
|
198
182
|
this._outboundEchoStream = undefined;
|
|
183
|
+
this._outboundHaloStream = undefined;
|
|
184
|
+
this._echoProcessor = undefined;
|
|
185
|
+
this._isOpen = false;
|
|
199
186
|
}
|
|
200
187
|
}
|
package/src/pipeline/index.ts
CHANGED
|
@@ -6,6 +6,6 @@ export * from './message-selector';
|
|
|
6
6
|
export * from './party-feed-provider';
|
|
7
7
|
export * from './party-processor';
|
|
8
8
|
export * from '../protocol/party-protocol-factory';
|
|
9
|
-
export * from './
|
|
10
|
-
export * from './party-
|
|
9
|
+
export * from './feed-muxer';
|
|
10
|
+
export * from './party-pipeline';
|
|
11
11
|
export * from './metadata-store';
|
|
@@ -7,7 +7,7 @@ import debug from 'debug';
|
|
|
7
7
|
|
|
8
8
|
import { PublicKey } from '@dxos/crypto';
|
|
9
9
|
import { failUndefined } from '@dxos/debug';
|
|
10
|
-
import { EchoMetadata, PartyMetadata, schema } from '@dxos/echo-protocol';
|
|
10
|
+
import { EchoMetadata, PartyMetadata, schema, Timeframe } from '@dxos/echo-protocol';
|
|
11
11
|
import { Directory } from '@dxos/random-access-multi-storage';
|
|
12
12
|
|
|
13
13
|
/**
|
|
@@ -90,7 +90,7 @@ export class MetadataStore {
|
|
|
90
90
|
*/
|
|
91
91
|
async clear (): Promise<void> {
|
|
92
92
|
log('Clearing all echo metadata...');
|
|
93
|
-
await this._directory.
|
|
93
|
+
await this._directory.delete();
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
/**
|
|
@@ -158,4 +158,10 @@ export class MetadataStore {
|
|
|
158
158
|
}
|
|
159
159
|
return !!party.feedKeys?.find(fk => feedKey.equals(fk));
|
|
160
160
|
}
|
|
161
|
+
|
|
162
|
+
async setTimeframe (partyKey: PublicKey, timeframe: Timeframe) {
|
|
163
|
+
const party = this.getParty(partyKey) ?? failUndefined();
|
|
164
|
+
party.latestTimeframe = timeframe;
|
|
165
|
+
await this._save();
|
|
166
|
+
}
|
|
161
167
|
}
|
|
@@ -8,7 +8,7 @@ import debug from 'debug';
|
|
|
8
8
|
import { Event, synchronized } from '@dxos/async';
|
|
9
9
|
import { Keyring, KeyType } from '@dxos/credentials';
|
|
10
10
|
import { PublicKey } from '@dxos/crypto';
|
|
11
|
-
import { FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
|
|
11
|
+
import { FeedSelector, FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
|
|
12
12
|
import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
|
|
13
13
|
import { ComplexMap } from '@dxos/util';
|
|
14
14
|
|
|
@@ -33,19 +33,6 @@ export class PartyFeedProvider {
|
|
|
33
33
|
return Array.from(this._feeds.values());
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
@synchronized
|
|
37
|
-
async openKnownFeeds () {
|
|
38
|
-
for (const feedKey of this._metadataStore.getParty(this._partyKey)?.feedKeys ?? []) {
|
|
39
|
-
if (!this._feeds.has(feedKey)) {
|
|
40
|
-
const fullKey = this._keyring.getFullKey(feedKey);
|
|
41
|
-
const feed = fullKey?.secretKey
|
|
42
|
-
? await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey)
|
|
43
|
-
: await this._feedStore.openReadOnlyFeed(feedKey);
|
|
44
|
-
this._trackFeed(feed);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
36
|
@synchronized
|
|
50
37
|
async createOrOpenWritableFeed () {
|
|
51
38
|
const partyMetadata = this._metadataStore.getParty(this._partyKey);
|
|
@@ -103,8 +90,8 @@ export class PartyFeedProvider {
|
|
|
103
90
|
return feed;
|
|
104
91
|
}
|
|
105
92
|
|
|
106
|
-
async createIterator (messageSelector: MessageSelector, initialTimeframe?: Timeframe) {
|
|
107
|
-
const iterator = new FeedStoreIterator(
|
|
93
|
+
async createIterator (messageSelector: MessageSelector, feedSelector: FeedSelector, initialTimeframe?: Timeframe) {
|
|
94
|
+
const iterator = new FeedStoreIterator(feedSelector, messageSelector, initialTimeframe ?? new Timeframe());
|
|
108
95
|
for (const feed of this._feeds.values()) {
|
|
109
96
|
iterator.addFeedDescriptor(feed);
|
|
110
97
|
}
|
|
@@ -17,12 +17,12 @@ import { ObjectModel } from '@dxos/object-model';
|
|
|
17
17
|
import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
|
|
18
18
|
import { afterTest } from '@dxos/testutils';
|
|
19
19
|
|
|
20
|
-
import { MetadataStore, PartyFeedProvider } from '
|
|
20
|
+
import { MetadataStore, PartyFeedProvider } from '.';
|
|
21
21
|
import { createReplicatorPlugin } from '../protocol/replicator-plugin';
|
|
22
22
|
import { SnapshotStore } from '../snapshots';
|
|
23
|
-
import {
|
|
23
|
+
import { PartyPipeline } from './party-pipeline';
|
|
24
24
|
|
|
25
|
-
describe('
|
|
25
|
+
describe('PartyPipeline', () => {
|
|
26
26
|
const setup = async () => {
|
|
27
27
|
const storage = createStorage('', StorageType.RAM);
|
|
28
28
|
const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
|
|
@@ -39,7 +39,7 @@ describe('PartyCore', () => {
|
|
|
39
39
|
|
|
40
40
|
const partyFeedProvider = new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey.publicKey);
|
|
41
41
|
|
|
42
|
-
const party = new
|
|
42
|
+
const party = new PartyPipeline(
|
|
43
43
|
partyKey.publicKey,
|
|
44
44
|
partyFeedProvider,
|
|
45
45
|
modelFactory,
|
|
@@ -48,7 +48,7 @@ describe('PartyCore', () => {
|
|
|
48
48
|
);
|
|
49
49
|
|
|
50
50
|
const feed = await partyFeedProvider.createOrOpenWritableFeed();
|
|
51
|
-
await party.open();
|
|
51
|
+
await party.open({ feedHints: [feed.key] });
|
|
52
52
|
afterTest(async () => party.close());
|
|
53
53
|
|
|
54
54
|
// PartyGenesis (self-signed by Party).
|
|
@@ -88,7 +88,7 @@ describe('PartyCore', () => {
|
|
|
88
88
|
});
|
|
89
89
|
|
|
90
90
|
test('create item with parent and then reload', async () => {
|
|
91
|
-
const { party } = await setup();
|
|
91
|
+
const { party, feedKey } = await setup();
|
|
92
92
|
|
|
93
93
|
{
|
|
94
94
|
const parent = await party.database.createItem({ model: ObjectModel, type: 'parent' });
|
|
@@ -103,7 +103,7 @@ describe('PartyCore', () => {
|
|
|
103
103
|
}
|
|
104
104
|
|
|
105
105
|
await party.close();
|
|
106
|
-
await party.open();
|
|
106
|
+
await party.open({ feedHints: [feedKey] });
|
|
107
107
|
|
|
108
108
|
{
|
|
109
109
|
await party.database.select().exec().update.waitFor(result => result.entities.length === 2);
|
|
@@ -149,7 +149,7 @@ describe('PartyCore', () => {
|
|
|
149
149
|
|
|
150
150
|
const otherFeedKey = PublicKey.random();
|
|
151
151
|
|
|
152
|
-
const party = new
|
|
152
|
+
const party = new PartyPipeline(
|
|
153
153
|
partyKey.publicKey,
|
|
154
154
|
partyFeedProvider,
|
|
155
155
|
modelFactory,
|
|
@@ -161,7 +161,7 @@ describe('PartyCore', () => {
|
|
|
161
161
|
|
|
162
162
|
const feedOpened = feedStore.feedOpenedEvent.waitForCount(1);
|
|
163
163
|
|
|
164
|
-
await party.open(
|
|
164
|
+
await party.open({ feedHints: [otherFeedKey] });
|
|
165
165
|
afterTest(async () => party.close());
|
|
166
166
|
|
|
167
167
|
await feedOpened;
|
|
@@ -250,6 +250,28 @@ describe('PartyCore', () => {
|
|
|
250
250
|
await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
|
|
251
251
|
});
|
|
252
252
|
|
|
253
|
+
test('wait to reach specific timeframe', async () => {
|
|
254
|
+
const { party, feedKey } = await setup();
|
|
255
|
+
|
|
256
|
+
{
|
|
257
|
+
const parent = await party.database.createItem({ model: ObjectModel, type: 'parent' });
|
|
258
|
+
const child = await party.database.createItem({
|
|
259
|
+
model: ObjectModel,
|
|
260
|
+
parent: parent.id,
|
|
261
|
+
type: 'child'
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
expect(child.parent).toEqual(parent);
|
|
265
|
+
expect(parent.children).toContain(child);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const timeframe = party.timeframe;
|
|
269
|
+
expect(timeframe.isEmpty()).toBeFalsy();
|
|
270
|
+
|
|
271
|
+
await party.close();
|
|
272
|
+
await party.open({ feedHints: [feedKey], targetTimeframe: timeframe });
|
|
273
|
+
});
|
|
274
|
+
|
|
253
275
|
test('two instances replicating', async () => {
|
|
254
276
|
const peer1 = await setup();
|
|
255
277
|
|
|
@@ -264,7 +286,7 @@ describe('PartyCore', () => {
|
|
|
264
286
|
|
|
265
287
|
const partyFeedProvider = new PartyFeedProvider(metadataStore, peer1.keyring, feedStore, peer1.party.key);
|
|
266
288
|
|
|
267
|
-
const party2 = new
|
|
289
|
+
const party2 = new PartyPipeline(
|
|
268
290
|
peer1.party.key,
|
|
269
291
|
partyFeedProvider,
|
|
270
292
|
modelFactory,
|
|
@@ -281,10 +303,9 @@ describe('PartyCore', () => {
|
|
|
281
303
|
[peer1.partyKey]
|
|
282
304
|
));
|
|
283
305
|
|
|
284
|
-
await party2.open(
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
}]);
|
|
306
|
+
await party2.open({
|
|
307
|
+
feedHints: [peer1.feedKey]
|
|
308
|
+
});
|
|
288
309
|
afterTest(async () => party2.close());
|
|
289
310
|
|
|
290
311
|
createTestProtocolPair(
|