@dxos/echo-db 2.33.5-dev.9e11cb97 → 2.33.5-dev.b4e42956
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/echo.d.ts.map +1 -1
- package/dist/src/echo.js +1 -1
- package/dist/src/echo.js.map +1 -1
- package/dist/src/echo.test.js +12 -4
- package/dist/src/echo.test.js.map +1 -1
- package/dist/src/halo/halo-factory.d.ts +4 -4
- package/dist/src/halo/halo-factory.d.ts.map +1 -1
- package/dist/src/halo/halo-factory.js +3 -2
- package/dist/src/halo/halo-factory.js.map +1 -1
- package/dist/src/halo/halo-party.d.ts +4 -3
- package/dist/src/halo/halo-party.d.ts.map +1 -1
- package/dist/src/halo/halo-party.js +8 -4
- package/dist/src/halo/halo-party.js.map +1 -1
- package/dist/src/halo/halo.d.ts +2 -2
- package/dist/src/halo/halo.d.ts.map +1 -1
- package/dist/src/halo/party-opener.d.ts.map +1 -1
- package/dist/src/halo/party-opener.js +3 -1
- package/dist/src/halo/party-opener.js.map +1 -1
- package/dist/src/invitations/greeting-initiator.d.ts +2 -2
- package/dist/src/invitations/greeting-initiator.d.ts.map +1 -1
- package/dist/src/invitations/greeting-initiator.js +2 -1
- package/dist/src/invitations/greeting-initiator.js.map +1 -1
- package/dist/src/packlets/database/data-mirror.test.js +2 -2
- package/dist/src/packlets/database/data-mirror.test.js.map +1 -1
- package/dist/src/packlets/database/database-backend.d.ts +4 -5
- package/dist/src/packlets/database/database-backend.d.ts.map +1 -1
- package/dist/src/packlets/database/database-backend.js +5 -6
- package/dist/src/packlets/database/database-backend.js.map +1 -1
- package/dist/src/packlets/database/item-demuxer.d.ts +2 -2
- package/dist/src/packlets/database/item-demuxer.d.ts.map +1 -1
- package/dist/src/packlets/database/item-demuxer.js +2 -3
- package/dist/src/packlets/database/item-demuxer.js.map +1 -1
- package/dist/src/packlets/database/item-demuxer.test.js +13 -9
- package/dist/src/packlets/database/item-demuxer.test.js.map +1 -1
- package/dist/src/packlets/database/testing.d.ts.map +1 -1
- package/dist/src/packlets/database/testing.js +3 -4
- package/dist/src/packlets/database/testing.js.map +1 -1
- package/dist/src/packlets/database/timeframe-clock.d.ts +1 -0
- package/dist/src/packlets/database/timeframe-clock.d.ts.map +1 -1
- package/dist/src/packlets/database/timeframe-clock.js +3 -0
- package/dist/src/packlets/database/timeframe-clock.js.map +1 -1
- package/dist/src/parties/data-party.d.ts +6 -5
- package/dist/src/parties/data-party.d.ts.map +1 -1
- package/dist/src/parties/data-party.js +26 -5
- package/dist/src/parties/data-party.js.map +1 -1
- package/dist/src/parties/data-party.test.js +24 -5
- package/dist/src/parties/data-party.test.js.map +1 -1
- package/dist/src/parties/party-factory.d.ts +5 -4
- package/dist/src/parties/party-factory.d.ts.map +1 -1
- package/dist/src/parties/party-factory.js +15 -11
- package/dist/src/parties/party-factory.js.map +1 -1
- package/dist/src/parties/party-manager.d.ts +3 -4
- package/dist/src/parties/party-manager.d.ts.map +1 -1
- package/dist/src/parties/party-manager.js +8 -7
- package/dist/src/parties/party-manager.js.map +1 -1
- package/dist/src/parties/party-manager.test.js +9 -9
- package/dist/src/parties/party-manager.test.js.map +1 -1
- package/dist/src/pipeline/{pipeline.d.ts → feed-muxer.d.ts} +8 -11
- package/dist/src/pipeline/feed-muxer.d.ts.map +1 -0
- package/dist/src/pipeline/{pipeline.js → feed-muxer.js} +34 -41
- package/dist/src/pipeline/feed-muxer.js.map +1 -0
- package/dist/src/pipeline/feed-muxer.test.d.ts +2 -0
- package/dist/src/pipeline/feed-muxer.test.d.ts.map +1 -0
- package/dist/src/pipeline/{pipeline.test.js → feed-muxer.test.js} +17 -15
- package/dist/src/pipeline/feed-muxer.test.js.map +1 -0
- package/dist/src/pipeline/index.d.ts +2 -2
- package/dist/src/pipeline/index.d.ts.map +1 -1
- package/dist/src/pipeline/index.js +2 -2
- package/dist/src/pipeline/index.js.map +1 -1
- package/dist/src/pipeline/message-selector.d.ts.map +1 -1
- package/dist/src/pipeline/message-selector.js +6 -5
- package/dist/src/pipeline/message-selector.js.map +1 -1
- package/dist/src/pipeline/metadata-store.d.ts +2 -1
- package/dist/src/pipeline/metadata-store.d.ts.map +1 -1
- package/dist/src/pipeline/metadata-store.js +7 -1
- package/dist/src/pipeline/metadata-store.js.map +1 -1
- package/dist/src/pipeline/party-feed-provider.d.ts +2 -3
- package/dist/src/pipeline/party-feed-provider.d.ts.map +1 -1
- package/dist/src/pipeline/party-feed-provider.js +2 -17
- package/dist/src/pipeline/party-feed-provider.js.map +1 -1
- package/dist/src/pipeline/{party-core.d.ts → party-pipeline.d.ts} +23 -9
- package/dist/src/pipeline/party-pipeline.d.ts.map +1 -0
- package/dist/src/pipeline/{party-core.js → party-pipeline.js} +26 -25
- package/dist/src/pipeline/party-pipeline.js.map +1 -0
- package/dist/src/pipeline/party-pipeline.test.d.ts +2 -0
- package/dist/src/pipeline/party-pipeline.test.d.ts.map +1 -0
- package/dist/src/pipeline/{party-core.test.js → party-pipeline.test.js} +50 -33
- package/dist/src/pipeline/party-pipeline.test.js.map +1 -0
- package/dist/src/snapshots/snapshot-generator.d.ts +2 -2
- package/dist/src/snapshots/snapshot-generator.d.ts.map +1 -1
- package/dist/src/snapshots/snapshot-generator.js.map +1 -1
- package/dist/src/snapshots/snapshot-store.js +1 -1
- package/dist/src/snapshots/snapshot-store.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +17 -17
- package/src/echo.test.ts +15 -4
- package/src/echo.ts +1 -0
- package/src/halo/halo-factory.ts +6 -6
- package/src/halo/halo-party.ts +10 -8
- package/src/halo/halo.ts +2 -2
- package/src/halo/party-opener.ts +3 -1
- package/src/invitations/greeting-initiator.ts +5 -4
- package/src/packlets/database/data-mirror.test.ts +2 -2
- package/src/packlets/database/database-backend.ts +7 -6
- package/src/packlets/database/item-demuxer.test.ts +20 -18
- package/src/packlets/database/item-demuxer.ts +5 -4
- package/src/packlets/database/testing.ts +3 -6
- package/src/packlets/database/timeframe-clock.ts +4 -0
- package/src/parties/data-party.test.ts +39 -8
- package/src/parties/data-party.ts +31 -9
- package/src/parties/party-factory.ts +19 -17
- package/src/parties/party-manager.test.ts +15 -9
- package/src/parties/party-manager.ts +9 -6
- package/src/pipeline/{pipeline.test.ts → feed-muxer.test.ts} +20 -16
- package/src/pipeline/{pipeline.ts → feed-muxer.ts} +39 -51
- package/src/pipeline/index.ts +2 -2
- package/src/pipeline/message-selector.ts +6 -5
- package/src/pipeline/metadata-store.ts +8 -2
- package/src/pipeline/party-feed-provider.ts +3 -16
- package/src/pipeline/{party-core.test.ts → party-pipeline.test.ts} +49 -27
- package/src/pipeline/{party-core.ts → party-pipeline.ts} +53 -24
- package/src/snapshots/snapshot-generator.ts +2 -2
- package/src/snapshots/snapshot-store.ts +1 -1
- package/dist/src/pipeline/party-core.d.ts.map +0 -1
- package/dist/src/pipeline/party-core.js.map +0 -1
- package/dist/src/pipeline/party-core.test.d.ts +0 -2
- package/dist/src/pipeline/party-core.test.d.ts.map +0 -1
- package/dist/src/pipeline/party-core.test.js.map +0 -1
- package/dist/src/pipeline/pipeline.d.ts.map +0 -1
- package/dist/src/pipeline/pipeline.js.map +0 -1
- package/dist/src/pipeline/pipeline.test.d.ts +0 -2
- package/dist/src/pipeline/pipeline.test.d.ts.map +0 -1
- package/dist/src/pipeline/pipeline.test.js.map +0 -1
|
@@ -9,7 +9,6 @@ import {
|
|
|
9
9
|
createEnvelopeMessage,
|
|
10
10
|
createFeedAdmitMessage,
|
|
11
11
|
createPartyGenesisMessage,
|
|
12
|
-
KeyHint,
|
|
13
12
|
KeyType,
|
|
14
13
|
SecretProvider,
|
|
15
14
|
wrapMessage
|
|
@@ -26,7 +25,7 @@ import {
|
|
|
26
25
|
GreetingInitiator, InvitationDescriptor, InvitationDescriptorType, OfflineInvitationClaimer
|
|
27
26
|
} from '../invitations';
|
|
28
27
|
import { IdentityNotInitializedError } from '../packlets/errors';
|
|
29
|
-
import { PartyFeedProvider,
|
|
28
|
+
import { MetadataStore, PartyFeedProvider, PipelineOptions } from '../pipeline';
|
|
30
29
|
import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
|
|
31
30
|
import { SnapshotStore } from '../snapshots';
|
|
32
31
|
import { DataParty, PARTY_ITEM_TYPE } from './data-party';
|
|
@@ -43,7 +42,8 @@ export class PartyFactory {
|
|
|
43
42
|
private readonly _modelFactory: ModelFactory,
|
|
44
43
|
private readonly _snapshotStore: SnapshotStore,
|
|
45
44
|
private readonly _feedProviderFactory: (partyKey: PublicKey) => PartyFeedProvider,
|
|
46
|
-
private readonly
|
|
45
|
+
private readonly _metadataStore: MetadataStore,
|
|
46
|
+
private readonly _options: PipelineOptions = {}
|
|
47
47
|
) {}
|
|
48
48
|
|
|
49
49
|
/**
|
|
@@ -57,11 +57,13 @@ export class PartyFactory {
|
|
|
57
57
|
const partyKey = await identity.keyring.createKeyRecord({ type: KeyType.PARTY });
|
|
58
58
|
const party = await this.constructParty(partyKey.publicKey);
|
|
59
59
|
|
|
60
|
+
const writableFeed = await party.getWriteFeed();
|
|
61
|
+
// Hint at the newly created writable feed so that we can start replicating from it.
|
|
62
|
+
party._setFeedHints([writableFeed.key]);
|
|
63
|
+
|
|
60
64
|
// Connect the pipeline.
|
|
61
65
|
await party.open();
|
|
62
66
|
|
|
63
|
-
const writableFeed = await party.getWriteFeed();
|
|
64
|
-
|
|
65
67
|
// PartyGenesis (self-signed by Party).
|
|
66
68
|
await party.credentialsWriter.write(createPartyGenesisMessage(
|
|
67
69
|
identity.keyring,
|
|
@@ -111,12 +113,9 @@ export class PartyFactory {
|
|
|
111
113
|
* @param partyKey
|
|
112
114
|
* @param hints
|
|
113
115
|
*/
|
|
114
|
-
async constructParty (partyKey: PartyKey,
|
|
116
|
+
async constructParty (partyKey: PartyKey, initialTimeframe?: Timeframe) {
|
|
115
117
|
const identity = this._identityProvider() ?? raise(new IdentityNotInitializedError());
|
|
116
118
|
|
|
117
|
-
// TODO(marik-d): Support read-only parties if this feed doesn't exist?
|
|
118
|
-
const feedProvider = this._feedProviderFactory(partyKey);
|
|
119
|
-
|
|
120
119
|
//
|
|
121
120
|
// Create the party.
|
|
122
121
|
//
|
|
@@ -124,11 +123,11 @@ export class PartyFactory {
|
|
|
124
123
|
partyKey,
|
|
125
124
|
this._modelFactory,
|
|
126
125
|
this._snapshotStore,
|
|
127
|
-
|
|
126
|
+
this._feedProviderFactory(partyKey),
|
|
127
|
+
this._metadataStore,
|
|
128
128
|
identity.createCredentialsSigner(),
|
|
129
129
|
identity.preferences,
|
|
130
130
|
this._networkManager,
|
|
131
|
-
hints,
|
|
132
131
|
initialTimeframe,
|
|
133
132
|
this._options
|
|
134
133
|
);
|
|
@@ -140,7 +139,7 @@ export class PartyFactory {
|
|
|
140
139
|
assert(snapshot.partyKey);
|
|
141
140
|
log(`Constructing ${humanize(snapshot.partyKey)} from snapshot at ${JSON.stringify(snapshot.timeframe)}.`);
|
|
142
141
|
|
|
143
|
-
const party = await this.constructParty(PublicKey.from(snapshot.partyKey),
|
|
142
|
+
const party = await this.constructParty(PublicKey.from(snapshot.partyKey), snapshot.timeframe);
|
|
144
143
|
await party.restoreFromSnapshot(snapshot);
|
|
145
144
|
return party;
|
|
146
145
|
}
|
|
@@ -173,7 +172,8 @@ export class PartyFactory {
|
|
|
173
172
|
|
|
174
173
|
await initiator.connect();
|
|
175
174
|
const { partyKey, hints } = await initiator.redeemInvitation(secretProvider);
|
|
176
|
-
const party = await this.constructParty(partyKey
|
|
175
|
+
const party = await this.constructParty(partyKey);
|
|
176
|
+
party._setFeedHints(hints);
|
|
177
177
|
await party.open();
|
|
178
178
|
await initiator.destroy();
|
|
179
179
|
|
|
@@ -198,11 +198,13 @@ export class PartyFactory {
|
|
|
198
198
|
const partyKey = await identity.keyring.createKeyRecord({ type: KeyType.PARTY });
|
|
199
199
|
const party = await this.constructParty(partyKey.publicKey);
|
|
200
200
|
|
|
201
|
+
const writableFeed = await party.getWriteFeed();
|
|
202
|
+
// Hint at the newly created writable feed so that we can start replicating from it.
|
|
203
|
+
party._setFeedHints([writableFeed.key]);
|
|
204
|
+
|
|
201
205
|
// Connect the pipeline.
|
|
202
206
|
await party.open();
|
|
203
207
|
|
|
204
|
-
const writableFeed = await party.getWriteFeed();
|
|
205
|
-
|
|
206
208
|
// PartyGenesis (self-signed by Party).
|
|
207
209
|
await party.credentialsWriter.write(createPartyGenesisMessage(
|
|
208
210
|
identity.keyring,
|
|
@@ -254,6 +256,7 @@ export class PartyFactory {
|
|
|
254
256
|
const feedWriter = createFeedWriter(writableFeed.feed);
|
|
255
257
|
for (const item of snapshot.database?.items || []) {
|
|
256
258
|
const message: FeedMessage = {
|
|
259
|
+
timeframe: new Timeframe(),
|
|
257
260
|
echo: {
|
|
258
261
|
itemId: item.itemId ?? failUndefined(),
|
|
259
262
|
genesis: {
|
|
@@ -264,8 +267,7 @@ export class PartyFactory {
|
|
|
264
267
|
itemMutation: {
|
|
265
268
|
parentId: item.parentId
|
|
266
269
|
},
|
|
267
|
-
snapshot: item.model
|
|
268
|
-
timeframe: new Timeframe()
|
|
270
|
+
snapshot: item.model
|
|
269
271
|
}
|
|
270
272
|
};
|
|
271
273
|
await feedWriter.write(message);
|
|
@@ -71,6 +71,7 @@ const setup = async () => {
|
|
|
71
71
|
modelFactory,
|
|
72
72
|
snapshotStore,
|
|
73
73
|
feedProviderFactory,
|
|
74
|
+
metadataStore,
|
|
74
75
|
{
|
|
75
76
|
writeLogger: messageLogger('<<<'),
|
|
76
77
|
readLogger: messageLogger('>>>')
|
|
@@ -140,16 +141,12 @@ describe('Party manager', () => {
|
|
|
140
141
|
const feedStream = createWritableFeedStream(feed);
|
|
141
142
|
feedStream.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
|
|
142
143
|
|
|
143
|
-
await partyManager.addParty(partyKey.publicKey, [
|
|
144
|
-
type: KeyType.FEED,
|
|
145
|
-
publicKey: PublicKey.from(feed.key)
|
|
146
|
-
}]);
|
|
144
|
+
await partyManager.addParty(partyKey.publicKey, [PublicKey.from(feed.key)]);
|
|
147
145
|
|
|
148
146
|
await update;
|
|
149
147
|
});
|
|
150
148
|
|
|
151
149
|
test('Create from cold start', async () => {
|
|
152
|
-
|
|
153
150
|
const storage = createStorage('', StorageType.RAM);
|
|
154
151
|
const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
|
|
155
152
|
const keyring = new Keyring();
|
|
@@ -165,7 +162,8 @@ describe('Party manager', () => {
|
|
|
165
162
|
networkManager,
|
|
166
163
|
modelFactory,
|
|
167
164
|
snapshotStore,
|
|
168
|
-
feedProviderFactory
|
|
165
|
+
feedProviderFactory,
|
|
166
|
+
metadataStore
|
|
169
167
|
);
|
|
170
168
|
const partyManager = new PartyManager(metadataStore, snapshotStore, () => identity, partyFactory);
|
|
171
169
|
|
|
@@ -191,15 +189,23 @@ describe('Party manager', () => {
|
|
|
191
189
|
assert(feedKey);
|
|
192
190
|
|
|
193
191
|
const feedStream = createWritableFeedStream(feed);
|
|
194
|
-
feedStream.write({ halo: createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identity.identityKey) });
|
|
195
192
|
feedStream.write({
|
|
193
|
+
timeframe: new Timeframe(),
|
|
194
|
+
halo: createPartyGenesisMessage(
|
|
195
|
+
keyring,
|
|
196
|
+
partyKey,
|
|
197
|
+
feedKey.publicKey,
|
|
198
|
+
identity.identityKey
|
|
199
|
+
)
|
|
200
|
+
});
|
|
201
|
+
feedStream.write({
|
|
202
|
+
timeframe: new Timeframe(),
|
|
196
203
|
echo: checkType<EchoEnvelope>({
|
|
197
204
|
itemId: 'foo',
|
|
198
205
|
genesis: {
|
|
199
206
|
itemType: PARTY_ITEM_TYPE,
|
|
200
207
|
modelType: ObjectModel.meta.type
|
|
201
|
-
}
|
|
202
|
-
timeframe: new Timeframe()
|
|
208
|
+
}
|
|
203
209
|
})
|
|
204
210
|
});
|
|
205
211
|
}
|
|
@@ -9,7 +9,7 @@ import unionWith from 'lodash.unionwith';
|
|
|
9
9
|
import { Event, synchronized } from '@dxos/async';
|
|
10
10
|
import { KeyHint, KeyType, SecretProvider } from '@dxos/credentials';
|
|
11
11
|
import { PublicKey } from '@dxos/crypto';
|
|
12
|
-
import { timed } from '@dxos/debug';
|
|
12
|
+
import { failUndefined, timed } from '@dxos/debug';
|
|
13
13
|
import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
|
|
14
14
|
import { ComplexMap, boolGuard } from '@dxos/util';
|
|
15
15
|
|
|
@@ -94,9 +94,13 @@ export class PartyManager {
|
|
|
94
94
|
const partyKey = partyKeys[i];
|
|
95
95
|
if (!this._parties.has(partyKey)) {
|
|
96
96
|
const snapshot = await this._snapshotStore.load(partyKey);
|
|
97
|
+
|
|
98
|
+
const metadata = this._metadataStore.getParty(partyKey) ?? failUndefined();
|
|
99
|
+
|
|
97
100
|
const party = snapshot
|
|
98
101
|
? await this._partyFactory.constructPartyFromSnapshot(snapshot)
|
|
99
102
|
: await this._partyFactory.constructParty(partyKey);
|
|
103
|
+
party._setFeedHints(metadata.feedKeys ?? []);
|
|
100
104
|
|
|
101
105
|
const isActive = identity?.preferences?.isPartyActive(partyKey) ?? true;
|
|
102
106
|
if (isActive) {
|
|
@@ -157,11 +161,9 @@ export class PartyManager {
|
|
|
157
161
|
|
|
158
162
|
/**
|
|
159
163
|
* Construct a party object and start replicating with the remote peer that created that party.
|
|
160
|
-
* @param partyKey
|
|
161
|
-
* @param hints
|
|
162
164
|
*/
|
|
163
165
|
@synchronized
|
|
164
|
-
async addParty (partyKey: PartyKey,
|
|
166
|
+
async addParty (partyKey: PartyKey, feedHints: PublicKey[] = []) {
|
|
165
167
|
assert(this._open, 'PartyManager is not open.');
|
|
166
168
|
|
|
167
169
|
/*
|
|
@@ -174,8 +176,9 @@ export class PartyManager {
|
|
|
174
176
|
return this._parties.get(partyKey);
|
|
175
177
|
}
|
|
176
178
|
|
|
177
|
-
log(`Adding party partyKey=${partyKey.toHex()} hints=${
|
|
178
|
-
const party = await this._partyFactory.constructParty(partyKey
|
|
179
|
+
log(`Adding party partyKey=${partyKey.toHex()} hints=${feedHints.length}`);
|
|
180
|
+
const party = await this._partyFactory.constructParty(partyKey);
|
|
181
|
+
party._setFeedHints(feedHints);
|
|
179
182
|
await party.open();
|
|
180
183
|
await this._metadataStore.addParty(party.key);
|
|
181
184
|
this._setParty(party);
|
|
@@ -10,19 +10,19 @@ import { waitForCondition, latch } from '@dxos/async';
|
|
|
10
10
|
import { createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
|
|
11
11
|
import { createId, createKeyPair, PublicKey } from '@dxos/crypto';
|
|
12
12
|
import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream, Timeframe } from '@dxos/echo-protocol';
|
|
13
|
-
import { FeedStore, createWritableFeedStream
|
|
13
|
+
import { FeedStore, createWritableFeedStream } from '@dxos/feed-store';
|
|
14
14
|
import { createSetPropertyMutation } from '@dxos/model-factory';
|
|
15
15
|
import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
|
|
16
16
|
import { jsonReplacer } from '@dxos/util';
|
|
17
17
|
|
|
18
18
|
import { TimeframeClock } from '../packlets/database';
|
|
19
|
+
import { FeedMuxer } from './feed-muxer';
|
|
19
20
|
import { PartyProcessor } from './party-processor';
|
|
20
|
-
import { Pipeline } from './pipeline';
|
|
21
21
|
|
|
22
22
|
const log = debug('dxos:echo:pipeline:test');
|
|
23
23
|
|
|
24
24
|
// TODO(burdon): Test read-only.
|
|
25
|
-
describe('
|
|
25
|
+
describe('FeedMuxer', () => {
|
|
26
26
|
test('streams', async () => {
|
|
27
27
|
const storage = createStorage('', StorageType.RAM);
|
|
28
28
|
const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
|
|
@@ -56,9 +56,6 @@ describe('pipeline', () => {
|
|
|
56
56
|
seq: 0
|
|
57
57
|
}
|
|
58
58
|
});
|
|
59
|
-
const pipeline = new Pipeline(partyProcessor, feedReadStream, new TimeframeClock());
|
|
60
|
-
const [readStream] = await pipeline.open();
|
|
61
|
-
expect(readStream).toBeTruthy();
|
|
62
59
|
|
|
63
60
|
//
|
|
64
61
|
// Pipeline consumer.
|
|
@@ -66,10 +63,14 @@ describe('pipeline', () => {
|
|
|
66
63
|
//
|
|
67
64
|
const numMessages = 5;
|
|
68
65
|
const [counter, updateCounter] = latch(numMessages);
|
|
69
|
-
|
|
66
|
+
const echoProcessor = async (message: IEchoStream) => {
|
|
70
67
|
log('Processed:', JSON.stringify(message, jsonReplacer, 2));
|
|
71
68
|
updateCounter();
|
|
72
|
-
}
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const pipeline = new FeedMuxer(partyProcessor, feedReadStream, new TimeframeClock());
|
|
72
|
+
pipeline.setEchoProcessor(echoProcessor);
|
|
73
|
+
await pipeline.open();
|
|
73
74
|
|
|
74
75
|
//
|
|
75
76
|
// Write directly to feed store.
|
|
@@ -105,17 +106,21 @@ describe('pipeline', () => {
|
|
|
105
106
|
});
|
|
106
107
|
|
|
107
108
|
const partyProcessor = new PartyProcessor(partyKey.publicKey);
|
|
108
|
-
|
|
109
|
+
|
|
110
|
+
const echoMessages: IEchoStream[] = [];
|
|
111
|
+
const echoProcessor = async (msg: IEchoStream) => {
|
|
112
|
+
echoMessages.push(msg);
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const pipeline = new FeedMuxer(
|
|
109
116
|
partyProcessor,
|
|
110
117
|
feedReadStream,
|
|
111
118
|
new TimeframeClock(),
|
|
112
119
|
createFeedWriter(feed)
|
|
113
120
|
);
|
|
121
|
+
pipeline.setEchoProcessor(echoProcessor);
|
|
114
122
|
await pipeline.open();
|
|
115
123
|
|
|
116
|
-
const writable = new WritableArray();
|
|
117
|
-
pipeline.inboundEchoStream!.pipe(writable);
|
|
118
|
-
|
|
119
124
|
await pipeline.outboundHaloStream!.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
|
|
120
125
|
await waitForCondition(() => !partyProcessor.genesisRequired);
|
|
121
126
|
|
|
@@ -126,15 +131,14 @@ describe('pipeline', () => {
|
|
|
126
131
|
}
|
|
127
132
|
});
|
|
128
133
|
|
|
129
|
-
await waitForCondition(() =>
|
|
134
|
+
await waitForCondition(() => echoMessages.length === 1);
|
|
130
135
|
|
|
131
136
|
expect(partyProcessor.genesisRequired).toEqual(false);
|
|
132
|
-
expect((
|
|
137
|
+
expect((echoMessages[0] as any).data).toEqual({
|
|
133
138
|
itemId: '123',
|
|
134
139
|
genesis: {
|
|
135
140
|
itemType: 'foo'
|
|
136
|
-
}
|
|
137
|
-
timeframe: expect.any(Timeframe)
|
|
141
|
+
}
|
|
138
142
|
});
|
|
139
143
|
});
|
|
140
144
|
});
|
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
|
|
5
5
|
import assert from 'assert';
|
|
6
6
|
import debug from 'debug';
|
|
7
|
-
import { Readable } from 'stream';
|
|
8
7
|
|
|
9
8
|
import { Event } from '@dxos/async';
|
|
10
9
|
import { Message as HaloMessage } from '@dxos/credentials';
|
|
@@ -13,10 +12,9 @@ import { checkType } from '@dxos/debug';
|
|
|
13
12
|
import {
|
|
14
13
|
createFeedMeta, EchoEnvelope, FeedMessage, FeedStoreIterator, FeedWriter, IEchoStream, mapFeedWriter, Timeframe
|
|
15
14
|
} from '@dxos/echo-protocol';
|
|
16
|
-
import { createReadable } from '@dxos/feed-store';
|
|
17
15
|
import { jsonReplacer } from '@dxos/util';
|
|
18
16
|
|
|
19
|
-
import { TimeframeClock } from '../packlets/database';
|
|
17
|
+
import { EchoProcessor, TimeframeClock } from '../packlets/database';
|
|
20
18
|
import { CredentialProcessor, PartyStateProvider } from './party-processor';
|
|
21
19
|
|
|
22
20
|
interface Options {
|
|
@@ -28,14 +26,12 @@ const log = debug('dxos:echo-db:pipeline');
|
|
|
28
26
|
|
|
29
27
|
/**
|
|
30
28
|
* Manages the inbound and outbound message streams for an individual party.
|
|
29
|
+
* Reads messages from individual feeds and splits them into ECHO and HALO streams.
|
|
31
30
|
*/
|
|
32
|
-
export class
|
|
31
|
+
export class FeedMuxer {
|
|
33
32
|
private readonly _errors = new Event<Error>();
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
* Messages to be consumed from the pipeline (e.g., mutations to model).
|
|
37
|
-
*/
|
|
38
|
-
private _inboundEchoStream: Readable | undefined;
|
|
34
|
+
private _isOpen = false;
|
|
39
35
|
|
|
40
36
|
/**
|
|
41
37
|
* Messages to write into pipeline (e.g., mutations from model).
|
|
@@ -47,6 +43,8 @@ export class Pipeline {
|
|
|
47
43
|
*/
|
|
48
44
|
private _outboundHaloStream: FeedWriter<HaloMessage> | undefined;
|
|
49
45
|
|
|
46
|
+
private _echoProcessor: EchoProcessor | undefined
|
|
47
|
+
|
|
50
48
|
/**
|
|
51
49
|
* @param _partyProcessor Processes HALO messages to update party state.
|
|
52
50
|
* @param _feedStorIterator Inbound messages from the feed store.
|
|
@@ -60,20 +58,32 @@ export class Pipeline {
|
|
|
60
58
|
private readonly _timeframeClock: TimeframeClock,
|
|
61
59
|
private readonly _feedWriter?: FeedWriter<FeedMessage>,
|
|
62
60
|
private readonly _options: Options = {}
|
|
63
|
-
) {
|
|
61
|
+
) {
|
|
62
|
+
if (this._feedWriter) {
|
|
63
|
+
const loggingWriter = mapFeedWriter<FeedMessage, FeedMessage>(async msg => {
|
|
64
|
+
this._options.writeLogger?.(msg);
|
|
65
|
+
return msg;
|
|
66
|
+
}, this._feedWriter);
|
|
67
|
+
|
|
68
|
+
this._outboundEchoStream = mapFeedWriter<EchoEnvelope, FeedMessage>(async message => ({
|
|
69
|
+
timeframe: this._timeframeClock.timeframe,
|
|
70
|
+
echo: message
|
|
71
|
+
}), loggingWriter);
|
|
72
|
+
this._outboundHaloStream = mapFeedWriter<HaloMessage, FeedMessage>(async message => ({
|
|
73
|
+
timeframe: this._timeframeClock.timeframe,
|
|
74
|
+
halo: message
|
|
75
|
+
}), loggingWriter);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
64
78
|
|
|
65
79
|
get isOpen () {
|
|
66
|
-
return this.
|
|
80
|
+
return this._isOpen;
|
|
67
81
|
}
|
|
68
82
|
|
|
69
83
|
get readOnly () {
|
|
70
84
|
return this._outboundEchoStream === undefined;
|
|
71
85
|
}
|
|
72
86
|
|
|
73
|
-
get inboundEchoStream () {
|
|
74
|
-
return this._inboundEchoStream;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
87
|
get outboundEchoStream () {
|
|
78
88
|
return this._outboundEchoStream;
|
|
79
89
|
}
|
|
@@ -86,6 +96,10 @@ export class Pipeline {
|
|
|
86
96
|
return this._errors;
|
|
87
97
|
}
|
|
88
98
|
|
|
99
|
+
setEchoProcessor (processor: EchoProcessor) {
|
|
100
|
+
this._echoProcessor = processor;
|
|
101
|
+
}
|
|
102
|
+
|
|
89
103
|
/**
|
|
90
104
|
* Create inbound and outbound pipielines.
|
|
91
105
|
* https://nodejs.org/api/stream.html#stream_stream_pipeline_source_transforms_destination_callback
|
|
@@ -96,10 +110,8 @@ export class Pipeline {
|
|
|
96
110
|
* Transform(dxos.echo.IEchoEnvelope => dxos.IFeedMessage): update clock
|
|
97
111
|
* Feed
|
|
98
112
|
*/
|
|
99
|
-
async open (): Promise<
|
|
100
|
-
const { readLogger
|
|
101
|
-
|
|
102
|
-
this._inboundEchoStream = createReadable();
|
|
113
|
+
async open (): Promise<FeedWriter<EchoEnvelope> | undefined> {
|
|
114
|
+
const { readLogger } = this._options;
|
|
103
115
|
|
|
104
116
|
// This will exit cleanly once FeedStoreIterator is closed.
|
|
105
117
|
setImmediate(async () => {
|
|
@@ -125,20 +137,19 @@ export class Pipeline {
|
|
|
125
137
|
//
|
|
126
138
|
|
|
127
139
|
if (message.echo) {
|
|
128
|
-
this._timeframeClock.updateTimeframe(PublicKey.from(block.key), block.seq);
|
|
129
140
|
const memberKey = this._partyProcessor.getFeedOwningMember(PublicKey.from(block.key));
|
|
130
141
|
assert(memberKey, `Ownership of feed ${keyToString(block.key)} could not be determined.`);
|
|
131
142
|
|
|
132
143
|
// Validate messge.
|
|
133
144
|
const { itemId } = message.echo;
|
|
134
145
|
if (itemId) {
|
|
135
|
-
assert(this.
|
|
136
|
-
this.
|
|
146
|
+
assert(this._echoProcessor);
|
|
147
|
+
await this._echoProcessor(checkType<IEchoStream>({
|
|
137
148
|
meta: {
|
|
138
149
|
seq: block.seq,
|
|
139
150
|
feedKey: block.key,
|
|
140
151
|
memberKey,
|
|
141
|
-
timeframe: message.
|
|
152
|
+
timeframe: message.timeframe ?? new Timeframe()
|
|
142
153
|
},
|
|
143
154
|
data: message.echo
|
|
144
155
|
}));
|
|
@@ -149,6 +160,8 @@ export class Pipeline {
|
|
|
149
160
|
// TODO(burdon): Can we throw and have the pipeline log (without breaking the stream)?
|
|
150
161
|
log(`Skipping invalid message: ${JSON.stringify(message, jsonReplacer)}`);
|
|
151
162
|
}
|
|
163
|
+
|
|
164
|
+
this._timeframeClock.updateTimeframe(PublicKey.from(block.key), block.seq);
|
|
152
165
|
} catch (err: any) {
|
|
153
166
|
console.error('Error in message processing.');
|
|
154
167
|
console.error(err);
|
|
@@ -156,30 +169,7 @@ export class Pipeline {
|
|
|
156
169
|
}
|
|
157
170
|
});
|
|
158
171
|
|
|
159
|
-
|
|
160
|
-
// Processes outbound messages (piped to the feed).
|
|
161
|
-
// Sets the current timeframe.
|
|
162
|
-
//
|
|
163
|
-
if (this._feedWriter) {
|
|
164
|
-
const loggingWriter = mapFeedWriter<FeedMessage, FeedMessage>(async msg => {
|
|
165
|
-
writeLogger?.(msg);
|
|
166
|
-
return msg;
|
|
167
|
-
}, this._feedWriter);
|
|
168
|
-
|
|
169
|
-
this._outboundEchoStream = mapFeedWriter<EchoEnvelope, FeedMessage>(async message => ({
|
|
170
|
-
echo: {
|
|
171
|
-
...message,
|
|
172
|
-
timeframe: this._timeframeClock.timeframe
|
|
173
|
-
}
|
|
174
|
-
}), loggingWriter);
|
|
175
|
-
this._outboundHaloStream =
|
|
176
|
-
mapFeedWriter<HaloMessage, FeedMessage>(async message => ({ halo: message }), loggingWriter);
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
return [
|
|
180
|
-
this._inboundEchoStream,
|
|
181
|
-
this._outboundEchoStream
|
|
182
|
-
];
|
|
172
|
+
return this._outboundEchoStream;
|
|
183
173
|
}
|
|
184
174
|
|
|
185
175
|
/**
|
|
@@ -189,11 +179,9 @@ export class Pipeline {
|
|
|
189
179
|
async close () {
|
|
190
180
|
await this._feedStorIterator.close();
|
|
191
181
|
|
|
192
|
-
if (this._inboundEchoStream) {
|
|
193
|
-
this._inboundEchoStream.destroy();
|
|
194
|
-
this._inboundEchoStream = undefined;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
182
|
this._outboundEchoStream = undefined;
|
|
183
|
+
this._outboundHaloStream = undefined;
|
|
184
|
+
this._echoProcessor = undefined;
|
|
185
|
+
this._isOpen = false;
|
|
198
186
|
}
|
|
199
187
|
}
|
package/src/pipeline/index.ts
CHANGED
|
@@ -6,6 +6,6 @@ export * from './message-selector';
|
|
|
6
6
|
export * from './party-feed-provider';
|
|
7
7
|
export * from './party-processor';
|
|
8
8
|
export * from '../protocol/party-protocol-factory';
|
|
9
|
-
export * from './
|
|
10
|
-
export * from './party-
|
|
9
|
+
export * from './feed-muxer';
|
|
10
|
+
export * from './party-pipeline';
|
|
11
11
|
export * from './metadata-store';
|
|
@@ -26,27 +26,28 @@ const log = debug('dxos:echo-db:message-selector');
|
|
|
26
26
|
export const createMessageSelector = (partyProcessor: PartyStateProvider, timeframeClock: TimeframeClock): MessageSelector => candidates => {
|
|
27
27
|
// Check ECHO message candidates first since they are less expensive than HALO cancidates.
|
|
28
28
|
for (let i = 0; i < candidates.length; i++) {
|
|
29
|
-
const { data: { echo } } = candidates[i];
|
|
29
|
+
const { data: { timeframe, echo } } = candidates[i];
|
|
30
30
|
const feedKey = PublicKey.from(candidates[i].key);
|
|
31
31
|
if (!echo) {
|
|
32
32
|
continue;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
assert(
|
|
36
|
-
if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(
|
|
35
|
+
assert(timeframe);
|
|
36
|
+
if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(timeframe)) {
|
|
37
37
|
return i;
|
|
38
38
|
}
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
// Check HALO message candidates.
|
|
42
42
|
for (let i = 0; i < candidates.length; i++) {
|
|
43
|
-
const { data: { halo } } = candidates[i];
|
|
43
|
+
const { data: { timeframe, halo } } = candidates[i];
|
|
44
44
|
const feedKey = PublicKey.from(candidates[i].key);
|
|
45
45
|
if (!halo) {
|
|
46
46
|
continue;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
|
|
49
|
+
assert(timeframe);
|
|
50
|
+
if (partyProcessor.isFeedAdmitted(feedKey) && !timeframeClock.hasGaps(timeframe)) {
|
|
50
51
|
return i;
|
|
51
52
|
}
|
|
52
53
|
|
|
@@ -7,7 +7,7 @@ import debug from 'debug';
|
|
|
7
7
|
|
|
8
8
|
import { PublicKey } from '@dxos/crypto';
|
|
9
9
|
import { failUndefined } from '@dxos/debug';
|
|
10
|
-
import { EchoMetadata, PartyMetadata, schema } from '@dxos/echo-protocol';
|
|
10
|
+
import { EchoMetadata, PartyMetadata, schema, Timeframe } from '@dxos/echo-protocol';
|
|
11
11
|
import { Directory } from '@dxos/random-access-multi-storage';
|
|
12
12
|
|
|
13
13
|
/**
|
|
@@ -90,7 +90,7 @@ export class MetadataStore {
|
|
|
90
90
|
*/
|
|
91
91
|
async clear (): Promise<void> {
|
|
92
92
|
log('Clearing all echo metadata...');
|
|
93
|
-
await this._directory.
|
|
93
|
+
await this._directory.delete();
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
/**
|
|
@@ -158,4 +158,10 @@ export class MetadataStore {
|
|
|
158
158
|
}
|
|
159
159
|
return !!party.feedKeys?.find(fk => feedKey.equals(fk));
|
|
160
160
|
}
|
|
161
|
+
|
|
162
|
+
async setTimeframe (partyKey: PublicKey, timeframe: Timeframe) {
|
|
163
|
+
const party = this.getParty(partyKey) ?? failUndefined();
|
|
164
|
+
party.latestTimeframe = timeframe;
|
|
165
|
+
await this._save();
|
|
166
|
+
}
|
|
161
167
|
}
|
|
@@ -8,7 +8,7 @@ import debug from 'debug';
|
|
|
8
8
|
import { Event, synchronized } from '@dxos/async';
|
|
9
9
|
import { Keyring, KeyType } from '@dxos/credentials';
|
|
10
10
|
import { PublicKey } from '@dxos/crypto';
|
|
11
|
-
import { FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
|
|
11
|
+
import { FeedSelector, FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
|
|
12
12
|
import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
|
|
13
13
|
import { ComplexMap } from '@dxos/util';
|
|
14
14
|
|
|
@@ -33,19 +33,6 @@ export class PartyFeedProvider {
|
|
|
33
33
|
return Array.from(this._feeds.values());
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
@synchronized
|
|
37
|
-
async openKnownFeeds () {
|
|
38
|
-
for (const feedKey of this._metadataStore.getParty(this._partyKey)?.feedKeys ?? []) {
|
|
39
|
-
if (!this._feeds.has(feedKey)) {
|
|
40
|
-
const fullKey = this._keyring.getFullKey(feedKey);
|
|
41
|
-
const feed = fullKey?.secretKey
|
|
42
|
-
? await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey)
|
|
43
|
-
: await this._feedStore.openReadOnlyFeed(feedKey);
|
|
44
|
-
this._trackFeed(feed);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
36
|
@synchronized
|
|
50
37
|
async createOrOpenWritableFeed () {
|
|
51
38
|
const partyMetadata = this._metadataStore.getParty(this._partyKey);
|
|
@@ -103,8 +90,8 @@ export class PartyFeedProvider {
|
|
|
103
90
|
return feed;
|
|
104
91
|
}
|
|
105
92
|
|
|
106
|
-
async createIterator (messageSelector: MessageSelector, initialTimeframe?: Timeframe) {
|
|
107
|
-
const iterator = new FeedStoreIterator(
|
|
93
|
+
async createIterator (messageSelector: MessageSelector, feedSelector: FeedSelector, initialTimeframe?: Timeframe) {
|
|
94
|
+
const iterator = new FeedStoreIterator(feedSelector, messageSelector, initialTimeframe ?? new Timeframe());
|
|
108
95
|
for (const feed of this._feeds.values()) {
|
|
109
96
|
iterator.addFeedDescriptor(feed);
|
|
110
97
|
}
|