@dxos/echo-db 2.33.5-dev.ebc105f7 → 2.33.5-dev.fa6b779b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/dist/src/echo.d.ts.map +1 -1
  2. package/dist/src/echo.js +1 -1
  3. package/dist/src/echo.js.map +1 -1
  4. package/dist/src/echo.test.js +12 -4
  5. package/dist/src/echo.test.js.map +1 -1
  6. package/dist/src/halo/halo-factory.d.ts +4 -4
  7. package/dist/src/halo/halo-factory.d.ts.map +1 -1
  8. package/dist/src/halo/halo-factory.js +3 -2
  9. package/dist/src/halo/halo-factory.js.map +1 -1
  10. package/dist/src/halo/halo-party.d.ts +4 -3
  11. package/dist/src/halo/halo-party.d.ts.map +1 -1
  12. package/dist/src/halo/halo-party.js +8 -4
  13. package/dist/src/halo/halo-party.js.map +1 -1
  14. package/dist/src/halo/halo.d.ts +2 -2
  15. package/dist/src/halo/halo.d.ts.map +1 -1
  16. package/dist/src/halo/party-opener.d.ts.map +1 -1
  17. package/dist/src/halo/party-opener.js +3 -1
  18. package/dist/src/halo/party-opener.js.map +1 -1
  19. package/dist/src/invitations/greeting-initiator.d.ts +2 -2
  20. package/dist/src/invitations/greeting-initiator.d.ts.map +1 -1
  21. package/dist/src/invitations/greeting-initiator.js +2 -1
  22. package/dist/src/invitations/greeting-initiator.js.map +1 -1
  23. package/dist/src/packlets/database/data-mirror.test.js +2 -2
  24. package/dist/src/packlets/database/data-mirror.test.js.map +1 -1
  25. package/dist/src/packlets/database/database-backend.d.ts +4 -5
  26. package/dist/src/packlets/database/database-backend.d.ts.map +1 -1
  27. package/dist/src/packlets/database/database-backend.js +5 -6
  28. package/dist/src/packlets/database/database-backend.js.map +1 -1
  29. package/dist/src/packlets/database/item-demuxer.d.ts +2 -2
  30. package/dist/src/packlets/database/item-demuxer.d.ts.map +1 -1
  31. package/dist/src/packlets/database/item-demuxer.js +2 -3
  32. package/dist/src/packlets/database/item-demuxer.js.map +1 -1
  33. package/dist/src/packlets/database/item-demuxer.test.js +13 -9
  34. package/dist/src/packlets/database/item-demuxer.test.js.map +1 -1
  35. package/dist/src/packlets/database/testing.d.ts.map +1 -1
  36. package/dist/src/packlets/database/testing.js +3 -4
  37. package/dist/src/packlets/database/testing.js.map +1 -1
  38. package/dist/src/packlets/database/timeframe-clock.d.ts +1 -0
  39. package/dist/src/packlets/database/timeframe-clock.d.ts.map +1 -1
  40. package/dist/src/packlets/database/timeframe-clock.js +3 -0
  41. package/dist/src/packlets/database/timeframe-clock.js.map +1 -1
  42. package/dist/src/parties/data-party.d.ts +6 -5
  43. package/dist/src/parties/data-party.d.ts.map +1 -1
  44. package/dist/src/parties/data-party.js +26 -5
  45. package/dist/src/parties/data-party.js.map +1 -1
  46. package/dist/src/parties/data-party.test.js +24 -5
  47. package/dist/src/parties/data-party.test.js.map +1 -1
  48. package/dist/src/parties/party-factory.d.ts +5 -4
  49. package/dist/src/parties/party-factory.d.ts.map +1 -1
  50. package/dist/src/parties/party-factory.js +13 -9
  51. package/dist/src/parties/party-factory.js.map +1 -1
  52. package/dist/src/parties/party-manager.d.ts +3 -4
  53. package/dist/src/parties/party-manager.d.ts.map +1 -1
  54. package/dist/src/parties/party-manager.js +8 -7
  55. package/dist/src/parties/party-manager.js.map +1 -1
  56. package/dist/src/parties/party-manager.test.js +3 -6
  57. package/dist/src/parties/party-manager.test.js.map +1 -1
  58. package/dist/src/pipeline/{pipeline.d.ts → feed-muxer.d.ts} +8 -11
  59. package/dist/src/pipeline/feed-muxer.d.ts.map +1 -0
  60. package/dist/src/pipeline/{pipeline.js → feed-muxer.js} +33 -40
  61. package/dist/src/pipeline/feed-muxer.js.map +1 -0
  62. package/dist/src/pipeline/feed-muxer.test.d.ts +2 -0
  63. package/dist/src/pipeline/feed-muxer.test.d.ts.map +1 -0
  64. package/dist/src/pipeline/{pipeline.test.js → feed-muxer.test.js} +16 -13
  65. package/dist/src/pipeline/feed-muxer.test.js.map +1 -0
  66. package/dist/src/pipeline/index.d.ts +2 -2
  67. package/dist/src/pipeline/index.d.ts.map +1 -1
  68. package/dist/src/pipeline/index.js +2 -2
  69. package/dist/src/pipeline/index.js.map +1 -1
  70. package/dist/src/pipeline/metadata-store.d.ts +2 -1
  71. package/dist/src/pipeline/metadata-store.d.ts.map +1 -1
  72. package/dist/src/pipeline/metadata-store.js +7 -1
  73. package/dist/src/pipeline/metadata-store.js.map +1 -1
  74. package/dist/src/pipeline/party-feed-provider.d.ts +2 -3
  75. package/dist/src/pipeline/party-feed-provider.d.ts.map +1 -1
  76. package/dist/src/pipeline/party-feed-provider.js +2 -17
  77. package/dist/src/pipeline/party-feed-provider.js.map +1 -1
  78. package/dist/src/pipeline/{party-core.d.ts → party-pipeline.d.ts} +23 -9
  79. package/dist/src/pipeline/party-pipeline.d.ts.map +1 -0
  80. package/dist/src/pipeline/{party-core.js → party-pipeline.js} +26 -25
  81. package/dist/src/pipeline/party-pipeline.js.map +1 -0
  82. package/dist/src/pipeline/party-pipeline.test.d.ts +2 -0
  83. package/dist/src/pipeline/party-pipeline.test.d.ts.map +1 -0
  84. package/dist/src/pipeline/{party-core.test.js → party-pipeline.test.js} +37 -21
  85. package/dist/src/pipeline/party-pipeline.test.js.map +1 -0
  86. package/dist/src/snapshots/snapshot-generator.d.ts +2 -2
  87. package/dist/src/snapshots/snapshot-generator.d.ts.map +1 -1
  88. package/dist/src/snapshots/snapshot-generator.js.map +1 -1
  89. package/dist/src/snapshots/snapshot-store.js +1 -1
  90. package/dist/src/snapshots/snapshot-store.js.map +1 -1
  91. package/dist/tsconfig.tsbuildinfo +1 -1
  92. package/package.json +17 -17
  93. package/src/echo.test.ts +15 -4
  94. package/src/echo.ts +1 -0
  95. package/src/halo/halo-factory.ts +6 -6
  96. package/src/halo/halo-party.ts +10 -8
  97. package/src/halo/halo.ts +2 -2
  98. package/src/halo/party-opener.ts +3 -1
  99. package/src/invitations/greeting-initiator.ts +5 -4
  100. package/src/packlets/database/data-mirror.test.ts +2 -2
  101. package/src/packlets/database/database-backend.ts +7 -6
  102. package/src/packlets/database/item-demuxer.test.ts +20 -18
  103. package/src/packlets/database/item-demuxer.ts +5 -4
  104. package/src/packlets/database/testing.ts +3 -6
  105. package/src/packlets/database/timeframe-clock.ts +4 -0
  106. package/src/parties/data-party.test.ts +39 -8
  107. package/src/parties/data-party.ts +31 -9
  108. package/src/parties/party-factory.ts +17 -15
  109. package/src/parties/party-manager.test.ts +4 -5
  110. package/src/parties/party-manager.ts +9 -6
  111. package/src/pipeline/{pipeline.test.ts → feed-muxer.test.ts} +19 -14
  112. package/src/pipeline/{pipeline.ts → feed-muxer.ts} +38 -51
  113. package/src/pipeline/index.ts +2 -2
  114. package/src/pipeline/metadata-store.ts +8 -2
  115. package/src/pipeline/party-feed-provider.ts +3 -16
  116. package/src/pipeline/{party-core.test.ts → party-pipeline.test.ts} +35 -14
  117. package/src/pipeline/{party-core.ts → party-pipeline.ts} +53 -24
  118. package/src/snapshots/snapshot-generator.ts +2 -2
  119. package/src/snapshots/snapshot-store.ts +1 -1
  120. package/dist/src/pipeline/party-core.d.ts.map +0 -1
  121. package/dist/src/pipeline/party-core.js.map +0 -1
  122. package/dist/src/pipeline/party-core.test.d.ts +0 -2
  123. package/dist/src/pipeline/party-core.test.d.ts.map +0 -1
  124. package/dist/src/pipeline/party-core.test.js.map +0 -1
  125. package/dist/src/pipeline/pipeline.d.ts.map +0 -1
  126. package/dist/src/pipeline/pipeline.js.map +0 -1
  127. package/dist/src/pipeline/pipeline.test.d.ts +0 -2
  128. package/dist/src/pipeline/pipeline.test.d.ts.map +0 -1
  129. package/dist/src/pipeline/pipeline.test.js.map +0 -1
@@ -9,7 +9,6 @@ import {
9
9
  createEnvelopeMessage,
10
10
  createFeedAdmitMessage,
11
11
  createPartyGenesisMessage,
12
- KeyHint,
13
12
  KeyType,
14
13
  SecretProvider,
15
14
  wrapMessage
@@ -26,7 +25,7 @@ import {
26
25
  GreetingInitiator, InvitationDescriptor, InvitationDescriptorType, OfflineInvitationClaimer
27
26
  } from '../invitations';
28
27
  import { IdentityNotInitializedError } from '../packlets/errors';
29
- import { PartyFeedProvider, PartyOptions } from '../pipeline';
28
+ import { MetadataStore, PartyFeedProvider, PipelineOptions } from '../pipeline';
30
29
  import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
31
30
  import { SnapshotStore } from '../snapshots';
32
31
  import { DataParty, PARTY_ITEM_TYPE } from './data-party';
@@ -43,7 +42,8 @@ export class PartyFactory {
43
42
  private readonly _modelFactory: ModelFactory,
44
43
  private readonly _snapshotStore: SnapshotStore,
45
44
  private readonly _feedProviderFactory: (partyKey: PublicKey) => PartyFeedProvider,
46
- private readonly _options: PartyOptions = {}
45
+ private readonly _metadataStore: MetadataStore,
46
+ private readonly _options: PipelineOptions = {}
47
47
  ) {}
48
48
 
49
49
  /**
@@ -57,11 +57,13 @@ export class PartyFactory {
57
57
  const partyKey = await identity.keyring.createKeyRecord({ type: KeyType.PARTY });
58
58
  const party = await this.constructParty(partyKey.publicKey);
59
59
 
60
+ const writableFeed = await party.getWriteFeed();
61
+ // Hint at the newly created writable feed so that we can start replicating from it.
62
+ party._setFeedHints([writableFeed.key]);
63
+
60
64
  // Connect the pipeline.
61
65
  await party.open();
62
66
 
63
- const writableFeed = await party.getWriteFeed();
64
-
65
67
  // PartyGenesis (self-signed by Party).
66
68
  await party.credentialsWriter.write(createPartyGenesisMessage(
67
69
  identity.keyring,
@@ -111,12 +113,9 @@ export class PartyFactory {
111
113
  * @param partyKey
112
114
  * @param hints
113
115
  */
114
- async constructParty (partyKey: PartyKey, hints: KeyHint[] = [], initialTimeframe?: Timeframe) {
116
+ async constructParty (partyKey: PartyKey, initialTimeframe?: Timeframe) {
115
117
  const identity = this._identityProvider() ?? raise(new IdentityNotInitializedError());
116
118
 
117
- // TODO(marik-d): Support read-only parties if this feed doesn't exist?
118
- const feedProvider = this._feedProviderFactory(partyKey);
119
-
120
119
  //
121
120
  // Create the party.
122
121
  //
@@ -124,11 +123,11 @@ export class PartyFactory {
124
123
  partyKey,
125
124
  this._modelFactory,
126
125
  this._snapshotStore,
127
- feedProvider,
126
+ this._feedProviderFactory(partyKey),
127
+ this._metadataStore,
128
128
  identity.createCredentialsSigner(),
129
129
  identity.preferences,
130
130
  this._networkManager,
131
- hints,
132
131
  initialTimeframe,
133
132
  this._options
134
133
  );
@@ -140,7 +139,7 @@ export class PartyFactory {
140
139
  assert(snapshot.partyKey);
141
140
  log(`Constructing ${humanize(snapshot.partyKey)} from snapshot at ${JSON.stringify(snapshot.timeframe)}.`);
142
141
 
143
- const party = await this.constructParty(PublicKey.from(snapshot.partyKey), [], snapshot.timeframe);
142
+ const party = await this.constructParty(PublicKey.from(snapshot.partyKey), snapshot.timeframe);
144
143
  await party.restoreFromSnapshot(snapshot);
145
144
  return party;
146
145
  }
@@ -173,7 +172,8 @@ export class PartyFactory {
173
172
 
174
173
  await initiator.connect();
175
174
  const { partyKey, hints } = await initiator.redeemInvitation(secretProvider);
176
- const party = await this.constructParty(partyKey, hints);
175
+ const party = await this.constructParty(partyKey);
176
+ party._setFeedHints(hints);
177
177
  await party.open();
178
178
  await initiator.destroy();
179
179
 
@@ -198,11 +198,13 @@ export class PartyFactory {
198
198
  const partyKey = await identity.keyring.createKeyRecord({ type: KeyType.PARTY });
199
199
  const party = await this.constructParty(partyKey.publicKey);
200
200
 
201
+ const writableFeed = await party.getWriteFeed();
202
+ // Hint at the newly created writable feed so that we can start replicating from it.
203
+ party._setFeedHints([writableFeed.key]);
204
+
201
205
  // Connect the pipeline.
202
206
  await party.open();
203
207
 
204
- const writableFeed = await party.getWriteFeed();
205
-
206
208
  // PartyGenesis (self-signed by Party).
207
209
  await party.credentialsWriter.write(createPartyGenesisMessage(
208
210
  identity.keyring,
@@ -71,6 +71,7 @@ const setup = async () => {
71
71
  modelFactory,
72
72
  snapshotStore,
73
73
  feedProviderFactory,
74
+ metadataStore,
74
75
  {
75
76
  writeLogger: messageLogger('<<<'),
76
77
  readLogger: messageLogger('>>>')
@@ -140,10 +141,7 @@ describe('Party manager', () => {
140
141
  const feedStream = createWritableFeedStream(feed);
141
142
  feedStream.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
142
143
 
143
- await partyManager.addParty(partyKey.publicKey, [{
144
- type: KeyType.FEED,
145
- publicKey: PublicKey.from(feed.key)
146
- }]);
144
+ await partyManager.addParty(partyKey.publicKey, [PublicKey.from(feed.key)]);
147
145
 
148
146
  await update;
149
147
  });
@@ -164,7 +162,8 @@ describe('Party manager', () => {
164
162
  networkManager,
165
163
  modelFactory,
166
164
  snapshotStore,
167
- feedProviderFactory
165
+ feedProviderFactory,
166
+ metadataStore
168
167
  );
169
168
  const partyManager = new PartyManager(metadataStore, snapshotStore, () => identity, partyFactory);
170
169
 
@@ -9,7 +9,7 @@ import unionWith from 'lodash.unionwith';
9
9
  import { Event, synchronized } from '@dxos/async';
10
10
  import { KeyHint, KeyType, SecretProvider } from '@dxos/credentials';
11
11
  import { PublicKey } from '@dxos/crypto';
12
- import { timed } from '@dxos/debug';
12
+ import { failUndefined, timed } from '@dxos/debug';
13
13
  import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
14
14
  import { ComplexMap, boolGuard } from '@dxos/util';
15
15
 
@@ -94,9 +94,13 @@ export class PartyManager {
94
94
  const partyKey = partyKeys[i];
95
95
  if (!this._parties.has(partyKey)) {
96
96
  const snapshot = await this._snapshotStore.load(partyKey);
97
+
98
+ const metadata = this._metadataStore.getParty(partyKey) ?? failUndefined();
99
+
97
100
  const party = snapshot
98
101
  ? await this._partyFactory.constructPartyFromSnapshot(snapshot)
99
102
  : await this._partyFactory.constructParty(partyKey);
103
+ party._setFeedHints(metadata.feedKeys ?? []);
100
104
 
101
105
  const isActive = identity?.preferences?.isPartyActive(partyKey) ?? true;
102
106
  if (isActive) {
@@ -157,11 +161,9 @@ export class PartyManager {
157
161
 
158
162
  /**
159
163
  * Construct a party object and start replicating with the remote peer that created that party.
160
- * @param partyKey
161
- * @param hints
162
164
  */
163
165
  @synchronized
164
- async addParty (partyKey: PartyKey, hints: KeyHint[] = []) {
166
+ async addParty (partyKey: PartyKey, feedHints: PublicKey[] = []) {
165
167
  assert(this._open, 'PartyManager is not open.');
166
168
 
167
169
  /*
@@ -174,8 +176,9 @@ export class PartyManager {
174
176
  return this._parties.get(partyKey);
175
177
  }
176
178
 
177
- log(`Adding party partyKey=${partyKey.toHex()} hints=${hints.length}`);
178
- const party = await this._partyFactory.constructParty(partyKey, hints);
179
+ log(`Adding party partyKey=${partyKey.toHex()} hints=${feedHints.length}`);
180
+ const party = await this._partyFactory.constructParty(partyKey);
181
+ party._setFeedHints(feedHints);
179
182
  await party.open();
180
183
  await this._metadataStore.addParty(party.key);
181
184
  this._setParty(party);
@@ -10,19 +10,19 @@ import { waitForCondition, latch } from '@dxos/async';
10
10
  import { createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
11
11
  import { createId, createKeyPair, PublicKey } from '@dxos/crypto';
12
12
  import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream, Timeframe } from '@dxos/echo-protocol';
13
- import { FeedStore, createWritableFeedStream, createWritable, WritableArray } from '@dxos/feed-store';
13
+ import { FeedStore, createWritableFeedStream } from '@dxos/feed-store';
14
14
  import { createSetPropertyMutation } from '@dxos/model-factory';
15
15
  import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
16
16
  import { jsonReplacer } from '@dxos/util';
17
17
 
18
18
  import { TimeframeClock } from '../packlets/database';
19
+ import { FeedMuxer } from './feed-muxer';
19
20
  import { PartyProcessor } from './party-processor';
20
- import { Pipeline } from './pipeline';
21
21
 
22
22
  const log = debug('dxos:echo:pipeline:test');
23
23
 
24
24
  // TODO(burdon): Test read-only.
25
- describe('pipeline', () => {
25
+ describe('FeedMuxer', () => {
26
26
  test('streams', async () => {
27
27
  const storage = createStorage('', StorageType.RAM);
28
28
  const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
@@ -56,9 +56,6 @@ describe('pipeline', () => {
56
56
  seq: 0
57
57
  }
58
58
  });
59
- const pipeline = new Pipeline(partyProcessor, feedReadStream, new TimeframeClock());
60
- const [readStream] = await pipeline.open();
61
- expect(readStream).toBeTruthy();
62
59
 
63
60
  //
64
61
  // Pipeline consumer.
@@ -66,10 +63,14 @@ describe('pipeline', () => {
66
63
  //
67
64
  const numMessages = 5;
68
65
  const [counter, updateCounter] = latch(numMessages);
69
- readStream.pipe(createWritable<IEchoStream>(async message => {
66
+ const echoProcessor = async (message: IEchoStream) => {
70
67
  log('Processed:', JSON.stringify(message, jsonReplacer, 2));
71
68
  updateCounter();
72
- }));
69
+ };
70
+
71
+ const pipeline = new FeedMuxer(partyProcessor, feedReadStream, new TimeframeClock());
72
+ pipeline.setEchoProcessor(echoProcessor);
73
+ await pipeline.open();
73
74
 
74
75
  //
75
76
  // Write directly to feed store.
@@ -105,17 +106,21 @@ describe('pipeline', () => {
105
106
  });
106
107
 
107
108
  const partyProcessor = new PartyProcessor(partyKey.publicKey);
108
- const pipeline = new Pipeline(
109
+
110
+ const echoMessages: IEchoStream[] = [];
111
+ const echoProcessor = async (msg: IEchoStream) => {
112
+ echoMessages.push(msg);
113
+ };
114
+
115
+ const pipeline = new FeedMuxer(
109
116
  partyProcessor,
110
117
  feedReadStream,
111
118
  new TimeframeClock(),
112
119
  createFeedWriter(feed)
113
120
  );
121
+ pipeline.setEchoProcessor(echoProcessor);
114
122
  await pipeline.open();
115
123
 
116
- const writable = new WritableArray();
117
- pipeline.inboundEchoStream!.pipe(writable);
118
-
119
124
  await pipeline.outboundHaloStream!.write(createPartyGenesisMessage(keyring, partyKey, feedKey.publicKey, identityKey));
120
125
  await waitForCondition(() => !partyProcessor.genesisRequired);
121
126
 
@@ -126,10 +131,10 @@ describe('pipeline', () => {
126
131
  }
127
132
  });
128
133
 
129
- await waitForCondition(() => writable.objects.length === 1);
134
+ await waitForCondition(() => echoMessages.length === 1);
130
135
 
131
136
  expect(partyProcessor.genesisRequired).toEqual(false);
132
- expect((writable.objects[0] as any).data).toEqual({
137
+ expect((echoMessages[0] as any).data).toEqual({
133
138
  itemId: '123',
134
139
  genesis: {
135
140
  itemType: 'foo'
@@ -4,7 +4,6 @@
4
4
 
5
5
  import assert from 'assert';
6
6
  import debug from 'debug';
7
- import { Readable } from 'stream';
8
7
 
9
8
  import { Event } from '@dxos/async';
10
9
  import { Message as HaloMessage } from '@dxos/credentials';
@@ -13,10 +12,9 @@ import { checkType } from '@dxos/debug';
13
12
  import {
14
13
  createFeedMeta, EchoEnvelope, FeedMessage, FeedStoreIterator, FeedWriter, IEchoStream, mapFeedWriter, Timeframe
15
14
  } from '@dxos/echo-protocol';
16
- import { createReadable } from '@dxos/feed-store';
17
15
  import { jsonReplacer } from '@dxos/util';
18
16
 
19
- import { TimeframeClock } from '../packlets/database';
17
+ import { EchoProcessor, TimeframeClock } from '../packlets/database';
20
18
  import { CredentialProcessor, PartyStateProvider } from './party-processor';
21
19
 
22
20
  interface Options {
@@ -28,14 +26,12 @@ const log = debug('dxos:echo-db:pipeline');
28
26
 
29
27
  /**
30
28
  * Manages the inbound and outbound message streams for an individual party.
29
+ * Reads messages from individual feeds and splits them into ECHO and HALO streams.
31
30
  */
32
- export class Pipeline {
31
+ export class FeedMuxer {
33
32
  private readonly _errors = new Event<Error>();
34
33
 
35
- /**
36
- * Messages to be consumed from the pipeline (e.g., mutations to model).
37
- */
38
- private _inboundEchoStream: Readable | undefined;
34
+ private _isOpen = false;
39
35
 
40
36
  /**
41
37
  * Messages to write into pipeline (e.g., mutations from model).
@@ -47,6 +43,8 @@ export class Pipeline {
47
43
  */
48
44
  private _outboundHaloStream: FeedWriter<HaloMessage> | undefined;
49
45
 
46
+ private _echoProcessor: EchoProcessor | undefined
47
+
50
48
  /**
51
49
  * @param _partyProcessor Processes HALO messages to update party state.
52
50
  * @param _feedStorIterator Inbound messages from the feed store.
@@ -60,20 +58,32 @@ export class Pipeline {
60
58
  private readonly _timeframeClock: TimeframeClock,
61
59
  private readonly _feedWriter?: FeedWriter<FeedMessage>,
62
60
  private readonly _options: Options = {}
63
- ) {}
61
+ ) {
62
+ if (this._feedWriter) {
63
+ const loggingWriter = mapFeedWriter<FeedMessage, FeedMessage>(async msg => {
64
+ this._options.writeLogger?.(msg);
65
+ return msg;
66
+ }, this._feedWriter);
67
+
68
+ this._outboundEchoStream = mapFeedWriter<EchoEnvelope, FeedMessage>(async message => ({
69
+ timeframe: this._timeframeClock.timeframe,
70
+ echo: message
71
+ }), loggingWriter);
72
+ this._outboundHaloStream = mapFeedWriter<HaloMessage, FeedMessage>(async message => ({
73
+ timeframe: this._timeframeClock.timeframe,
74
+ halo: message
75
+ }), loggingWriter);
76
+ }
77
+ }
64
78
 
65
79
  get isOpen () {
66
- return this._inboundEchoStream !== undefined;
80
+ return this._isOpen;
67
81
  }
68
82
 
69
83
  get readOnly () {
70
84
  return this._outboundEchoStream === undefined;
71
85
  }
72
86
 
73
- get inboundEchoStream () {
74
- return this._inboundEchoStream;
75
- }
76
-
77
87
  get outboundEchoStream () {
78
88
  return this._outboundEchoStream;
79
89
  }
@@ -86,6 +96,10 @@ export class Pipeline {
86
96
  return this._errors;
87
97
  }
88
98
 
99
+ setEchoProcessor (processor: EchoProcessor) {
100
+ this._echoProcessor = processor;
101
+ }
102
+
89
103
  /**
90
104
  * Create inbound and outbound pipielines.
91
105
  * https://nodejs.org/api/stream.html#stream_stream_pipeline_source_transforms_destination_callback
@@ -96,10 +110,8 @@ export class Pipeline {
96
110
  * Transform(dxos.echo.IEchoEnvelope => dxos.IFeedMessage): update clock
97
111
  * Feed
98
112
  */
99
- async open (): Promise<[NodeJS.ReadableStream, FeedWriter<EchoEnvelope>?]> {
100
- const { readLogger, writeLogger } = this._options;
101
-
102
- this._inboundEchoStream = createReadable();
113
+ async open (): Promise<FeedWriter<EchoEnvelope> | undefined> {
114
+ const { readLogger } = this._options;
103
115
 
104
116
  // This will exit cleanly once FeedStoreIterator is closed.
105
117
  setImmediate(async () => {
@@ -109,8 +121,6 @@ export class Pipeline {
109
121
  try {
110
122
  const { data: message } = block;
111
123
 
112
- this._timeframeClock.updateTimeframe(PublicKey.from(block.key), block.seq);
113
-
114
124
  //
115
125
  // HALO
116
126
  //
@@ -133,8 +143,8 @@ export class Pipeline {
133
143
  // Validate messge.
134
144
  const { itemId } = message.echo;
135
145
  if (itemId) {
136
- assert(this._inboundEchoStream);
137
- this._inboundEchoStream.push(checkType<IEchoStream>({
146
+ assert(this._echoProcessor);
147
+ await this._echoProcessor(checkType<IEchoStream>({
138
148
  meta: {
139
149
  seq: block.seq,
140
150
  feedKey: block.key,
@@ -150,6 +160,8 @@ export class Pipeline {
150
160
  // TODO(burdon): Can we throw and have the pipeline log (without breaking the stream)?
151
161
  log(`Skipping invalid message: ${JSON.stringify(message, jsonReplacer)}`);
152
162
  }
163
+
164
+ this._timeframeClock.updateTimeframe(PublicKey.from(block.key), block.seq);
153
165
  } catch (err: any) {
154
166
  console.error('Error in message processing.');
155
167
  console.error(err);
@@ -157,30 +169,7 @@ export class Pipeline {
157
169
  }
158
170
  });
159
171
 
160
- //
161
- // Processes outbound messages (piped to the feed).
162
- // Sets the current timeframe.
163
- //
164
- if (this._feedWriter) {
165
- const loggingWriter = mapFeedWriter<FeedMessage, FeedMessage>(async msg => {
166
- writeLogger?.(msg);
167
- return msg;
168
- }, this._feedWriter);
169
-
170
- this._outboundEchoStream = mapFeedWriter<EchoEnvelope, FeedMessage>(async message => ({
171
- timeframe: this._timeframeClock.timeframe,
172
- echo: message
173
- }), loggingWriter);
174
- this._outboundHaloStream = mapFeedWriter<HaloMessage, FeedMessage>(async message => ({
175
- timeframe: this._timeframeClock.timeframe,
176
- halo: message
177
- }), loggingWriter);
178
- }
179
-
180
- return [
181
- this._inboundEchoStream,
182
- this._outboundEchoStream
183
- ];
172
+ return this._outboundEchoStream;
184
173
  }
185
174
 
186
175
  /**
@@ -190,11 +179,9 @@ export class Pipeline {
190
179
  async close () {
191
180
  await this._feedStorIterator.close();
192
181
 
193
- if (this._inboundEchoStream) {
194
- this._inboundEchoStream.destroy();
195
- this._inboundEchoStream = undefined;
196
- }
197
-
198
182
  this._outboundEchoStream = undefined;
183
+ this._outboundHaloStream = undefined;
184
+ this._echoProcessor = undefined;
185
+ this._isOpen = false;
199
186
  }
200
187
  }
@@ -6,6 +6,6 @@ export * from './message-selector';
6
6
  export * from './party-feed-provider';
7
7
  export * from './party-processor';
8
8
  export * from '../protocol/party-protocol-factory';
9
- export * from './pipeline';
10
- export * from './party-core';
9
+ export * from './feed-muxer';
10
+ export * from './party-pipeline';
11
11
  export * from './metadata-store';
@@ -7,7 +7,7 @@ import debug from 'debug';
7
7
 
8
8
  import { PublicKey } from '@dxos/crypto';
9
9
  import { failUndefined } from '@dxos/debug';
10
- import { EchoMetadata, PartyMetadata, schema } from '@dxos/echo-protocol';
10
+ import { EchoMetadata, PartyMetadata, schema, Timeframe } from '@dxos/echo-protocol';
11
11
  import { Directory } from '@dxos/random-access-multi-storage';
12
12
 
13
13
  /**
@@ -90,7 +90,7 @@ export class MetadataStore {
90
90
  */
91
91
  async clear (): Promise<void> {
92
92
  log('Clearing all echo metadata...');
93
- await this._directory.destroy();
93
+ await this._directory.delete();
94
94
  }
95
95
 
96
96
  /**
@@ -158,4 +158,10 @@ export class MetadataStore {
158
158
  }
159
159
  return !!party.feedKeys?.find(fk => feedKey.equals(fk));
160
160
  }
161
+
162
+ async setTimeframe (partyKey: PublicKey, timeframe: Timeframe) {
163
+ const party = this.getParty(partyKey) ?? failUndefined();
164
+ party.latestTimeframe = timeframe;
165
+ await this._save();
166
+ }
161
167
  }
@@ -8,7 +8,7 @@ import debug from 'debug';
8
8
  import { Event, synchronized } from '@dxos/async';
9
9
  import { Keyring, KeyType } from '@dxos/credentials';
10
10
  import { PublicKey } from '@dxos/crypto';
11
- import { FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
11
+ import { FeedSelector, FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
12
12
  import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
13
13
  import { ComplexMap } from '@dxos/util';
14
14
 
@@ -33,19 +33,6 @@ export class PartyFeedProvider {
33
33
  return Array.from(this._feeds.values());
34
34
  }
35
35
 
36
- @synchronized
37
- async openKnownFeeds () {
38
- for (const feedKey of this._metadataStore.getParty(this._partyKey)?.feedKeys ?? []) {
39
- if (!this._feeds.has(feedKey)) {
40
- const fullKey = this._keyring.getFullKey(feedKey);
41
- const feed = fullKey?.secretKey
42
- ? await this._feedStore.openReadWriteFeed(fullKey.publicKey, fullKey.secretKey)
43
- : await this._feedStore.openReadOnlyFeed(feedKey);
44
- this._trackFeed(feed);
45
- }
46
- }
47
- }
48
-
49
36
  @synchronized
50
37
  async createOrOpenWritableFeed () {
51
38
  const partyMetadata = this._metadataStore.getParty(this._partyKey);
@@ -103,8 +90,8 @@ export class PartyFeedProvider {
103
90
  return feed;
104
91
  }
105
92
 
106
- async createIterator (messageSelector: MessageSelector, initialTimeframe?: Timeframe) {
107
- const iterator = new FeedStoreIterator(() => true, messageSelector, initialTimeframe ?? new Timeframe());
93
+ async createIterator (messageSelector: MessageSelector, feedSelector: FeedSelector, initialTimeframe?: Timeframe) {
94
+ const iterator = new FeedStoreIterator(feedSelector, messageSelector, initialTimeframe ?? new Timeframe());
108
95
  for (const feed of this._feeds.values()) {
109
96
  iterator.addFeedDescriptor(feed);
110
97
  }
@@ -17,12 +17,12 @@ import { ObjectModel } from '@dxos/object-model';
17
17
  import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
18
18
  import { afterTest } from '@dxos/testutils';
19
19
 
20
- import { MetadataStore, PartyFeedProvider } from '../pipeline';
20
+ import { MetadataStore, PartyFeedProvider } from '.';
21
21
  import { createReplicatorPlugin } from '../protocol/replicator-plugin';
22
22
  import { SnapshotStore } from '../snapshots';
23
- import { PartyCore } from './party-core';
23
+ import { PartyPipeline } from './party-pipeline';
24
24
 
25
- describe('PartyCore', () => {
25
+ describe('PartyPipeline', () => {
26
26
  const setup = async () => {
27
27
  const storage = createStorage('', StorageType.RAM);
28
28
  const feedStore = new FeedStore(storage.directory('feed'), { valueEncoding: codec });
@@ -39,7 +39,7 @@ describe('PartyCore', () => {
39
39
 
40
40
  const partyFeedProvider = new PartyFeedProvider(metadataStore, keyring, feedStore, partyKey.publicKey);
41
41
 
42
- const party = new PartyCore(
42
+ const party = new PartyPipeline(
43
43
  partyKey.publicKey,
44
44
  partyFeedProvider,
45
45
  modelFactory,
@@ -48,7 +48,7 @@ describe('PartyCore', () => {
48
48
  );
49
49
 
50
50
  const feed = await partyFeedProvider.createOrOpenWritableFeed();
51
- await party.open();
51
+ await party.open({ feedHints: [feed.key] });
52
52
  afterTest(async () => party.close());
53
53
 
54
54
  // PartyGenesis (self-signed by Party).
@@ -88,7 +88,7 @@ describe('PartyCore', () => {
88
88
  });
89
89
 
90
90
  test('create item with parent and then reload', async () => {
91
- const { party } = await setup();
91
+ const { party, feedKey } = await setup();
92
92
 
93
93
  {
94
94
  const parent = await party.database.createItem({ model: ObjectModel, type: 'parent' });
@@ -103,7 +103,7 @@ describe('PartyCore', () => {
103
103
  }
104
104
 
105
105
  await party.close();
106
- await party.open();
106
+ await party.open({ feedHints: [feedKey] });
107
107
 
108
108
  {
109
109
  await party.database.select().exec().update.waitFor(result => result.entities.length === 2);
@@ -149,7 +149,7 @@ describe('PartyCore', () => {
149
149
 
150
150
  const otherFeedKey = PublicKey.random();
151
151
 
152
- const party = new PartyCore(
152
+ const party = new PartyPipeline(
153
153
  partyKey.publicKey,
154
154
  partyFeedProvider,
155
155
  modelFactory,
@@ -161,7 +161,7 @@ describe('PartyCore', () => {
161
161
 
162
162
  const feedOpened = feedStore.feedOpenedEvent.waitForCount(1);
163
163
 
164
- await party.open([{ type: KeyType.FEED, publicKey: otherFeedKey }]);
164
+ await party.open({ feedHints: [otherFeedKey] });
165
165
  afterTest(async () => party.close());
166
166
 
167
167
  await feedOpened;
@@ -250,6 +250,28 @@ describe('PartyCore', () => {
250
250
  await promiseTimeout(party.database.waitForItem({ id: itemId }), 1000, new Error('timeout'));
251
251
  });
252
252
 
253
+ test('wait to reach specific timeframe', async () => {
254
+ const { party, feedKey } = await setup();
255
+
256
+ {
257
+ const parent = await party.database.createItem({ model: ObjectModel, type: 'parent' });
258
+ const child = await party.database.createItem({
259
+ model: ObjectModel,
260
+ parent: parent.id,
261
+ type: 'child'
262
+ });
263
+
264
+ expect(child.parent).toEqual(parent);
265
+ expect(parent.children).toContain(child);
266
+ }
267
+
268
+ const timeframe = party.timeframe;
269
+ expect(timeframe.isEmpty()).toBeFalsy();
270
+
271
+ await party.close();
272
+ await party.open({ feedHints: [feedKey], targetTimeframe: timeframe });
273
+ });
274
+
253
275
  test('two instances replicating', async () => {
254
276
  const peer1 = await setup();
255
277
 
@@ -264,7 +286,7 @@ describe('PartyCore', () => {
264
286
 
265
287
  const partyFeedProvider = new PartyFeedProvider(metadataStore, peer1.keyring, feedStore, peer1.party.key);
266
288
 
267
- const party2 = new PartyCore(
289
+ const party2 = new PartyPipeline(
268
290
  peer1.party.key,
269
291
  partyFeedProvider,
270
292
  modelFactory,
@@ -281,10 +303,9 @@ describe('PartyCore', () => {
281
303
  [peer1.partyKey]
282
304
  ));
283
305
 
284
- await party2.open([{
285
- publicKey: peer1.feedKey,
286
- type: KeyType.FEED
287
- }]);
306
+ await party2.open({
307
+ feedHints: [peer1.feedKey]
308
+ });
288
309
  afterTest(async () => party2.close());
289
310
 
290
311
  createTestProtocolPair(