@dxos/echo-db 2.33.5-dev.9e11cb97 → 2.33.5-dev.b4e42956

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/dist/src/echo.d.ts.map +1 -1
  2. package/dist/src/echo.js +1 -1
  3. package/dist/src/echo.js.map +1 -1
  4. package/dist/src/echo.test.js +12 -4
  5. package/dist/src/echo.test.js.map +1 -1
  6. package/dist/src/halo/halo-factory.d.ts +4 -4
  7. package/dist/src/halo/halo-factory.d.ts.map +1 -1
  8. package/dist/src/halo/halo-factory.js +3 -2
  9. package/dist/src/halo/halo-factory.js.map +1 -1
  10. package/dist/src/halo/halo-party.d.ts +4 -3
  11. package/dist/src/halo/halo-party.d.ts.map +1 -1
  12. package/dist/src/halo/halo-party.js +8 -4
  13. package/dist/src/halo/halo-party.js.map +1 -1
  14. package/dist/src/halo/halo.d.ts +2 -2
  15. package/dist/src/halo/halo.d.ts.map +1 -1
  16. package/dist/src/halo/party-opener.d.ts.map +1 -1
  17. package/dist/src/halo/party-opener.js +3 -1
  18. package/dist/src/halo/party-opener.js.map +1 -1
  19. package/dist/src/invitations/greeting-initiator.d.ts +2 -2
  20. package/dist/src/invitations/greeting-initiator.d.ts.map +1 -1
  21. package/dist/src/invitations/greeting-initiator.js +2 -1
  22. package/dist/src/invitations/greeting-initiator.js.map +1 -1
  23. package/dist/src/packlets/database/data-mirror.test.js +2 -2
  24. package/dist/src/packlets/database/data-mirror.test.js.map +1 -1
  25. package/dist/src/packlets/database/database-backend.d.ts +4 -5
  26. package/dist/src/packlets/database/database-backend.d.ts.map +1 -1
  27. package/dist/src/packlets/database/database-backend.js +5 -6
  28. package/dist/src/packlets/database/database-backend.js.map +1 -1
  29. package/dist/src/packlets/database/item-demuxer.d.ts +2 -2
  30. package/dist/src/packlets/database/item-demuxer.d.ts.map +1 -1
  31. package/dist/src/packlets/database/item-demuxer.js +2 -3
  32. package/dist/src/packlets/database/item-demuxer.js.map +1 -1
  33. package/dist/src/packlets/database/item-demuxer.test.js +13 -9
  34. package/dist/src/packlets/database/item-demuxer.test.js.map +1 -1
  35. package/dist/src/packlets/database/testing.d.ts.map +1 -1
  36. package/dist/src/packlets/database/testing.js +3 -4
  37. package/dist/src/packlets/database/testing.js.map +1 -1
  38. package/dist/src/packlets/database/timeframe-clock.d.ts +1 -0
  39. package/dist/src/packlets/database/timeframe-clock.d.ts.map +1 -1
  40. package/dist/src/packlets/database/timeframe-clock.js +3 -0
  41. package/dist/src/packlets/database/timeframe-clock.js.map +1 -1
  42. package/dist/src/parties/data-party.d.ts +6 -5
  43. package/dist/src/parties/data-party.d.ts.map +1 -1
  44. package/dist/src/parties/data-party.js +26 -5
  45. package/dist/src/parties/data-party.js.map +1 -1
  46. package/dist/src/parties/data-party.test.js +24 -5
  47. package/dist/src/parties/data-party.test.js.map +1 -1
  48. package/dist/src/parties/party-factory.d.ts +5 -4
  49. package/dist/src/parties/party-factory.d.ts.map +1 -1
  50. package/dist/src/parties/party-factory.js +15 -11
  51. package/dist/src/parties/party-factory.js.map +1 -1
  52. package/dist/src/parties/party-manager.d.ts +3 -4
  53. package/dist/src/parties/party-manager.d.ts.map +1 -1
  54. package/dist/src/parties/party-manager.js +8 -7
  55. package/dist/src/parties/party-manager.js.map +1 -1
  56. package/dist/src/parties/party-manager.test.js +9 -9
  57. package/dist/src/parties/party-manager.test.js.map +1 -1
  58. package/dist/src/pipeline/{pipeline.d.ts → feed-muxer.d.ts} +8 -11
  59. package/dist/src/pipeline/feed-muxer.d.ts.map +1 -0
  60. package/dist/src/pipeline/{pipeline.js → feed-muxer.js} +34 -41
  61. package/dist/src/pipeline/feed-muxer.js.map +1 -0
  62. package/dist/src/pipeline/feed-muxer.test.d.ts +2 -0
  63. package/dist/src/pipeline/feed-muxer.test.d.ts.map +1 -0
  64. package/dist/src/pipeline/{pipeline.test.js → feed-muxer.test.js} +17 -15
  65. package/dist/src/pipeline/feed-muxer.test.js.map +1 -0
  66. package/dist/src/pipeline/index.d.ts +2 -2
  67. package/dist/src/pipeline/index.d.ts.map +1 -1
  68. package/dist/src/pipeline/index.js +2 -2
  69. package/dist/src/pipeline/index.js.map +1 -1
  70. package/dist/src/pipeline/message-selector.d.ts.map +1 -1
  71. package/dist/src/pipeline/message-selector.js +6 -5
  72. package/dist/src/pipeline/message-selector.js.map +1 -1
  73. package/dist/src/pipeline/metadata-store.d.ts +2 -1
  74. package/dist/src/pipeline/metadata-store.d.ts.map +1 -1
  75. package/dist/src/pipeline/metadata-store.js +7 -1
  76. package/dist/src/pipeline/metadata-store.js.map +1 -1
  77. package/dist/src/pipeline/party-feed-provider.d.ts +2 -3
  78. package/dist/src/pipeline/party-feed-provider.d.ts.map +1 -1
  79. package/dist/src/pipeline/party-feed-provider.js +2 -17
  80. package/dist/src/pipeline/party-feed-provider.js.map +1 -1
  81. package/dist/src/pipeline/{party-core.d.ts → party-pipeline.d.ts} +23 -9
  82. package/dist/src/pipeline/party-pipeline.d.ts.map +1 -0
  83. package/dist/src/pipeline/{party-core.js → party-pipeline.js} +26 -25
  84. package/dist/src/pipeline/party-pipeline.js.map +1 -0
  85. package/dist/src/pipeline/party-pipeline.test.d.ts +2 -0
  86. package/dist/src/pipeline/party-pipeline.test.d.ts.map +1 -0
  87. package/dist/src/pipeline/{party-core.test.js → party-pipeline.test.js} +50 -33
  88. package/dist/src/pipeline/party-pipeline.test.js.map +1 -0
  89. package/dist/src/snapshots/snapshot-generator.d.ts +2 -2
  90. package/dist/src/snapshots/snapshot-generator.d.ts.map +1 -1
  91. package/dist/src/snapshots/snapshot-generator.js.map +1 -1
  92. package/dist/src/snapshots/snapshot-store.js +1 -1
  93. package/dist/src/snapshots/snapshot-store.js.map +1 -1
  94. package/dist/tsconfig.tsbuildinfo +1 -1
  95. package/package.json +17 -17
  96. package/src/echo.test.ts +15 -4
  97. package/src/echo.ts +1 -0
  98. package/src/halo/halo-factory.ts +6 -6
  99. package/src/halo/halo-party.ts +10 -8
  100. package/src/halo/halo.ts +2 -2
  101. package/src/halo/party-opener.ts +3 -1
  102. package/src/invitations/greeting-initiator.ts +5 -4
  103. package/src/packlets/database/data-mirror.test.ts +2 -2
  104. package/src/packlets/database/database-backend.ts +7 -6
  105. package/src/packlets/database/item-demuxer.test.ts +20 -18
  106. package/src/packlets/database/item-demuxer.ts +5 -4
  107. package/src/packlets/database/testing.ts +3 -6
  108. package/src/packlets/database/timeframe-clock.ts +4 -0
  109. package/src/parties/data-party.test.ts +39 -8
  110. package/src/parties/data-party.ts +31 -9
  111. package/src/parties/party-factory.ts +19 -17
  112. package/src/parties/party-manager.test.ts +15 -9
  113. package/src/parties/party-manager.ts +9 -6
  114. package/src/pipeline/{pipeline.test.ts → feed-muxer.test.ts} +20 -16
  115. package/src/pipeline/{pipeline.ts → feed-muxer.ts} +39 -51
  116. package/src/pipeline/index.ts +2 -2
  117. package/src/pipeline/message-selector.ts +6 -5
  118. package/src/pipeline/metadata-store.ts +8 -2
  119. package/src/pipeline/party-feed-provider.ts +3 -16
  120. package/src/pipeline/{party-core.test.ts → party-pipeline.test.ts} +49 -27
  121. package/src/pipeline/{party-core.ts → party-pipeline.ts} +53 -24
  122. package/src/snapshots/snapshot-generator.ts +2 -2
  123. package/src/snapshots/snapshot-store.ts +1 -1
  124. package/dist/src/pipeline/party-core.d.ts.map +0 -1
  125. package/dist/src/pipeline/party-core.js.map +0 -1
  126. package/dist/src/pipeline/party-core.test.d.ts +0 -2
  127. package/dist/src/pipeline/party-core.test.d.ts.map +0 -1
  128. package/dist/src/pipeline/party-core.test.js.map +0 -1
  129. package/dist/src/pipeline/pipeline.d.ts.map +0 -1
  130. package/dist/src/pipeline/pipeline.js.map +0 -1
  131. package/dist/src/pipeline/pipeline.test.d.ts +0 -2
  132. package/dist/src/pipeline/pipeline.test.d.ts.map +0 -1
  133. package/dist/src/pipeline/pipeline.test.js.map +0 -1
@@ -13,8 +13,7 @@ import {
13
13
  Keyring,
14
14
  KeyType,
15
15
  Filter,
16
- SecretProvider,
17
- KeyHint
16
+ SecretProvider
18
17
  } from '@dxos/credentials';
19
18
  import { keyToString, PublicKey, keyPairFromSeedPhrase } from '@dxos/crypto';
20
19
  import { ModelFactory } from '@dxos/model-factory';
@@ -23,7 +22,7 @@ import { ObjectModel } from '@dxos/object-model';
23
22
 
24
23
  import { createHaloPartyAdmissionMessage, GreetingInitiator, HaloRecoveryInitiator, InvitationDescriptor, InvitationDescriptorType, OfflineInvitationClaimer } from '../invitations';
25
24
  import { PARTY_ITEM_TYPE } from '../parties';
26
- import { PartyFeedProvider, PartyOptions } from '../pipeline';
25
+ import { PartyFeedProvider, PipelineOptions } from '../pipeline';
27
26
  import { CredentialsSigner } from '../protocol/credentials-signer';
28
27
  import { SnapshotStore } from '../snapshots';
29
28
  import {
@@ -51,19 +50,20 @@ export class HaloFactory {
51
50
  private readonly _snapshotStore: SnapshotStore,
52
51
  private readonly _feedProviderFactory: (partyKey: PublicKey) => PartyFeedProvider,
53
52
  private readonly _keyring: Keyring,
54
- private readonly _options: PartyOptions = {}
53
+ private readonly _options: PipelineOptions = {}
55
54
  ) {}
56
55
 
57
- async constructParty (hints: KeyHint[]): Promise<HaloParty> {
56
+ async constructParty (feedHints: PublicKey[]): Promise<HaloParty> {
58
57
  const credentialsSigner = CredentialsSigner.createDirectDeviceSigner(this._keyring);
59
58
  const feedProvider = this._feedProviderFactory(credentialsSigner.getIdentityKey().publicKey);
59
+ const writableFeed = await feedProvider.createOrOpenWritableFeed();
60
60
  const halo = new HaloParty(
61
61
  this._modelFactory,
62
62
  this._snapshotStore,
63
63
  feedProvider,
64
64
  credentialsSigner,
65
65
  this._networkManager,
66
- hints,
66
+ [...feedHints, writableFeed.key],
67
67
  undefined,
68
68
  this._options
69
69
  );
@@ -14,7 +14,7 @@ import { NetworkManager } from '@dxos/network-manager';
14
14
 
15
15
  import { InvitationAuthenticator, InvitationDescriptor, InvitationFactory, InvitationOptions } from '../invitations';
16
16
  import { PARTY_ITEM_TYPE } from '../parties';
17
- import { PartyFeedProvider, PartyProtocolFactory, PartyCore, PartyOptions } from '../pipeline';
17
+ import { PartyFeedProvider, PartyProtocolFactory, PartyPipeline, PipelineOptions } from '../pipeline';
18
18
  import { createAuthenticator, createAuthPlugin, createCredentialsProvider, createHaloRecoveryPlugin } from '../protocol';
19
19
  import { CredentialsSigner } from '../protocol/credentials-signer';
20
20
  import { createReplicatorPlugin } from '../protocol/replicator-plugin';
@@ -41,7 +41,7 @@ export interface JoinedParty {
41
41
  export class HaloParty {
42
42
  public readonly update = new Event<void>();
43
43
 
44
- private readonly _partyCore: PartyCore;
44
+ private readonly _partyCore: PartyPipeline;
45
45
  private _invitationManager?: InvitationFactory;
46
46
  private _protocol?: PartyProtocolFactory;
47
47
 
@@ -54,17 +54,16 @@ export class HaloParty {
54
54
  private readonly _feedProvider: PartyFeedProvider,
55
55
  private readonly _credentialsSigner: CredentialsSigner,
56
56
  private readonly _networkManager: NetworkManager,
57
- private readonly _hints: KeyHint[] = [],
58
- _initialTimeframe: Timeframe | undefined,
59
- _options: PartyOptions
57
+ private readonly _feedHints: PublicKey[] = [],
58
+ private readonly _initialTimeframe: Timeframe | undefined,
59
+ _options: PipelineOptions
60
60
  ) {
61
- this._partyCore = new PartyCore(
61
+ this._partyCore = new PartyPipeline(
62
62
  _credentialsSigner.getIdentityKey().publicKey,
63
63
  _feedProvider,
64
64
  modelFactory,
65
65
  snapshotStore,
66
66
  _credentialsSigner.getIdentityKey().publicKey,
67
- _initialTimeframe,
68
67
  _options
69
68
  );
70
69
 
@@ -145,7 +144,10 @@ export class HaloParty {
145
144
  return this;
146
145
  }
147
146
 
148
- await this._partyCore.open(this._hints);
147
+ await this._partyCore.open({
148
+ feedHints: this._feedHints,
149
+ initialTimeframe: this._initialTimeframe
150
+ });
149
151
 
150
152
  this._invitationManager = new InvitationFactory(
151
153
  this._partyCore.processor,
package/src/halo/halo.ts CHANGED
@@ -15,7 +15,7 @@ import { NetworkManager } from '@dxos/network-manager';
15
15
  import { ResultSet } from '../api';
16
16
  import { InvitationAuthenticator, InvitationDescriptor, InvitationOptions } from '../invitations';
17
17
  import { OpenProgress } from '../parties';
18
- import { MetadataStore, PartyOptions, PartyFeedProvider } from '../pipeline';
18
+ import { MetadataStore, PipelineOptions, PartyFeedProvider } from '../pipeline';
19
19
  import { SnapshotStore } from '../snapshots';
20
20
  import { Contact } from './contact-manager';
21
21
  import { HaloFactory } from './halo-factory';
@@ -36,7 +36,7 @@ export interface HaloConfiguration {
36
36
  modelFactory: ModelFactory,
37
37
  snapshotStore: SnapshotStore,
38
38
  feedProviderFactory: (partyKey: PublicKey) => PartyFeedProvider,
39
- options: PartyOptions
39
+ options: PipelineOptions
40
40
  }
41
41
 
42
42
  /**
@@ -4,6 +4,7 @@
4
4
 
5
5
  import debug from 'debug';
6
6
 
7
+ import { KeyType } from '@dxos/credentials';
7
8
  import { SubscriptionGroup, Unsubscribe } from '@dxos/util';
8
9
 
9
10
  import { PartyManager } from '../parties';
@@ -25,7 +26,8 @@ export const autoPartyOpener = (preferences: Preferences, partyManager: PartyMan
25
26
  for (const partyDesc of values) {
26
27
  if (!partyManager.parties.some(x => x.key === partyDesc.partyKey)) {
27
28
  log(`Auto-opening new Party from HALO: ${partyDesc.partyKey.toHex()} hints=${JSON.stringify(partyDesc.keyHints)}`);
28
- await partyManager.addParty(partyDesc.partyKey, partyDesc.keyHints);
29
+ const feedHints = partyDesc.keyHints.filter(hint => hint.type === KeyType.FEED).map(hint => hint.publicKey!);
30
+ await partyManager.addParty(partyDesc.partyKey, feedHints);
29
31
  }
30
32
  }
31
33
  }));
@@ -20,7 +20,8 @@ import {
20
20
  SecretProvider,
21
21
  WithTypeUrl,
22
22
  ERR_GREET_CONNECTED_TO_SWARM_TIMEOUT,
23
- SignedMessage
23
+ SignedMessage,
24
+ NotarizeResponse
24
25
  } from '@dxos/credentials';
25
26
  import { keyToString, PublicKey } from '@dxos/crypto';
26
27
  import { FullyConnectedTopology, NetworkManager } from '@dxos/network-manager';
@@ -111,7 +112,7 @@ export class GreetingInitiator {
111
112
  /**
112
113
  * Called after connecting to initiate greeting protocol exchange.
113
114
  */
114
- async redeemInvitation (secretProvider: SecretProvider) {
115
+ async redeemInvitation (secretProvider: SecretProvider): Promise<{ partyKey: PublicKey, hints: PublicKey[] }> {
115
116
  assert(this._state === GreetingState.CONNECTED);
116
117
  const { swarmKey } = this._invitationDescriptor;
117
118
 
@@ -150,7 +151,7 @@ export class GreetingInitiator {
150
151
  const credentialMessages = await this._getMessagesToNotarize(PublicKey.from(partyKey), nonce);
151
152
 
152
153
  // Send the signed payload to the greeting responder.
153
- const notarizeResponse = await this._greeterPlugin.send(responderPeerId,
154
+ const notarizeResponse: NotarizeResponse = await this._greeterPlugin.send(responderPeerId,
154
155
  createGreetingNotarizeMessage(secret, credentialMessages as WithTypeUrl<Message>[]));
155
156
 
156
157
  //
@@ -171,7 +172,7 @@ export class GreetingInitiator {
171
172
  this._state = GreetingState.SUCCEEDED;
172
173
  return {
173
174
  partyKey,
174
- hints: notarizeResponse.hints
175
+ hints: notarizeResponse.feedHints ?? []
175
176
  };
176
177
  }
177
178
 
@@ -26,8 +26,8 @@ describe('DataMirror', () => {
26
26
  const itemManager = new ItemManager(modelFactory, PublicKey.random(), feed);
27
27
  const itemDemuxer = new ItemDemuxer(itemManager, modelFactory, { snapshots: true });
28
28
 
29
- const stream = itemDemuxer.open();
30
- feed.written.on(([msg, meta]) => stream.write({
29
+ const process = itemDemuxer.open();
30
+ feed.written.on(([msg, meta]) => process({
31
31
  data: msg,
32
32
  meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe() }
33
33
  } as any));
@@ -10,7 +10,7 @@ import { ModelFactory } from '@dxos/model-factory';
10
10
 
11
11
  import { DataMirror } from './data-mirror';
12
12
  import { DataServiceHost } from './data-service-host';
13
- import { ItemDemuxer, ItemDemuxerOptions } from './item-demuxer';
13
+ import { EchoProcessor, ItemDemuxer, ItemDemuxerOptions } from './item-demuxer';
14
14
  import { ItemManager } from './item-manager';
15
15
 
16
16
  const log = debug('dxos:echo-db:database-backend');
@@ -42,12 +42,11 @@ export interface DatabaseBackend {
42
42
  * Write operations result in mutations being written to the outgoing stream.
43
43
  */
44
44
  export class FeedDatabaseBackend implements DatabaseBackend {
45
- private _itemDemuxerInboundStream!: NodeJS.WritableStream
45
+ private _echoProcessor!: EchoProcessor;
46
46
  private _itemManager!: ItemManager;
47
47
  private _itemDemuxer!: ItemDemuxer;
48
48
 
49
49
  constructor (
50
- private readonly _inboundStream: NodeJS.ReadableStream,
51
50
  private readonly _outboundStream: FeedWriter<EchoEnvelope> | undefined,
52
51
  private readonly _snapshot?: DatabaseSnapshot,
53
52
  private readonly _options: ItemDemuxerOptions = {}
@@ -56,16 +55,18 @@ export class FeedDatabaseBackend implements DatabaseBackend {
56
55
  async open (itemManager: ItemManager, modelFactory: ModelFactory) {
57
56
  this._itemManager = itemManager;
58
57
  this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory, this._options);
59
- this._itemDemuxerInboundStream = this._itemDemuxer.open();
60
- this._inboundStream.pipe(this._itemDemuxerInboundStream);
58
+ this._echoProcessor = this._itemDemuxer.open();
61
59
 
62
60
  if (this._snapshot) {
63
61
  await this._itemDemuxer.restoreFromSnapshot(this._snapshot);
64
62
  }
65
63
  }
66
64
 
65
+ get echoProcessor () {
66
+ return this._echoProcessor;
67
+ }
68
+
67
69
  async close () {
68
- this._inboundStream?.unpipe(this._itemDemuxerInboundStream);
69
70
  }
70
71
 
71
72
  get isReadOnly (): boolean {
@@ -9,8 +9,7 @@ import { it as test } from 'mocha';
9
9
  import { latch } from '@dxos/async';
10
10
  import { createId, PublicKey } from '@dxos/crypto';
11
11
  import { checkType } from '@dxos/debug';
12
- import { createMockFeedWriterFromStream, EchoEnvelope, IEchoStream, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
13
- import { createTransform } from '@dxos/feed-store';
12
+ import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
14
13
  import { ModelFactory, TestModel } from '@dxos/model-factory';
15
14
  import { ObjectModel } from '@dxos/object-model';
16
15
 
@@ -32,7 +31,7 @@ describe('Item demuxer', () => {
32
31
  const itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
33
32
 
34
33
  const inboundStream = itemDemuxer.open();
35
- feedWriter.written.on(([msg, meta]) => inboundStream.write({
34
+ feedWriter.written.on(([msg, meta]) => inboundStream({
36
35
  data: msg,
37
36
  meta: { ...meta, memberKey }
38
37
  } as any));
@@ -96,28 +95,31 @@ describe('Item demuxer', () => {
96
95
  const modelFactory = new ModelFactory()
97
96
  .registerModel(ObjectModel);
98
97
 
99
- const writeStream = createTransform<EchoEnvelope, IEchoStream>(
100
- async (message: EchoEnvelope): Promise<IEchoStream> => ({
101
- meta: {
102
- feedKey: PublicKey.random(),
103
- memberKey: PublicKey.random(),
104
- seq: 0,
105
- timeframe: new Timeframe()
106
- },
107
- data: message
108
- })
109
- );
110
- const itemManager = new ItemManager(modelFactory, PublicKey.random(), createMockFeedWriterFromStream(writeStream));
98
+ const itemManager = new ItemManager(modelFactory, PublicKey.random(), {
99
+ write: async (message) => {
100
+ void processEchoMessage(message);
101
+ return { feedKey: PublicKey.random(), seq: 0 };
102
+ }
103
+ });
111
104
  const itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
112
- writeStream.pipe(itemDemuxer.open());
105
+ const processor = itemDemuxer.open();
106
+ const processEchoMessage = (message: EchoEnvelope) => processor({
107
+ meta: {
108
+ feedKey: PublicKey.random(),
109
+ memberKey: PublicKey.random(),
110
+ seq: 0,
111
+ timeframe: new Timeframe()
112
+ },
113
+ data: message
114
+ });
113
115
 
114
- writeStream.write(checkType<EchoEnvelope>({
116
+ void processEchoMessage(checkType<EchoEnvelope>({
115
117
  itemId: 'foo',
116
118
  genesis: {
117
119
  modelType: TestModel.meta.type
118
120
  }
119
121
  }));
120
- writeStream.write(checkType<EchoEnvelope>({
122
+ void processEchoMessage(checkType<EchoEnvelope>({
121
123
  itemId: 'bar',
122
124
  genesis: {
123
125
  modelType: ObjectModel.meta.type
@@ -8,7 +8,6 @@ import debug from 'debug';
8
8
  import { Event } from '@dxos/async';
9
9
  import { failUndefined } from '@dxos/debug';
10
10
  import { DatabaseSnapshot, IEchoStream, ItemID, ItemSnapshot, LinkSnapshot } from '@dxos/echo-protocol';
11
- import { createWritable } from '@dxos/feed-store';
12
11
  import { Model, ModelFactory, ModelMessage } from '@dxos/model-factory';
13
12
 
14
13
  import { Entity } from './entity';
@@ -22,6 +21,8 @@ export interface ItemDemuxerOptions {
22
21
  snapshots?: boolean
23
22
  }
24
23
 
24
+ export type EchoProcessor = (message: IEchoStream) => Promise<void>
25
+
25
26
  /**
26
27
  * Creates a stream that consumes `IEchoStream` messages and routes them to the associated items.
27
28
  * @param itemManager
@@ -35,7 +36,7 @@ export class ItemDemuxer {
35
36
  private readonly _options: ItemDemuxerOptions = {}
36
37
  ) {}
37
38
 
38
- open (): NodeJS.WritableStream {
39
+ open (): EchoProcessor {
39
40
  this._modelFactory.registered.on(async model => {
40
41
  for (const item of this._itemManager.getUninitializedEntities()) {
41
42
  if (item._stateManager.modelType === model.meta.type) {
@@ -46,7 +47,7 @@ export class ItemDemuxer {
46
47
 
47
48
  // TODO(burdon): Factor out.
48
49
  // TODO(burdon): Should this implement some "back-pressure" (hints) to the PartyProcessor?
49
- return createWritable<IEchoStream>(async (message: IEchoStream) => {
50
+ return async (message: IEchoStream) => {
50
51
  const { data: { itemId, genesis, itemMutation, mutation, snapshot }, meta } = message;
51
52
  assert(itemId);
52
53
 
@@ -111,7 +112,7 @@ export class ItemDemuxer {
111
112
  }
112
113
 
113
114
  this.mutation.emit(message);
114
- });
115
+ };
115
116
  }
116
117
 
117
118
  createSnapshot (): DatabaseSnapshot {
@@ -2,8 +2,6 @@
2
2
  // Copyright 2021 DXOS.org
3
3
  //
4
4
 
5
- import { Readable } from 'stream';
6
-
7
5
  import { PublicKey } from '@dxos/crypto';
8
6
  import { EchoEnvelope, MockFeedWriter, Timeframe } from '@dxos/echo-protocol';
9
7
  import { ModelFactory } from '@dxos/model-factory';
@@ -15,12 +13,11 @@ import { FeedDatabaseBackend, RemoteDatabaseBackend } from './database-backend';
15
13
 
16
14
  export const createInMemoryDatabase = async (modelFactory: ModelFactory) => {
17
15
  const feed = new MockFeedWriter<EchoEnvelope>();
18
- const inboundStream = new Readable({ read: () => {}, objectMode: true });
19
- feed.written.on(([data, meta]) => inboundStream.push({ data, meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe([[meta.feedKey, meta.seq]]) } }));
20
-
16
+ const backend = new FeedDatabaseBackend(feed, undefined, { snapshots: true });
17
+ feed.written.on(([data, meta]) => backend.echoProcessor({ data, meta: { ...meta, memberKey: PublicKey.random(), timeframe: new Timeframe([[meta.feedKey, meta.seq]]) } }));
21
18
  const database = new Database(
22
19
  modelFactory,
23
- new FeedDatabaseBackend(inboundStream, feed, undefined, { snapshots: true }),
20
+ backend,
24
21
  PublicKey.random()
25
22
  );
26
23
 
@@ -28,4 +28,8 @@ export class TimeframeClock {
28
28
  const gaps = Timeframe.dependencies(timeframe, this._timeframe);
29
29
  return !gaps.isEmpty();
30
30
  }
31
+
32
+ async waitUntilReached (target: Timeframe) {
33
+ await this.update.waitForCondition(() => Timeframe.dependencies(target, this._timeframe).isEmpty());
34
+ }
31
35
  }
@@ -5,7 +5,7 @@
5
5
  import expect from 'expect';
6
6
  import { it as test } from 'mocha';
7
7
 
8
- import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, KeyHint, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
8
+ import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
9
9
  import { PublicKey } from '@dxos/crypto';
10
10
  import { codec } from '@dxos/echo-protocol';
11
11
  import { FeedStore } from '@dxos/feed-store';
@@ -22,7 +22,7 @@ import { SnapshotStore } from '../snapshots';
22
22
  import { DataParty } from './data-party';
23
23
 
24
24
  describe('DataParty', () => {
25
- const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, hints: KeyHint[]) => {
25
+ const createParty = async (identity: IdentityCredentials, partyKey: PublicKey, feedHints: PublicKey[]) => {
26
26
 
27
27
  const storage = createStorage('', StorageType.RAM);
28
28
  const snapshotStore = new SnapshotStore(storage.directory('snapshots'));
@@ -31,17 +31,20 @@ describe('DataParty', () => {
31
31
  const modelFactory = new ModelFactory().registerModel(ObjectModel);
32
32
  const networkManager = new NetworkManager();
33
33
  const partyFeedProvider = new PartyFeedProvider(metadataStore, identity.keyring, feedStore, partyKey);
34
+ const writableFeed = await partyFeedProvider.createOrOpenWritableFeed();
34
35
 
35
- return new DataParty(
36
+ const party = new DataParty(
36
37
  partyKey,
37
38
  modelFactory,
38
39
  snapshotStore,
39
40
  partyFeedProvider,
41
+ metadataStore,
40
42
  identity.createCredentialsSigner(),
41
43
  identity.preferences,
42
- networkManager,
43
- hints
44
+ networkManager
44
45
  );
46
+ party._setFeedHints([...feedHints, writableFeed.key]);
47
+ return party;
45
48
  };
46
49
 
47
50
  test('open & close', async () => {
@@ -74,6 +77,33 @@ describe('DataParty', () => {
74
77
  await party.close();
75
78
  });
76
79
 
80
+ test('data is immediately available after re-opening', async () => {
81
+ const keyring = new Keyring();
82
+ const identity = await createTestIdentityCredentials(keyring);
83
+ const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
84
+ const party = await createParty(identity, partyKey.publicKey, []);
85
+ await party.open();
86
+
87
+ const feed = await party.getWriteFeed();
88
+ await party.credentialsWriter.write(createPartyGenesisMessage(
89
+ keyring,
90
+ partyKey,
91
+ feed.key,
92
+ partyKey
93
+ ));
94
+
95
+ for (let i = 0; i < 10; i++) {
96
+ await party.database.createItem({ type: 'test:item' });
97
+ }
98
+
99
+ await party.close();
100
+ await party.open();
101
+
102
+ expect(party.database.select({ type: 'test:item' }).exec().entities).toHaveLength(10);
103
+
104
+ await party.close();
105
+ });
106
+
77
107
  test('authenticates its own credentials', async () => {
78
108
  const keyring = new Keyring();
79
109
  const identity = await createTestIdentityCredentials(keyring);
@@ -88,6 +118,7 @@ describe('DataParty', () => {
88
118
  feed.key,
89
119
  partyKey
90
120
  ));
121
+ await party.processor.feedAdded.waitForCount(1);
91
122
 
92
123
  const authenticator = createAuthenticator(party.processor, identity.createCredentialsSigner(), party.credentialsWriter);
93
124
  const credentialsProvider = createCredentialsProvider(identity.createCredentialsSigner(), party.key, feed.key);
@@ -112,6 +143,8 @@ describe('DataParty', () => {
112
143
  feed.key,
113
144
  partyKey
114
145
  ));
146
+ await party.processor.feedAdded.waitForCount(1);
147
+
115
148
  const authenticator = createAuthenticator(party.processor, identityA.createCredentialsSigner(), party.credentialsWriter);
116
149
 
117
150
  const identityB = await deriveTestDeviceCredentials(identityA);
@@ -145,9 +178,7 @@ describe('DataParty', () => {
145
178
  ));
146
179
 
147
180
  const identityB = await deriveTestDeviceCredentials(identityA);
148
- const partyB = await createParty(identityB, partyKey.publicKey, [
149
- { type: KeyType.FEED, publicKey: feedA.key }
150
- ]);
181
+ const partyB = await createParty(identityB, partyKey.publicKey, [feedA.key]);
151
182
  await partyB.open();
152
183
 
153
184
  await partyA.database.createItem({ type: 'test:item-a' });
@@ -5,7 +5,6 @@
5
5
  import assert from 'assert';
6
6
 
7
7
  import { synchronized, Event } from '@dxos/async';
8
- import { KeyHint } from '@dxos/credentials';
9
8
  import { PublicKey } from '@dxos/crypto';
10
9
  import { timed } from '@dxos/debug';
11
10
  import { PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
@@ -18,7 +17,7 @@ import { ResultSet } from '../api';
18
17
  import { ActivationOptions, PartyPreferences, Preferences } from '../halo';
19
18
  import { InvitationFactory } from '../invitations';
20
19
  import { Database, Item } from '../packlets/database';
21
- import { PartyFeedProvider, PartyProtocolFactory, PartyCore, PartyOptions } from '../pipeline';
20
+ import { PartyFeedProvider, PartyProtocolFactory, PartyPipeline, PipelineOptions, MetadataStore } from '../pipeline';
22
21
  import { createAuthPlugin, createOfflineInvitationPlugin, createAuthenticator, createCredentialsProvider } from '../protocol';
23
22
  import { CredentialsSigner } from '../protocol/credentials-signer';
24
23
  import { createReplicatorPlugin } from '../protocol/replicator-plugin';
@@ -43,31 +42,31 @@ export interface PartyMember {
43
42
  export class DataParty {
44
43
  public readonly update = new Event<void>();
45
44
 
46
- private readonly _partyCore: PartyCore;
45
+ private readonly _partyCore: PartyPipeline;
47
46
  private readonly _preferences?: PartyPreferences;
48
47
  private _invitationManager?: InvitationFactory;
49
48
  private _protocol?: PartyProtocolFactory;
49
+ private _feedHints: PublicKey[] = []
50
50
 
51
51
  constructor (
52
52
  partyKey: PartyKey,
53
53
  modelFactory: ModelFactory,
54
54
  snapshotStore: SnapshotStore,
55
55
  private readonly _feedProvider: PartyFeedProvider,
56
+ private readonly _metadataStore: MetadataStore,
56
57
  private readonly _credentialsSigner: CredentialsSigner,
57
58
  // TODO(dmaretskyi): Pull this out to a higher level. Should preferences be part of client API instead?
58
59
  private readonly _profilePreferences: Preferences | undefined,
59
60
  private readonly _networkManager: NetworkManager,
60
- private readonly _hints: KeyHint[] = [],
61
- _initialTimeframe?: Timeframe,
62
- _options: PartyOptions = {}
61
+ private readonly _initialTimeframe?: Timeframe,
62
+ _options: PipelineOptions = {}
63
63
  ) {
64
- this._partyCore = new PartyCore(
64
+ this._partyCore = new PartyPipeline(
65
65
  partyKey,
66
66
  _feedProvider,
67
67
  modelFactory,
68
68
  snapshotStore,
69
69
  this._credentialsSigner.getIdentityKey().publicKey,
70
- _initialTimeframe,
71
70
  _options
72
71
  );
73
72
 
@@ -142,6 +141,13 @@ export class DataParty {
142
141
  await this._preferences?.setLastKnownTitle(title);
143
142
  }
144
143
 
144
+ /**
145
+ * @internal
146
+ */
147
+ _setFeedHints (feedHints: PublicKey[]) {
148
+ this._feedHints = feedHints;
149
+ }
150
+
145
151
  /**
146
152
  * Opens the pipeline and connects the streams.
147
153
  */
@@ -152,7 +158,20 @@ export class DataParty {
152
158
  return this;
153
159
  }
154
160
 
155
- await this._partyCore.open(this._hints);
161
+ // TODO(dmaretskyi): May be undefined in some tests.
162
+ const party = this._metadataStore.getParty(this._partyCore.key);
163
+
164
+ await this._partyCore.open({
165
+ feedHints: this._feedHints,
166
+ initialTimeframe: this._initialTimeframe,
167
+ targetTimeframe: party?.latestTimeframe
168
+ });
169
+
170
+ // Keep updating latest reached timeframe in the metadata.
171
+ // This timeframe will be waited for when opening the party next time.
172
+ this._partyCore.timeframeUpdate.on(timeframe => {
173
+ void this._metadataStore.setTimeframe(this._partyCore.key, timeframe);
174
+ });
156
175
 
157
176
  this._invitationManager = new InvitationFactory(
158
177
  this._partyCore.processor,
@@ -197,6 +216,9 @@ export class DataParty {
197
216
  return this;
198
217
  }
199
218
 
219
+ // Save the latest reached timeframe.
220
+ await this._metadataStore.setTimeframe(this._partyCore.key, this._partyCore.timeframe);
221
+
200
222
  await this._partyCore.close();
201
223
  await this._protocol?.stop();
202
224