@dxos/echo-db 2.33.9-dev.7d11f506 → 2.33.9-dev.9bbef4e2

package/src/packlets/database/link.ts

@@ -2,7 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
+import assert from 'node:assert';
 
 import { ItemID, ItemType } from '@dxos/echo-protocol';
 import { Model, StateManager } from '@dxos/model-factory';

package/src/packlets/database/selection/result.ts

@@ -2,7 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
+import assert from 'node:assert';
 
 import { Event } from '@dxos/async';
 

package/src/parties/data-party.test.ts

@@ -5,7 +5,9 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
+import {
+  createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec
+} from '@dxos/credentials';
 import { codec } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';

package/src/parties/data-party.ts

@@ -2,7 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
+import assert from 'node:assert';
 
 import { synchronized, Event } from '@dxos/async';
 import { timed } from '@dxos/debug';
@@ -30,7 +30,7 @@ export const PARTY_TITLE_PROPERTY = 'title'; // TODO(burdon): Remove (should not
 
 // TODO(burdon): Factor out public API.
 export interface PartyMember {
-  publicKey: PublicKey,
+  publicKey: PublicKey
   displayName?: string
 }
 
@@ -142,7 +142,7 @@ export class DataParty {
     await this._preferences?.setLastKnownTitle(title);
   }
 
-  get genesisFeedKey () {
+  get genesisFeedKey (): PublicKey {
     assert(this._genesisFeedKey);
     return this._genesisFeedKey;
   }

package/src/parties/party-factory.ts

@@ -2,8 +2,8 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import {
   createEnvelopeMessage,
@@ -19,7 +19,7 @@ import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
 import { PublicKey, Timeframe } from '@dxos/protocols';
-import { humanize } from '@dxos/util';
+import { humanize, Provider } from '@dxos/util';
 
 import {
   createDataPartyAdmissionMessages,
@@ -27,7 +27,7 @@ import {
 } from '../invitations';
 import { IdentityNotInitializedError } from '../packlets/errors';
 import { MetadataStore, PartyFeedProvider, PipelineOptions } from '../pipeline';
-import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
+import { IdentityCredentials } from '../protocol/identity-credentials';
 import { SnapshotStore } from '../snapshots';
 import { DataParty, PARTY_ITEM_TYPE } from './data-party';
 
@@ -38,7 +38,7 @@ const log = debug('dxos:echo-db:party-factory');
  */
 export class PartyFactory {
   constructor (
-    private readonly _identityProvider: IdentityCredentialsProvider,
+    private readonly _identityProvider: Provider<IdentityCredentials | undefined>,
     private readonly _networkManager: NetworkManager,
     private readonly _modelFactory: ModelFactory,
     private readonly _snapshotStore: SnapshotStore,

package/src/parties/party-manager.test.ts

@@ -4,10 +4,10 @@
 
 /* eslint-disable jest/no-conditional-expect */
 
-import assert from 'assert';
 import debug from 'debug';
 import expect from 'expect';
 import { it as test } from 'mocha';
+import assert from 'node:assert';
 
 import { latch } from '@dxos/async';
 import {

package/src/parties/party-manager.ts

@@ -2,20 +2,20 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
 import unionWith from 'lodash.unionwith';
+import assert from 'node:assert';
 
 import { Event, synchronized } from '@dxos/async';
 import { SecretProvider } from '@dxos/credentials';
 import { failUndefined, timed } from '@dxos/debug';
 import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
 import { PublicKey } from '@dxos/protocols';
-import { ComplexMap, boolGuard } from '@dxos/util';
+import { ComplexMap, boolGuard, Provider } from '@dxos/util';
 
 import { InvitationDescriptor } from '../invitations';
 import { MetadataStore } from '../pipeline';
-import { IdentityCredentialsProvider } from '../protocol/identity-credentials';
+import { IdentityCredentials } from '../protocol/identity-credentials';
 import { SnapshotStore } from '../snapshots';
 import { DataParty, PARTY_ITEM_TYPE, PARTY_TITLE_PROPERTY } from './data-party';
 import { PartyFactory } from './party-factory';
@@ -25,9 +25,9 @@ export const CONTACT_DEBOUNCE_INTERVAL = 500;
 const log = debug('dxos:echo-db:party-manager');
 
 export interface OpenProgress {
-  haloOpened: boolean;
-  partiesOpened?: number;
-  totalParties?: number;
+  haloOpened: boolean
+  partiesOpened?: number
+  totalParties?: number
 }
 
 /**
@@ -42,16 +42,12 @@ export class PartyManager {
   // Map of parties by party key.
   private readonly _parties = new ComplexMap<PublicKey, DataParty>(key => key.toHex());
 
-  // Unsubscribe handlers.
-  // TODO(burdon): Never used.
-  private readonly _onCloseHandlers: (() => void)[] = [];
-
   private _open = false;
 
   constructor (
     private readonly _metadataStore: MetadataStore,
     private readonly _snapshotStore: SnapshotStore,
-    private readonly _identityProvider: IdentityCredentialsProvider,
+    private readonly _identityProvider: Provider<IdentityCredentials | undefined>,
     private readonly _partyFactory: PartyFactory
   ) {}
 
@@ -129,9 +125,6 @@ export class PartyManager {
     }
     this._open = false;
 
-    // Clean-up.
-    this._onCloseHandlers.forEach(callback => callback());
-
     // Close parties.
     for (const party of this._parties.values()) {
       if (party.isOpen) {

package/src/parties/party-preferences.ts

@@ -2,14 +2,14 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
+import assert from 'node:assert';
 
-import { PARTY_TITLE_PROPERTY, DataParty } from '.';
 import { Preferences } from '../halo/preferences';
+import { PARTY_TITLE_PROPERTY, DataParty } from './data-party';
 
 export interface ActivationOptions {
-  global?: boolean;
-  device?: boolean;
+  global?: boolean
+  device?: boolean
 }
 
 /**

package/src/pipeline/feed-muxer.test.ts

@@ -128,7 +128,8 @@ describe('FeedMuxer', () => {
     await pipeline.outboundEchoStream!.write({
       itemId: '123',
       genesis: {
-        itemType: 'foo'
+        itemType: 'foo',
+        modelType: 'bar'
       }
     });
 
@@ -138,7 +139,8 @@ describe('FeedMuxer', () => {
     expect((echoMessages[0] as any).data).toEqual({
       itemId: '123',
       genesis: {
-        itemType: 'foo'
+        itemType: 'foo',
+        modelType: 'bar'
       }
     });
   });

package/src/pipeline/feed-muxer.ts

@@ -2,8 +2,8 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { Event } from '@dxos/async';
 import { Message as HaloMessage } from '@dxos/credentials';

package/src/pipeline/message-selector.ts

@@ -2,8 +2,8 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { MessageSelector } from '@dxos/echo-protocol';
 

package/src/pipeline/metadata-store.ts

@@ -2,8 +2,8 @@
 // Copyright 2021 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { synchronized } from '@dxos/async';
 import { failUndefined } from '@dxos/debug';
@@ -25,13 +25,15 @@ export interface AddPartyOptions {
   genesisFeed: PublicKey
 }
 
+const emptyEchoMetadata = (): EchoMetadata => ({
+  version: STORAGE_VERSION,
+  parties: [],
+  created: new Date(),
+  updated: new Date()
+});
+
 export class MetadataStore {
-  private _metadata: EchoMetadata = {
-    version: STORAGE_VERSION,
-    parties: [],
-    created: new Date(),
-    updated: new Date()
-  };
+  private _metadata: EchoMetadata = emptyEchoMetadata();
 
   constructor (
     private readonly _directory: Directory
@@ -56,19 +58,26 @@ export class MetadataStore {
   async load (): Promise<void> {
     const file = this._directory.createOrOpen('EchoMetadata');
     try {
-      const { size } = await file.stat();
-      if (size === 0) {
+      const { size: fileLength } = await file.stat();
+      if (fileLength < 4) {
         return;
       }
+      // Loading file size from first 4 bytes.
+      const dataSize = fromBytesInt32(await file.read(0, 4));
+      log(`Load: data size ${dataSize}`);
+
+      // Sanity check.
+      {
+        if (fileLength < dataSize + 4) {
+          throw new Error('Metadata storage is corrupted');
+        }
+      }
 
-      const data = await file.read(0, size);
+      const data = await file.read(4, dataSize);
       this._metadata = schema.getCodecForType('dxos.echo.metadata.EchoMetadata').decode(data);
     } catch (err: any) {
-      if (err.code === 'ENOENT') {
-        return;
-      } else {
-        throw err;
-      }
+      log(`Error loading metadata: ${err}`);
+      this._metadata = emptyEchoMetadata();
     } finally {
       await file.close();
     }
@@ -87,22 +96,14 @@ export class MetadataStore {
 
     try {
       const encoded = Buffer.from(schema.getCodecForType('dxos.echo.metadata.EchoMetadata').encode(data));
-      await file.write(0, encoded);
 
-      // Truncate the rest of the file.
-      {
-        const { size } = await file.stat();
-        if (size > encoded.length) {
-          await file.truncate(encoded.length, size);
-        }
-      }
+      // Saving file size at first 4 bytes.
+      log(`Save: data size ${encoded.length}`);
+      await file.write(0, toBytesInt32(encoded.length));
+
+      // Saving data.
+      await file.write(4, encoded);
 
-      // Sanity check.
-      const { size } = await file.stat();
-      if (size !== encoded.length) {
-        console.log('SANITY!');
-      }
-      assert(size === encoded.length);
     } finally {
       await file.close();
     }
@@ -196,3 +197,11 @@ export class MetadataStore {
     await this._save();
   }
 }
+
+const toBytesInt32 = (num: number) => {
+  const buf = Buffer.alloc(4);
+  buf.writeInt32LE(num, 0);
+  return buf;
+};
+
+const fromBytesInt32 = (buf: Buffer) => buf.readInt32LE(0);

package/src/pipeline/party-feed-provider.ts

@@ -2,8 +2,8 @@
 // Copyright 2021 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { Event, synchronized } from '@dxos/async';
 import { Keyring, KeyType } from '@dxos/credentials';

package/src/pipeline/party-pipeline.test.ts

@@ -18,9 +18,10 @@ import { PublicKey, Timeframe } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 
-import { MetadataStore, PartyFeedProvider } from '.';
-import { createReplicatorPlugin } from '../protocol/replicator-plugin';
+import { createReplicatorPlugin } from '../protocol';
 import { SnapshotStore } from '../snapshots';
+import { MetadataStore } from './metadata-store';
+import { PartyFeedProvider } from './party-feed-provider';
 import { PartyPipeline } from './party-pipeline';
 
 describe('PartyPipeline', () => {

package/src/pipeline/party-pipeline.ts

@@ -2,7 +2,7 @@
 // Copyright 2021 DXOS.org
 //
 
-import assert from 'assert';
+import assert from 'node:assert';
 
 import { synchronized } from '@dxos/async';
 import { KeyType, Message as HaloMessage } from '@dxos/credentials';
@@ -14,20 +14,23 @@ import { ModelFactory } from '@dxos/model-factory';
 import { PublicKey, Timeframe } from '@dxos/protocols';
 import { SubscriptionGroup } from '@dxos/util';
 
-import { createMessageSelector, PartyProcessor, PartyFeedProvider, FeedMuxer } from '.';
 import { Database, FeedDatabaseBackend, TimeframeClock } from '../packlets/database';
 import { createAutomaticSnapshots, SnapshotStore } from '../snapshots';
+import { FeedMuxer } from './feed-muxer';
+import { createMessageSelector } from './message-selector';
+import { PartyFeedProvider } from './party-feed-provider';
+import { PartyProcessor } from './party-processor';
 
 const DEFAULT_SNAPSHOT_INTERVAL = 100; // Every 100 messages.
 
 export interface PipelineOptions {
-  readLogger?: (msg: any) => void;
-  writeLogger?: (msg: any) => void;
-  readOnly?: boolean;
+  readLogger?: (msg: any) => void
+  writeLogger?: (msg: any) => void
+  readOnly?: boolean
   // TODO(burdon): Hierarchical options.
   // snapshots: { enabled: true, interval: 100 } }
-  snapshots?: boolean;
-  snapshotInterval?: number;
+  snapshots?: boolean
+  snapshotInterval?: number
 }
 
 export interface OpenOptions {

package/src/pipeline/party-processor.ts

@@ -2,8 +2,8 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { Event } from '@dxos/async';
 import {

package/src/protocol/authenticator.test.ts

@@ -5,21 +5,18 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import { createAuthMessage, createKeyAdmitMessage, createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
+import { createAuthMessage, createEnvelopeMessage, createFeedAdmitMessage, createKeyAdmitMessage, createPartyGenesisMessage, Keyring, KeyType, wrapMessage } from '@dxos/credentials';
 
 import { PartyProcessor } from '../pipeline';
 import { createAuthenticator } from './authenticator';
-import { CredentialsSigner } from './credentials-signer';
+import { createTestIdentityCredentials } from './identity-credentials';
 
 describe('authenticator', () => {
-  // TODO(dmaretskyi): Figure out how credentials work and if this test makes sense.
-  test.skip('authenticates admitted peer', async () => {
+  test('authenticates party creator', async () => {
     const keyring = new Keyring();
+    const identity = await createTestIdentityCredentials(keyring);
     const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
-    const identityKey = await keyring.createKeyRecord({ type: KeyType.IDENTITY });
-    const deviceKey = await keyring.createKeyRecord({ type: KeyType.DEVICE });
-    const feedKey = await keyring.createKeyRecord({ type: KeyType.FEED });
-    const signer = CredentialsSigner.createDirectDeviceSigner(keyring);
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
 
     const partyProcessor = new PartyProcessor(partyKey.publicKey);
     await partyProcessor.processMessage({
@@ -27,16 +24,106 @@ describe('authenticator', () => {
         keyring,
         partyKey,
         feedKey.publicKey,
-        identityKey
+        partyKey
       ),
       meta: {} as any
     });
+    await partyProcessor.processMessage({
+      data: createEnvelopeMessage(
+        identity.keyring,
+        partyKey.publicKey,
+        wrapMessage(identity.identityGenesis),
+        [partyKey]
+      ),
+      meta: {} as any
+    });
+    await partyProcessor.processMessage({
+      data: createFeedAdmitMessage(
+        keyring,
+        partyKey.publicKey,
+        feedKey.publicKey,
+        [identity.deviceKeyChain]
+      ),
+      meta: {} as any
+    });
+
+    const authenticator = createAuthenticator(partyProcessor, identity.createCredentialsSigner(), null as any);
+
+    //
+    // This test follows the same party creation routing as party factory.
+    // Oddly, it does not admit the device key to the party.
+    // This means that authentication is actually done using the signature created using the feed key.
+    //
+
+    // Does not authenticate without the feed key.
+    {
+      const credential = createAuthMessage(
+        keyring,
+        partyKey.publicKey,
+        identity.identityKey,
+        identity.deviceKey
+      );
+      expect(await authenticator.authenticate(credential.payload)).toEqual(false);
+    }
+
+    // Does authenticate with the feed key.
+    {
+      const credential = createAuthMessage(
+        keyring,
+        partyKey.publicKey,
+        identity.identityKey,
+        identity.deviceKey,
+        feedKey.publicKey
+      );
+      expect(await authenticator.authenticate(credential.payload)).toEqual(true);
+    }
+  });
+
+  test('authenticates another identity', async () => {
+    const keyring = new Keyring();
+    const identity = await createTestIdentityCredentials(keyring);
+    const partyKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+    const feedKey = await keyring.createKeyRecord({ type: KeyType.PARTY });
+
+    const partyProcessor = new PartyProcessor(partyKey.publicKey);
+    await partyProcessor.processMessage({
+      data: createPartyGenesisMessage(
+        keyring,
+        partyKey,
+        feedKey.publicKey,
+        partyKey
+      ),
+      meta: {} as any
+    });
+    await partyProcessor.processMessage({
+      data: createEnvelopeMessage(
+        identity.keyring,
+        partyKey.publicKey,
+        wrapMessage(identity.identityGenesis),
+        [partyKey]
+      ),
+      meta: {} as any
+    });
+    await partyProcessor.processMessage({
+      data: createFeedAdmitMessage(
+        keyring,
+        partyKey.publicKey,
+        feedKey.publicKey,
+        [identity.deviceKeyChain]
+      ),
+      meta: {} as any
+    });
+
+    const authenticator = createAuthenticator(partyProcessor, identity.createCredentialsSigner(), null as any);
+
+    const identity2 = await createTestIdentityCredentials(keyring);
+
     await partyProcessor.processMessage({
       data: createKeyAdmitMessage(
         keyring,
         partyKey.publicKey,
-        identityKey,
-        [partyKey, identityKey]
+        identity2.identityKey,
+        [identity.deviceKeyChain]
       ),
       meta: {} as any
     });
@@ -44,20 +131,18 @@ describe('authenticator', () => {
       data: createKeyAdmitMessage(
         keyring,
         partyKey.publicKey,
-        deviceKey,
-        [identityKey, deviceKey]
+        identity2.deviceKey,
+        [identity.deviceKeyChain]
       ),
       meta: {} as any
     });
 
-    const authenticator = createAuthenticator(partyProcessor, signer, null as any);
     const credential = createAuthMessage(
       keyring,
       partyKey.publicKey,
-      identityKey,
-      deviceKey
+      identity2.identityKey,
+      identity2.deviceKey
     );
-
     expect(await authenticator.authenticate(credential.payload)).toEqual(true);
   });
 });

package/src/protocol/identity-credentials.ts

@@ -27,8 +27,6 @@ export interface IdentityCredentials {
   contacts: ContactManager | undefined
 }
 
-export type IdentityCredentialsProvider = () => IdentityCredentials | undefined
-
 export const createTestIdentityCredentials = async (keyring: Keyring): Promise<IdentityCredentials> => {
   const identityKey = await keyring.createKeyRecord({ type: KeyType.IDENTITY });
   const deviceKey = await keyring.createKeyRecord({ type: KeyType.DEVICE });

package/src/protocol/index.ts

@@ -8,3 +8,4 @@ export * from './auth-plugin';
 export * from './halo-recovery-plugin';
 export * from './offline-invitation-plugin';
 export * from './party-protocol-factory';
+export * from './replicator-plugin';

package/src/protocol/party-protocol-factory.ts

@@ -11,7 +11,7 @@ import { MMSTTopology, NetworkManager, Plugin } from '@dxos/network-manager';
 import { PresencePlugin } from '@dxos/protocol-plugin-presence';
 import { PublicKey } from '@dxos/protocols';
 
-import { CredentialsProvider } from '.';
+import { CredentialsProvider } from './authenticator';
 
 const log = debug('dxos:echo-db:party-protocol-factory');
 

package/src/snapshots/snapshot-store.test.ts

@@ -26,10 +26,14 @@ describe('SnapshotStore', () => {
 
     const snapshot: PartySnapshot = {
       partyKey: key1.asBuffer(),
+      halo: {
+        messages: []
+      },
       database: {
         items: [{
           itemId: createId(),
-          itemType: 'example:test'
+          itemType: 'example:test',
+          modelType: 'example:model'
         }],
         links: []
       }

package/src/snapshots/snapshot-store.ts

@@ -2,8 +2,8 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
 import debug from 'debug';
+import assert from 'node:assert';
 
 import { schema, PartyKey, PartySnapshot } from '@dxos/echo-protocol';
 import { PublicKey } from '@dxos/protocols';

package/src/testing/testing-factories.ts

@@ -2,7 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import assert from 'assert';
+import assert from 'node:assert';
 
 import { Model } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';