@dxos/echo-db 2.33.6 → 2.33.7-dev.40cf0994

package/src/packlets/errors/index.ts

@@ -1,43 +1,5 @@
 //
-// Copyright 2021 DXOS.org
+// Copyright 2022 DXOS.org
 //
 
-import { PublicKey } from '@dxos/crypto';
-import { DXOSError } from '@dxos/debug';
-import { ItemID } from '@dxos/echo-protocol';
-
-export class PartyNotFoundError extends DXOSError {
-  constructor (partyKey: PublicKey) {
-    super('PARTY_NOT_FOUND', `Party with key not found: ${partyKey}`);
-  }
-}
-
-export class EntityNotFoundError extends DXOSError {
-  constructor (entityId: ItemID) {
-    super('ENTITY_NOT_FOUND', `Entitiy not found: ${entityId}`);
-  }
-}
-
-export class UnknownModelError extends DXOSError {
-  constructor (model: string) {
-    super('UNKNOWN_MODEL', `Unknown model ${model}.`);
-  }
-}
-
-export class IdentityNotInitializedError extends DXOSError {
-  constructor () {
-    super('IDENTITY_NOT_INITIALIZED', 'Identity not initialized.');
-  }
-}
-
-export class InvalidInvitationError extends DXOSError {
-  constructor () {
-    super('INVALID_INVITATION', 'Invitation is invalid.');
-  }
-}
-
-export class InvalidStorageVersionError extends DXOSError {
-  constructor (expected: number, actual: number) {
-    super('INVALID_STORAGE_VERSION', `Invalid storage version: Expected ${expected}, actual ${actual}.`);
-  }
-}
+export * from './errors';

package/src/parties/data-party.test.ts

@@ -6,12 +6,12 @@ import expect from 'expect';
 import { it as test } from 'mocha';
 
 import { createKeyAdmitMessage, createPartyGenesisMessage, defaultSecretProvider, Keyring, KeyType, codec as haloCodec } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
 import { codec } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
+import { PublicKey } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 
 import { createDataPartyAdmissionMessages, defaultInvitationAuthenticator, GreetingInitiator } from '../invitations';

package/src/parties/data-party.ts

@@ -5,13 +5,13 @@
 import assert from 'assert';
 
 import { synchronized, Event } from '@dxos/async';
-import { PublicKey } from '@dxos/crypto';
 import { timed } from '@dxos/debug';
 import { PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
 import { FeedDescriptor } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
+import { PublicKey } from '@dxos/protocols';
 
 import { ResultSet } from '../api';
 import { ActivationOptions, PartyPreferences, Preferences } from '../halo';
@@ -81,7 +81,7 @@ export class DataParty {
       key: this.key.toHex(),
       isOpen: this.isOpen,
       isActive: this.isActive,
-      feedKeys: this._feedProvider.getFeeds().length,
+      feedKeys: this._feedProvider.getFeeds().length, // TODO(burdon): feeds.
       timeframe: this.isOpen ? this._partyCore.timeframe : undefined,
       properties: this.isOpen ? this.getPropertiesSet().expectOne().model.toObject() : undefined
     };

package/src/parties/party-factory.ts

@@ -13,12 +13,13 @@ import {
   SecretProvider,
   wrapMessage
 } from '@dxos/credentials';
-import { humanize, keyToString, PublicKey } from '@dxos/crypto';
+import { humanize, keyToString } from '@dxos/crypto';
 import { failUndefined, raise, timed } from '@dxos/debug';
 import { createFeedWriter, FeedMessage, PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
+import { PublicKey } from '@dxos/protocols';
 
 import {
   createDataPartyAdmissionMessages,

package/src/parties/party-manager.test.ts

@@ -18,7 +18,8 @@ import {
   SecretValidator
 } from '@dxos/credentials';
 import {
-  createKeyPair, PublicKey,
+  createKeyPair,
+  humanize,
   randomBytes,
   sign,
   SIGNATURE_LENGTH, verify
@@ -29,6 +30,7 @@ import { createWritableFeedStream, FeedStore } from '@dxos/feed-store';
 import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { ObjectModel } from '@dxos/object-model';
+import { PublicKey } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest, testTimeout } from '@dxos/testutils';
 
@@ -318,11 +320,11 @@ describe('Party manager', () => {
       expect(members.length).toBe(2);
       for (const member of members) {
         if (identityA.identityKey!.publicKey.equals(member.publicKey)) {
-          expect(member.displayName).toEqual(identityA.identityKey!.publicKey.humanize());
+          expect(member.displayName).toEqual(humanize(identityA.identityKey!.publicKey));
           expect(member.displayName).toEqual(identityA.displayName);
         }
         if (identityB.identityKey!.publicKey.equals(member.publicKey)) {
-          expect(member.displayName).toEqual(identityB.identityKey!.publicKey.humanize());
+          expect(member.displayName).toEqual(humanize(identityB.identityKey!.publicKey));
           expect(member.displayName).toEqual(identityB.displayName);
         }
       }
@@ -399,10 +401,10 @@ describe('Party manager', () => {
       expect(members.length).toBe(2);
       for (const member of members) {
         if (identityA.identityKey!.publicKey.equals(member.publicKey)) {
-          expect(member.displayName).toEqual(identityA.identityKey!.publicKey.humanize());
+          expect(member.displayName).toEqual(humanize(identityA.identityKey!.publicKey));
         }
         if (identityB.identityKey!.publicKey.equals(member.publicKey)) {
-          expect(member.displayName).toEqual(identityB.identityKey!.publicKey.humanize());
+          expect(member.displayName).toEqual(humanize(identityB.identityKey!.publicKey));
         }
       }
     }
@@ -456,11 +458,11 @@ describe('Party manager', () => {
       expect(members.length).toBe(2);
       for (const member of members) {
         if (identityA.identityKey!.publicKey.equals(member.publicKey)) {
-          expect(member.displayName).toEqual(identityA.identityKey!.publicKey.humanize());
+          expect(member.displayName).toEqual(humanize(identityA.identityKey!.publicKey));
           expect(member.displayName).toEqual(identityA.displayName);
         }
         if (identityB.identityKey!.publicKey.equals(member.publicKey)) {
-          expect(member.displayName).toEqual(identityB.identityKey!.publicKey.humanize());
+          expect(member.displayName).toEqual(humanize(identityB.identityKey!.publicKey));
           expect(member.displayName).toEqual(identityB.displayName);
         }
       }

package/src/parties/party-manager.ts

@@ -8,9 +8,9 @@ import unionWith from 'lodash.unionwith';
 
 import { Event, synchronized } from '@dxos/async';
 import { KeyHint, KeyType, SecretProvider } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
 import { failUndefined, timed } from '@dxos/debug';
 import { PartyKey, PartySnapshot } from '@dxos/echo-protocol';
+import { PublicKey } from '@dxos/protocols';
 import { ComplexMap, boolGuard } from '@dxos/util';
 
 import { InvitationDescriptor } from '../invitations';

package/src/pipeline/feed-muxer.test.ts

@@ -8,10 +8,11 @@ import { it as test } from 'mocha';
 
 import { waitForCondition, latch } from '@dxos/async';
 import { createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
-import { createId, createKeyPair, PublicKey } from '@dxos/crypto';
+import { createId, createKeyPair } from '@dxos/crypto';
 import { codec, createFeedWriter, FeedSelector, FeedStoreIterator, IEchoStream, Timeframe } from '@dxos/echo-protocol';
 import { FeedStore, createWritableFeedStream } from '@dxos/feed-store';
 import { createSetPropertyMutation } from '@dxos/model-factory';
+import { PublicKey } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { jsonReplacer } from '@dxos/util';
 

package/src/pipeline/feed-muxer.ts

@@ -7,11 +7,12 @@ import debug from 'debug';
 
 import { Event } from '@dxos/async';
 import { Message as HaloMessage } from '@dxos/credentials';
-import { keyToString, PublicKey } from '@dxos/crypto';
+import { keyToString } from '@dxos/crypto';
 import { checkType } from '@dxos/debug';
 import {
   createFeedMeta, EchoEnvelope, FeedMessage, FeedStoreIterator, FeedWriter, IEchoStream, mapFeedWriter, Timeframe
 } from '@dxos/echo-protocol';
+import { PublicKey } from '@dxos/protocols';
 import { jsonReplacer } from '@dxos/util';
 
 import { EchoProcessor, TimeframeClock } from '../packlets/database';

package/src/pipeline/message-selector.ts

@@ -6,8 +6,8 @@ import assert from 'assert';
 import debug from 'debug';
 
 import { getPartyCredentialMessageType, PartyCredential } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
 import { MessageSelector } from '@dxos/echo-protocol';
+import { PublicKey } from '@dxos/protocols';
 
 import { TimeframeClock } from '../packlets/database';
 import { PartyStateProvider } from './party-processor';

package/src/pipeline/metadata-store.test.ts

@@ -4,7 +4,7 @@
 
 import expect from 'expect';
 
-import { PublicKey } from '@dxos/crypto';
+import { PublicKey } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 
 import { MetadataStore } from './metadata-store';
@@ -76,4 +76,21 @@ describe('MetadataStore in-memory', () => {
     await store.clear();
     expect(store.parties?.length).toEqual(0);
   });
+
+  it('not corrupted', async () => {
+    const storage = createStorage('', StorageType.RAM);
+    const dir = storage.directory('metadata');
+    const metadataStore = new MetadataStore(dir);
+
+    // writing something in metadataStore to save.
+    await metadataStore.setDataFeed(PublicKey.random(), PublicKey.random());
+    await metadataStore.setDataFeed(PublicKey.random(), PublicKey.random());
+    await metadataStore.setDataFeed(PublicKey.random(), PublicKey.random());
+
+    // using same directory to test if truncates.
+    const metadataStore2 = new MetadataStore(dir);
+    // should owerride previous data.
+    await metadataStore2.setDataFeed(PublicKey.random(), PublicKey.random());
+    await metadataStore2.load();
+  });
 });

package/src/pipeline/metadata-store.ts

@@ -5,9 +5,10 @@
 import assert from 'assert';
 import debug from 'debug';
 
-import { PublicKey } from '@dxos/crypto';
+import { synchronized } from '@dxos/async';
 import { failUndefined } from '@dxos/debug';
 import { EchoMetadata, PartyMetadata, schema, Timeframe } from '@dxos/echo-protocol';
+import { PublicKey } from '@dxos/protocols';
 import { Directory } from '@dxos/random-access-multi-storage';
 
 /**
@@ -46,6 +47,7 @@ export class MetadataStore {
   /**
    * Loads metadata from persistent storage.
    */
+  @synchronized
   async load (): Promise<void> {
     const file = this._directory.createOrOpen('EchoMetadata');
     try {
@@ -67,6 +69,7 @@ export class MetadataStore {
     }
   }
 
+  @synchronized
   private async _save (): Promise<void> {
     const data: EchoMetadata = {
       ...this._metadata,
@@ -80,6 +83,21 @@ export class MetadataStore {
     try {
       const encoded = Buffer.from(schema.getCodecForType('dxos.echo.metadata.EchoMetadata').encode(data));
       await file.write(0, encoded);
+
+      // Truncate the rest of the file.
+      {
+        const { size } = await file.stat();
+        if (size > encoded.length) {
+          await file.truncate(encoded.length, size);
+        }
+      }
+
+      // Sanity check.
+      const { size } = await file.stat();
+      if (size !== encoded.length) {
+        console.log('SANITY!');
+      }
+      assert(size === encoded.length);
     } finally {
       await file.close();
     }

package/src/pipeline/party-feed-provider.ts

@@ -7,9 +7,9 @@ import debug from 'debug';
 
 import { Event, synchronized } from '@dxos/async';
 import { Keyring, KeyType } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
 import { FeedSelector, FeedStoreIterator, MessageSelector, Timeframe } from '@dxos/echo-protocol';
 import { FeedDescriptor, FeedStore } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/protocols';
 import { ComplexMap } from '@dxos/util';
 
 import { MetadataStore } from './metadata-store';

package/src/pipeline/party-pipeline.test.ts

@@ -7,13 +7,14 @@ import { it as test } from 'mocha';
 
 import { promiseTimeout } from '@dxos/async';
 import { createFeedAdmitMessage, createPartyGenesisMessage, Keyring, KeyType } from '@dxos/credentials';
-import { createId, PublicKey } from '@dxos/crypto';
+import { createId } from '@dxos/crypto';
 import { checkType } from '@dxos/debug';
 import { codec, FeedMessage, Timeframe } from '@dxos/echo-protocol';
 import { FeedStore } from '@dxos/feed-store';
 import { createTestProtocolPair } from '@dxos/mesh-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { ObjectModel } from '@dxos/object-model';
+import { PublicKey } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 import { afterTest } from '@dxos/testutils';
 

package/src/pipeline/party-pipeline.ts

@@ -6,10 +6,10 @@ import assert from 'assert';
 
 import { synchronized } from '@dxos/async';
 import { KeyType, Message as HaloMessage } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
 import { timed } from '@dxos/debug';
 import { createFeedWriter, DatabaseSnapshot, FeedSelector, FeedWriter, PartyKey, PartySnapshot, Timeframe } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
+import { PublicKey } from '@dxos/protocols';
 import { SubscriptionGroup } from '@dxos/util';
 
 import { createMessageSelector, PartyProcessor, PartyFeedProvider, FeedMuxer } from '.';

package/src/pipeline/party-processor.ts

@@ -15,8 +15,8 @@ import {
   PartyEventType,
   SignedMessage
 } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
 import { FeedKey, IHaloStream, PartyKey, HaloStateSnapshot } from '@dxos/echo-protocol';
+import { PublicKey } from '@dxos/protocols';
 import { jsonReplacer } from '@dxos/util';
 
 const log = debug('dxos:echo-db:party-processor');

package/src/protocol/auth-plugin.ts

@@ -3,8 +3,8 @@
 //
 
 import { Authenticator, AuthPlugin } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
 import { Replicator } from '@dxos/protocol-plugin-replicator';
+import { PublicKey } from '@dxos/protocols';
 
 /**
  * Creates authenticator network-protocol plugin that guards access to the replicator.

package/src/protocol/halo-recovery-plugin.ts

@@ -3,7 +3,7 @@
 //
 
 import { GreetingCommandPlugin } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
+import { PublicKey } from '@dxos/protocols';
 
 import { HaloRecoveryInitiator, InvitationFactory } from '../invitations';
 

package/src/protocol/identity-credentials.ts

@@ -3,6 +3,7 @@
 //
 
 import { createIdentityInfoMessage, createKeyAdmitMessage, createPartyGenesisMessage, KeyChain, KeyRecord, Keyring, KeyType, SignedMessage } from '@dxos/credentials';
+import { humanize } from '@dxos/crypto';
 
 import { ContactManager, Preferences } from '../halo';
 import { CredentialsSigner } from './credentials-signer';
@@ -38,7 +39,7 @@ export const createTestIdentityCredentials = async (keyring: Keyring): Promise<I
   messageMap.set(deviceKey.publicKey.toHex(), partyGenesis);
   const deviceKeyChain = Keyring.buildKeyChain(deviceKey.publicKey, messageMap, [feedKey.publicKey]);
 
-  const displayName = identityKey.publicKey.humanize();
+  const displayName = humanize(identityKey.publicKey);
   const identityInfo = createIdentityInfoMessage(keyring, displayName, identityKey);
 
   return {

package/src/protocol/offline-invitation-plugin.ts

@@ -3,7 +3,7 @@
 //
 
 import { GreetingCommandPlugin } from '@dxos/credentials';
-import { PublicKey } from '@dxos/crypto';
+import { PublicKey } from '@dxos/protocols';
 
 import { InvitationFactory, OfflineInvitationClaimer } from '../invitations';
 

package/src/protocol/party-protocol-factory.ts

@@ -4,11 +4,12 @@
 
 import debug from 'debug';
 
-import { discoveryKey, keyToString, PublicKey } from '@dxos/crypto';
+import { discoveryKey, keyToString } from '@dxos/crypto';
 import { PartyKey } from '@dxos/echo-protocol';
 import { Protocol } from '@dxos/mesh-protocol';
 import { MMSTTopology, NetworkManager, Plugin } from '@dxos/network-manager';
 import { PresencePlugin } from '@dxos/protocol-plugin-presence';
+import { PublicKey } from '@dxos/protocols';
 
 import { CredentialsProvider } from '.';
 

package/src/snapshots/snapshot-generator.ts

@@ -4,6 +4,8 @@
 
 import debug from 'debug';
 
+import { humanize } from '@dxos/crypto';
+
 import { TimeframeClock } from '../packlets/database';
 import { PartyPipeline } from '../pipeline';
 import { SnapshotStore } from './snapshot-store';
@@ -13,7 +15,7 @@ const log = debug('dxos:snapshot-generator');
 export const createAutomaticSnapshots = (party: PartyPipeline, clock: TimeframeClock, store: SnapshotStore, interval: number) => clock.update.on(async timeframe => {
   const totalMessages = timeframe.totalMessages();
   if (totalMessages > 0 && totalMessages % interval === 0) {
-    log(`Saving snapshot of ${party.key.humanize()}...`);
+    log(`Saving snapshot of ${humanize(party.key)}...`);
     try {
       const snapshot = party.createSnapshot();
       await store.save(snapshot);

package/src/snapshots/snapshot-store.test.ts

@@ -5,8 +5,9 @@
 import expect from 'expect';
 import { it as test } from 'mocha';
 
-import { createId, createKeyPair, PublicKey } from '@dxos/crypto';
+import { createId, createKeyPair } from '@dxos/crypto';
 import { PartySnapshot } from '@dxos/echo-protocol';
+import { PublicKey } from '@dxos/protocols';
 import { createStorage, StorageType } from '@dxos/random-access-multi-storage';
 
 import { SnapshotStore } from './snapshot-store';

package/src/snapshots/snapshot.test.ts

@@ -7,10 +7,10 @@ import expect from 'expect';
 import { it as test } from 'mocha';
 
 import { waitForCondition } from '@dxos/async';
-import { PublicKey } from '@dxos/crypto';
 import { schema, ItemID, MockFeedWriter, PartyKey } from '@dxos/echo-protocol';
 import { ModelFactory } from '@dxos/model-factory';
 import { ObjectModel, ValueUtil } from '@dxos/object-model';
+import { PublicKey } from '@dxos/protocols';
 
 import { ItemDemuxer, ItemManager } from '../packlets/database';
 import { createTestInstance } from '../testing';