@dxos/feed-store 2.12.20 → 2.13.0

package/src/feed-store.test.ts

@@ -4,6 +4,7 @@
 
 /* eslint-disable jest/no-done-callback */
 
+import assert from 'assert';
 import hypercore from 'hypercore';
 import hypertrie from 'hypertrie';
 import pify from 'pify';
@@ -13,6 +14,7 @@ import { sleep } from '@dxos/async';
 import { PublicKey, createKeyPair } from '@dxos/crypto';
 import { IStorage, STORAGE_NODE, STORAGE_RAM, createStorage } from '@dxos/random-access-multi-storage';
 
+import { FeedDescriptor } from './feed-descriptor';
 import { FeedStore } from './feed-store';
 import { HypercoreFeed } from './hypercore-types';
 
@@ -23,9 +25,8 @@ interface KeyPair {
 
 const feedNames = ['booksFeed', 'usersFeed', 'groupsFeed'];
 
-const createFeedStore = async (storage: IStorage, options = {}) => {
+const createFeedStore = (storage: IStorage, options = {}) => {
   const feedStore = new FeedStore(storage, options);
-  await feedStore.open();
   return feedStore;
 };
 
@@ -34,23 +35,21 @@ async function createDefault () {
 
   return {
     directory,
-    feedStore: await createFeedStore(createStorage(directory, STORAGE_NODE), { valueEncoding: 'utf-8' })
+    feedStore: createFeedStore(createStorage(directory, STORAGE_NODE), { valueEncoding: 'utf-8' })
   };
 }
 
-async function defaultFeeds (feedStore: FeedStore, keys: Record<string, KeyPair>) : Promise<Record<string, HypercoreFeed>> {
+async function defaultFeeds (feedStore: FeedStore, keys: Record<string, KeyPair>) : Promise<Record<string, FeedDescriptor>> {
   return Object.fromEntries(await Promise.all(Object.entries<KeyPair>(keys).map(async ([feed, keyPair]) =>
-    feed === 'booksFeed'
-      ? [feed, await feedStore.createReadWriteFeed({ key: keyPair.key, secretKey: keyPair.secretKey, metadata: { topic: 'books' } })]
-      : [feed, await feedStore.createReadWriteFeed({ key: keyPair.key, secretKey: keyPair.secretKey })]
+    [feed, await feedStore.openReadWriteFeed(keyPair.key, keyPair.secretKey)]
   )));
 }
 
-function append (feed: any, message: any) {
+function append (feed: HypercoreFeed, message: any) {
   return pify(feed.append.bind(feed))(message);
 }
 
-function head (feed: any) {
+function head (feed: HypercoreFeed) {
   return pify(feed.head.bind(feed))();
 }
 
@@ -67,13 +66,9 @@ describe('FeedStore', () => {
   test('Config default', async () => {
     const feedStore = await createFeedStore(createStorage('', STORAGE_RAM));
     expect(feedStore).toBeInstanceOf(FeedStore);
-    expect(feedStore.opened).toBeTruthy();
 
     const feedStore2 = new FeedStore(createStorage('', STORAGE_RAM));
     expect(feedStore2).toBeInstanceOf(FeedStore);
-    expect(feedStore2.opened).toBeFalsy();
-    await feedStore2.open();
-    expect(feedStore2.opened).toBeTruthy();
   });
 
   test('Config default + custom database + custom hypercore', async () => {
@@ -85,140 +80,99 @@ describe('FeedStore', () => {
     const database = hypertrie(storage.createOrOpen.bind(storage), { valueEncoding: 'json' });
     database.list = jest.fn((_, cb) => cb(null, []));
 
-    const feedStore = await createFeedStore(createStorage('', STORAGE_RAM), {
+    const feedStore = createFeedStore(createStorage('', STORAGE_RAM), {
       hypercore: customHypercore
     });
 
     expect(feedStore).toBeInstanceOf(FeedStore);
 
-    await feedStore.createReadOnlyFeed({ key: PublicKey.random() });
+    await feedStore.openReadOnlyFeed(PublicKey.random());
 
     expect(customHypercore.mock.calls.length).toBe(1);
   });
 
   test('Create feed', async () => {
     const { feedStore } = await createDefault();
-    const { booksFeed } = await defaultFeeds(feedStore, keys);
+    const { booksFeed: { feed: booksFeed } } = await defaultFeeds(feedStore, keys);
 
     expect(booksFeed).toBeInstanceOf(hypercore);
 
-    const booksFeedDescriptor = feedStore.getDescriptors().find(fd => fd.key.equals(booksFeed.key));
+    const booksFeedDescriptor = await feedStore.openReadWriteFeed(PublicKey.from(booksFeed.key), booksFeed.secretKey);
     expect(booksFeedDescriptor).toHaveProperty('key', PublicKey.from(booksFeed.key));
-    expect(booksFeedDescriptor?.metadata).toHaveProperty('topic', 'books');
 
     await append(booksFeed, 'Foundation and Empire');
     await expect(head(booksFeed)).resolves.toBe('Foundation and Empire');
 
     // It should return the same opened instance.
-    await expect(feedStore.openFeed(PublicKey.from(booksFeed.key))).resolves.toBe(booksFeed);
+    await expect(feedStore.openReadOnlyFeed(PublicKey.from(booksFeed.key))).resolves.toBe(booksFeedDescriptor);
   });
 
   test('Create duplicate feed', async () => {
     const { feedStore } = await createDefault();
 
-    const fds = await feedStore.createReadWriteFeed({ key: keys.usersFeed.key, secretKey: keys.usersFeed.secretKey });
+    const { feed: fds } = await feedStore.openReadWriteFeed(keys.usersFeed.key, keys.usersFeed.secretKey);
+    assert(fds.secretKey);
 
     const [usersFeed, feed2] = await Promise.all([
-      feedStore.openFeed(PublicKey.from(fds.key)),
-      feedStore.openFeed(PublicKey.from(fds.key))
+      feedStore.openReadWriteFeed(PublicKey.from(fds.key), fds.secretKey),
+      feedStore.openReadWriteFeed(PublicKey.from(fds.key), fds.secretKey)
     ]);
-    expect(usersFeed).toBe(feed2);
+    expect(usersFeed.feed).toBe(feed2.feed);
 
-    await append(usersFeed, 'alice');
-    await expect(head(usersFeed)).resolves.toBe('alice');
+    await append(usersFeed.feed, 'alice');
+    await expect(head(usersFeed.feed)).resolves.toBe('alice');
   });
 
   test('Create and close a feed', async () => {
     const { feedStore } = await createDefault();
     const publicKey = PublicKey.random();
 
-    await expect(feedStore.closeFeed(publicKey)).rejects.toThrow(/Feed not found/);
-
-    const foo = await feedStore.createReadOnlyFeed({ key: publicKey });
+    const foo = await feedStore.openReadOnlyFeed(publicKey);
     expect(foo.opened).toBeTruthy();
-    expect(foo.closed).toBeFalsy();
 
-    await feedStore.closeFeed(publicKey);
-    expect(foo.closed).toBeTruthy();
+    await feedStore.close();
+    expect(foo.opened).toBeFalsy();
   });
 
   test('Descriptors', async () => {
     const { feedStore } = await createDefault();
-    const { booksFeed } = await defaultFeeds(feedStore, keys);
+    await defaultFeeds(feedStore, keys);
 
-    expect(feedStore.getDescriptors().map(fd => fd.key)).toEqual(Object.entries(keys).map(([, keyPair]) => keyPair.key));
-    expect(feedStore.getDescriptorByDiscoveryKey(booksFeed.discoveryKey)?.key).toEqual(keys.booksFeed.key);
+    expect(Array.from((feedStore as any)._descriptors.values()).map((fd: any) => fd.key))
+      .toEqual(Object.entries(keys).map(([, keyPair]) => keyPair.key)
+      );
   });
 
   test('Feeds', async () => {
     const { feedStore } = await createDefault();
     const { booksFeed, usersFeed, groupsFeed } = await defaultFeeds(feedStore, keys);
 
-    expect(feedStore.getOpenFeeds().map(f => f.key)).toEqual([booksFeed.key, usersFeed.key, groupsFeed.key]);
-    expect(feedStore.getOpenFeed(fd => fd.key.equals(booksFeed.key))).toBe(booksFeed);
-    expect(feedStore.getOpenFeed(() => false)).toBeUndefined();
-    expect(feedStore.getOpenFeeds(fd => fd.key.equals(keys.booksFeed.key))).toEqual([booksFeed]);
-  });
-
-  test('Close/Load feed', async () => {
-    const { feedStore } = await createDefault();
-    const { booksFeed } = await defaultFeeds(feedStore, keys);
-
-    await feedStore.closeFeed(keys.booksFeed.key);
-    expect(feedStore.getDescriptors().find(fd => fd.key.equals(keys.booksFeed.key))).toHaveProperty('opened', false);
-
-    const [feed] = await feedStore.openFeeds(fd => fd.key.equals(keys.booksFeed.key));
-    expect(feed).toBeDefined();
-    expect(feed.key).toEqual(booksFeed.key);
-    expect(feedStore.getDescriptors().find(fd => fd.key.equals(keys.booksFeed.key))).toHaveProperty('opened', true);
+    expect(Array.from((feedStore as any)._descriptors.values()).map((fd: any) => fd.key))
+      .toEqual([booksFeed.key, usersFeed.key, groupsFeed.key]);
   });
 
   test('Close feedStore and their feeds', async () => {
     const { feedStore } = await createDefault();
     await defaultFeeds(feedStore, keys);
 
-    expect(feedStore.opened).toBe(true);
-    expect(feedStore.closed).toBe(false);
-    expect(feedStore.getDescriptors().filter(fd => fd.opened).length).toBe(3);
-
+    expect(Array.from((feedStore as any)._descriptors.values()).map((fd: any) => fd.key).length).toBe(3);
     await feedStore.close();
-    expect(feedStore.getDescriptors().filter(fd => fd.opened).length).toBe(0);
-    expect(feedStore.opened).toBe(false);
-    expect(feedStore.closed).toBe(true);
+    expect(Array.from((feedStore as any)._descriptors.values()).map((fd: any) => fd.key).length).toBe(0);
   });
 
   test('Default codec: binary', async () => {
-    const feedStore = await createFeedStore(createStorage('', STORAGE_RAM));
+    const feedStore = createFeedStore(createStorage('', STORAGE_RAM));
     expect(feedStore).toBeInstanceOf(FeedStore);
 
     const { publicKey, secretKey } = createKeyPair();
-    const feed = await feedStore.createReadWriteFeed({ key: PublicKey.from(publicKey), secretKey });
+    const { feed } = await feedStore.openReadWriteFeed(PublicKey.from(publicKey), secretKey);
     expect(feed).toBeInstanceOf(hypercore);
     await append(feed, 'test');
     await expect(head(feed)).resolves.toBeInstanceOf(Buffer);
   });
 
-  test('on open error should unlock the descriptor', async () => {
-    const feedStore = await createFeedStore(createStorage('', STORAGE_RAM), {
-      hypercore: () => {
-        throw new Error('open error');
-      }
-    });
-
-    const publicKey = PublicKey.random();
-    await expect(feedStore.createReadOnlyFeed({ key: publicKey })).rejects.toThrow(/open error/);
-
-    const fd = feedStore.getDescriptors().find(fd => fd.key.equals(publicKey));
-
-    if (!fd) {
-      throw new Error('Descriptor not found');
-    }
-
-    await expect(fd.lock.executeSynchronized(async () => 'Unlocked')).resolves.toBe('Unlocked');
-  });
-
   test('on close error should unlock the descriptor', async () => {
-    const feedStore = await createFeedStore(createStorage('', STORAGE_RAM), {
+    const feedStore = createFeedStore(createStorage('', STORAGE_RAM), {
       hypercore: () => ({
         opened: true,
         ready (cb: () => void) {
@@ -232,38 +186,9 @@ describe('FeedStore', () => {
     });
 
     const publicKey = PublicKey.random();
-    await feedStore.createReadOnlyFeed({ key: publicKey });
-    const fd = feedStore.getDescriptors().find(fd => fd.key.equals(publicKey));
-
-    if (!fd) {
-      throw new Error('Descriptor not found');
-    }
+    await feedStore.openReadOnlyFeed(publicKey);
 
-    await expect(feedStore.closeFeed(publicKey)).rejects.toThrow(/close error/);
     await expect(feedStore.close()).rejects.toThrow(/close error/);
-
-    await expect(fd.lock.executeSynchronized(async () => 'Unlocked')).resolves.toBe('Unlocked');
-  });
-
-  test('append event', async (done) => {
-    const feedStore = await createFeedStore(createStorage('', STORAGE_RAM));
-    const { publicKey, secretKey } = createKeyPair();
-    const feed = await feedStore.createReadWriteFeed({ key: PublicKey.from(publicKey), secretKey });
-
-    feedStore.appendEvent.on((descriptor) => {
-      expect(descriptor.feed).toBe(feed);
-      done();
-    });
-
-    feed.append('test');
-  });
-
-  test('openFeed should wait until FeedStore is ready', async () => {
-    const feedStore = new FeedStore(createStorage('', STORAGE_RAM));
-    await feedStore.open();
-    const publicKey = PublicKey.random();
-    const feed = await feedStore.createReadOnlyFeed({ key: publicKey });
-    expect(feed).toBeDefined();
   });
 
   test('feed event does not get called twice', async () => {
@@ -275,11 +200,11 @@ describe('FeedStore', () => {
     });
 
     const key = PublicKey.random();
-    await feedStore.createReadOnlyFeed({ key });
+    await feedStore.openReadOnlyFeed(key);
 
-    await feedStore.openFeed(key);
-    await feedStore.openFeed(key);
-    await feedStore.openFeed(key);
+    await feedStore.openReadOnlyFeed(key);
+    await feedStore.openReadOnlyFeed(key);
+    await feedStore.openReadOnlyFeed(key);
 
     await sleep(20); // To flush events
 

package/src/feed-store.ts

@@ -6,30 +6,25 @@ import assert from 'assert';
 import defaultHypercore from 'hypercore';
 
 import { synchronized, Event } from '@dxos/async';
-import { PublicKey, PublicKeyLike } from '@dxos/crypto';
+import { PublicKey } from '@dxos/crypto';
 import { IStorage } from '@dxos/random-access-multi-storage';
 
 import FeedDescriptor from './feed-descriptor';
-import type { HypercoreFeed, Hypercore } from './hypercore-types';
+import type { Hypercore } from './hypercore-types';
 import type { ValueEncoding } from './types';
 
-type DescriptorCallback = (descriptor: FeedDescriptor) => boolean;
-
 export interface CreateDescriptorOptions {
   key: PublicKey,
-  secretKey?: Buffer,
-  metadata?: any
+  secretKey?: Buffer
 }
 
 export interface CreateReadWriteFeedOptions {
   key: PublicKey,
   secretKey: Buffer
-  metadata?: any
 }
 
 export interface CreateReadOnlyFeedOptions {
   key: PublicKey
-  metadata?: any
 }
 
 export interface FeedStoreOptions {
@@ -54,12 +49,6 @@ export class FeedStore {
   private _valueEncoding: ValueEncoding | undefined;
   private _hypercore: Hypercore;
   private _descriptors: Map<string, FeedDescriptor>;
-  private _open: boolean;
-
-  /**
-   * Is emitted when data gets appended to one of the FeedStore's feeds represented by FeedDescriptors.
-   */
-  readonly appendEvent = new Event<FeedDescriptor>();
 
   /**
    * Is emitted when a new feed represented by FeedDescriptor is opened.
@@ -84,16 +73,6 @@ export class FeedStore {
     this._hypercore = hypercore;
 
     this._descriptors = new Map();
-
-    this._open = false;
-  }
-
-  get opened () {
-    return this._open;
-  }
-
-  get closed () {
-    return !this._open;
   }
 
   /**
@@ -103,187 +82,52 @@ export class FeedStore {
     return this._storage;
   }
 
-  @synchronized
-  async open () {
-    if (this._open) {
-      return;
-    }
-    this._open = true;
-  }
-
   @synchronized
   async close () {
-    if (!this._open) {
-      return;
-    }
-    this._open = false;
-
-    await Promise.all(this
-      .getDescriptors()
-      .map(descriptor => descriptor.close())
-    );
-
+    await Promise.all(Array.from(this._descriptors.values()).map(descriptor => descriptor.close()));
     this._descriptors.clear();
   }
 
-  /**
-   * Get the list of descriptors.
-   */
-  getDescriptors () {
-    return Array.from(this._descriptors.values());
-  }
-
-  /**
-   * Get desciptor by its public key
-   */
-  getDescriptor (key: PublicKeyLike) {
-    return this.getDescriptors().find(descriptor => descriptor.key.equals(key));
-  }
-
-  /**
-   * Fast access to a descriptor
-   */
-  getDescriptorByDiscoveryKey (discoverKey: Buffer) {
-    return this._descriptors.get(discoverKey.toString('hex'));
-  }
-
-  /**
-   * Get the list of opened feeds, with optional filter.
-   */
-  getOpenFeeds (callback?: DescriptorCallback): HypercoreFeed[] {
-    const notNull = <T>(value: T | null): value is T => Boolean(value);
-    return this.getDescriptors()
-      .filter(descriptor => descriptor.opened && (!callback || callback(descriptor)))
-      .map(descriptor => descriptor.feed)
-      .filter(notNull);
-  }
-
-  /**
-   * Find an opened feed using a filter callback.
-   */
-  getOpenFeed (callback: DescriptorCallback): HypercoreFeed | undefined {
-    const descriptor = this.getDescriptors()
-      .find(descriptor => descriptor.opened && callback(descriptor));
-
-    if (descriptor && descriptor.feed) {
-      return descriptor.feed;
-    }
-
-    return undefined;
-  }
-
-  /**
-   * Checks if feedstore has a feed with specified key.
-   */
-  hasFeed (key: PublicKeyLike): boolean {
-    return this.getDescriptors().some(fd => fd.key.equals(key));
-  }
-
-  /**
-   * Open multiple feeds using a filter callback.
-   */
-  @synchronized
-  async openFeeds (callback: DescriptorCallback): Promise<HypercoreFeed[]> {
-    assert(this._open, 'FeedStore closed');
-
-    const descriptors = this.getDescriptors()
-      .filter(descriptor => callback(descriptor));
-
-    return Promise.all(descriptors.map(descriptor => descriptor.open()));
-  }
-
-  /**
-   * Open a feed to FeedStore.
-   */
-  @synchronized
-  async openFeed (key: PublicKey): Promise<HypercoreFeed> {
-    assert(this._open, 'FeedStore closed');
-
-    const descriptor = this.getDescriptors().find(fd => fd.key.equals(key));
-
-    assert(descriptor, 'Descriptor not found');
-
-    const feed = await descriptor.open();
-
-    return feed;
-  }
-
   /**
    * Create a feed to Feedstore
    */
-  async createReadWriteFeed (options: CreateReadWriteFeedOptions): Promise<HypercoreFeed> {
-    this._createDescriptor(options);
-    return this.openFeed(options.key);
+  async openReadWriteFeed (key: PublicKey, secretKey: Buffer): Promise<FeedDescriptor> {
+    const descriptor = this._descriptors.get(key.toString());
+    if (descriptor && descriptor.secretKey) {
+      return descriptor;
+    }
+    return this._createDescriptor({ key, secretKey });
   }
 
   /**
    * Create a readonly feed to Feedstore
    */
-  async createReadOnlyFeed (options: CreateReadOnlyFeedOptions): Promise<HypercoreFeed> {
-    this._createDescriptor(options);
-    return this.openFeed(options.key);
-  }
-
-  /**
-   * Close a feed by the key.
-   */
-  @synchronized
-  async closeFeed (key: PublicKey) {
-    assert(this._open, 'FeedStore closed');
-
-    const descriptor = this.getDescriptors().find(fd => fd.key.equals(key));
-
-    if (!descriptor) {
-      throw new Error(`Feed not found: ${key.toString()}`);
-    }
-
-    await descriptor.close();
+  async openReadOnlyFeed (key: PublicKey): Promise<FeedDescriptor> {
+    const descriptor = this._descriptors.get(key.toString()) ?? await this._createDescriptor({ key });
+    return descriptor;
   }
 
   /**
    * Factory to create a new FeedDescriptor.
    */
-  private _createDescriptor (options: CreateDescriptorOptions) {
-    const { key, secretKey, metadata } = options;
-
-    const existing = this.getDescriptors().find(fd => fd.key.equals(key));
-    if (existing) {
-      return existing;
-    }
+  private async _createDescriptor (options: CreateDescriptorOptions) {
+    const { key, secretKey } = options;
 
     const descriptor = new FeedDescriptor({
       storage: this._storage,
       key,
       secretKey,
       valueEncoding: this._valueEncoding,
-      metadata,
       hypercore: this._hypercore
     });
 
     this._descriptors.set(
-      descriptor.discoveryKey.toString('hex'),
+      descriptor.key.toString(),
       descriptor
     );
 
-    const append = () => this.appendEvent.emit(descriptor);
-
-    descriptor.watch(async (event) => {
-      if (event === 'updated') {
-        return;
-      }
-
-      const { feed } = descriptor;
-
-      if (event === 'opened' && feed) {
-        feed.on('append', append);
-        this.feedOpenedEvent.emit(descriptor);
-        return;
-      }
-
-      if (event === 'closed' && feed) {
-        feed.removeListener('append', append);
-      }
-    });
+    await descriptor.open();
+    this.feedOpenedEvent.emit(descriptor);
 
     return descriptor;
   }