@dxos/feed-store 2.33.5-dev.22471d71 → 2.33.5-dev.33d2877e

package/src/feed-descriptor.test.ts

@@ -20,7 +20,7 @@ describe('FeedDescriptor', () => {
   beforeEach(async () => {
     const { publicKey, secretKey } = createKeyPair();
     fd = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       hypercore: defaultHypercore
@@ -41,7 +41,7 @@ describe('FeedDescriptor', () => {
     // When this behaviour was changed, suddenly `protocol-plugin-replicator` tests started hanging forever on network generation.
     const { publicKey } = createKeyPair();
     const key = PublicKey.from(publicKey);
-    const fd = new FeedDescriptor({ key, storage: createStorage('', StorageType.NODE), hypercore: defaultHypercore });
+    const fd = new FeedDescriptor({ key, directory: createStorage('', StorageType.NODE).directory('feed'), hypercore: defaultHypercore });
     expect(fd.key).toEqual(key);
     expect(fd.secretKey).toBeUndefined();
   });
@@ -50,7 +50,7 @@ describe('FeedDescriptor', () => {
     const { publicKey, secretKey } = createKeyPair();
 
     const fd = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       valueEncoding: 'json',
@@ -92,7 +92,7 @@ describe('FeedDescriptor', () => {
     // If we try to close a feed that is opening should wait for the open result.
     const { publicKey, secretKey } = createKeyPair();
     const fd2 = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       hypercore: defaultHypercore
@@ -108,7 +108,7 @@ describe('FeedDescriptor', () => {
 
     const { publicKey, secretKey } = createKeyPair();
     const fd = new FeedDescriptor({
-      storage: createStorage(root, StorageType.NODE),
+      directory: createStorage(root, StorageType.NODE).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       valueEncoding: 'utf-8',
@@ -135,7 +135,7 @@ describe('FeedDescriptor', () => {
   test('on open error should unlock the resource', async () => {
     const { publicKey, secretKey } = createKeyPair();
     const fd = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       hypercore: () => {
@@ -149,16 +149,16 @@ describe('FeedDescriptor', () => {
   test('on close error should unlock the resource', async () => {
     const { publicKey, secretKey } = createKeyPair();
     const fd = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       hypercore: () => ({
         opened: true,
-        on () {},
-        ready (cb: () => void) {
+        on: () => {},
+        ready: (cb: () => void) => {
           cb();
         },
-        close () {
+        close: () => {
           throw new Error('close error');
         }
       } as any)

package/src/feed-descriptor.ts

@@ -5,16 +5,17 @@
 import assert from 'assert';
 import defaultHypercore from 'hypercore';
 import pify from 'pify';
+import { callbackify } from 'util';
 
 import { Lock } from '@dxos/async';
 import { PublicKey } from '@dxos/crypto';
-import type { File, Storage } from '@dxos/random-access-multi-storage';
+import type { Directory } from '@dxos/random-access-multi-storage';
 
 import type { HypercoreFeed, Hypercore } from './hypercore-types';
 import type { ValueEncoding } from './types';
 
 interface FeedDescriptorOptions {
-  storage: Storage,
+  directory: Directory,
   key: PublicKey,
   hypercore: Hypercore,
   secretKey?: Buffer,
@@ -28,7 +29,7 @@ interface FeedDescriptorOptions {
  * Abstract handler for an Hypercore instance.
  */
 export class FeedDescriptor {
-  private readonly _storage: Storage;
+  private readonly _directory: Directory;
   private readonly _key: PublicKey;
   private readonly _secretKey?: Buffer;
   private readonly _valueEncoding?: ValueEncoding;
@@ -40,7 +41,7 @@ export class FeedDescriptor {
 
   constructor (options: FeedDescriptorOptions) {
     const {
-      storage,
+      directory,
       key,
       secretKey,
       valueEncoding,
@@ -48,7 +49,7 @@ export class FeedDescriptor {
       disableSigning = false
     } = options;
 
-    this._storage = storage;
+    this._directory = directory;
     this._valueEncoding = valueEncoding;
     this._hypercore = hypercore;
     this._key = key;
@@ -119,9 +120,19 @@ export class FeedDescriptor {
    * Defines the real path where the Hypercore is going
    * to work with the RandomAccessStorage specified.
    */
-  private _createStorage (dir = ''): (name: string) => File {
+
+  private _createStorage (dir = ''): (name: string) => HypercoreFile {
     return (name) => {
-      return this._storage.createOrOpen(`${dir}/${name}`);
+      const file = this._directory.createOrOpen(`${dir}/${name}`);
+      // Separation between our internal File API and Hypercore's.
+      return {
+        read: callbackify(file.read.bind(file)),
+        write: callbackify(file.write.bind(file)),
+        del: callbackify(file.truncate.bind(file)),
+        stat: callbackify(file.stat.bind(file)),
+        close: callbackify(file.close.bind(file)),
+        destroy: callbackify(file.delete.bind(file))
+      } as HypercoreFile;
     };
   }
 
@@ -155,3 +166,15 @@ const MOCK_CRYPTO = {
     cb(null, true);
   }
 };
+
+/**
+ * File API that hypercore uses to read/write from storage.
+ */
+interface HypercoreFile {
+  read (offset: number, size: number, cb?: (err: Error | null, data?: Buffer) => void): void;
+  write (offset: number, data: Buffer, cb?: (err: Error | null) => void): void;
+  del (offset: number, size: number, cb?: (err: Error | null) => void): void;
+  stat (cb: (err: Error | null, data?: {size: number}) => void): void;
+  close (cb?: (err: Error | null) => void): void;
+  destroy (cb?: (err: Error | null) => void): void;
+}

package/src/feed-store.test.ts

@@ -6,7 +6,6 @@
 
 import assert from 'assert';
 import hypercore from 'hypercore';
-import hypertrie from 'hypertrie';
 import pify from 'pify';
 import tempy from 'tempy';
 
@@ -26,71 +25,43 @@ interface KeyPair {
 const feedNames = ['booksFeed', 'usersFeed', 'groupsFeed'];
 
 const createFeedStore = (storage: Storage, options = {}) => {
-  const feedStore = new FeedStore(storage, options);
+  const feedStore = new FeedStore(storage.directory('feed'), options);
   return feedStore;
 };
 
-async function createDefault () {
+const createDefault = async () => {
   const directory = tempy.directory();
 
   return {
     directory,
     feedStore: createFeedStore(createStorage(directory, StorageType.NODE), { valueEncoding: 'utf-8' })
   };
-}
+};
 
-async function defaultFeeds (feedStore: FeedStore, keys: Record<string, KeyPair>) : Promise<Record<string, FeedDescriptor>> {
-  return Object.fromEntries(await Promise.all(Object.entries<KeyPair>(keys).map(async ([feed, keyPair]) =>
-    [feed, await feedStore.openReadWriteFeed(keyPair.key, keyPair.secretKey)]
-  )));
-}
+const defaultFeeds = async (feedStore: FeedStore, keys: Record<string, KeyPair>): Promise<Record<string, FeedDescriptor>> => Object.fromEntries(await Promise.all(Object.entries<KeyPair>(keys).map(async ([feed, keyPair]) =>
+  [feed, await feedStore.openReadWriteFeed(keyPair.key, keyPair.secretKey)]
+)));
 
-function append (feed: HypercoreFeed, message: any) {
-  return pify(feed.append.bind(feed))(message);
-}
+const append = (feed: HypercoreFeed, message: any) => pify(feed.append.bind(feed))(message);
 
-function head (feed: HypercoreFeed) {
-  return pify(feed.head.bind(feed))();
-}
+const head = (feed: HypercoreFeed) => pify(feed.head.bind(feed))();
 
-const createKeyPairs = () => {
-  return Object.fromEntries<KeyPair>(feedNames.map(feed => {
-    const { publicKey, secretKey } = createKeyPair();
-    return [feed, { key: PublicKey.from(publicKey), secretKey }];
-  }));
-};
+const createKeyPairs = () => Object.fromEntries<KeyPair>(feedNames.map(feed => {
+  const { publicKey, secretKey } = createKeyPair();
+  return [feed, { key: PublicKey.from(publicKey), secretKey }];
+}));
 
 describe('FeedStore', () => {
   const keys = createKeyPairs();
 
   test('Config default', async () => {
-    const feedStore = await createFeedStore(createStorage('feed', StorageType.RAM));
+    const feedStore = await createFeedStore(createStorage('', StorageType.RAM));
     expect(feedStore).toBeInstanceOf(FeedStore);
 
-    const feedStore2 = new FeedStore(createStorage('feed', StorageType.RAM));
+    const feedStore2 = new FeedStore(createStorage('', StorageType.RAM).directory('feed'));
     expect(feedStore2).toBeInstanceOf(FeedStore);
   });
 
-  test('Config default + custom database + custom hypercore', async () => {
-    const customHypercore = jest.fn((...args) => {
-      return hypercore(args[0], args[1], args[2]);
-    });
-
-    const storage = createStorage('', StorageType.RAM);
-    const database = hypertrie(storage.createOrOpen.bind(storage), { valueEncoding: 'json' });
-    database.list = jest.fn((_, cb) => cb(null, []));
-
-    const feedStore = createFeedStore(createStorage('feed', StorageType.RAM), {
-      hypercore: customHypercore
-    });
-
-    expect(feedStore).toBeInstanceOf(FeedStore);
-
-    await feedStore.openReadOnlyFeed(PublicKey.random());
-
-    expect(customHypercore.mock.calls.length).toBe(1);
-  });
-
   test('Create feed', async () => {
     const { feedStore } = await createDefault();
     const { booksFeed: { feed: booksFeed } } = await defaultFeeds(feedStore, keys);
@@ -161,7 +132,7 @@ describe('FeedStore', () => {
   });
 
   test('Default codec: binary', async () => {
-    const feedStore = createFeedStore(createStorage('feed', StorageType.RAM));
+    const feedStore = createFeedStore(createStorage('', StorageType.RAM));
     expect(feedStore).toBeInstanceOf(FeedStore);
 
     const { publicKey, secretKey } = createKeyPair();
@@ -172,14 +143,14 @@ describe('FeedStore', () => {
   });
 
   test('on close error should unlock the descriptor', async () => {
-    const feedStore = createFeedStore(createStorage('feed', StorageType.RAM), {
+    const feedStore = createFeedStore(createStorage('', StorageType.RAM), {
       hypercore: () => ({
         opened: true,
-        ready (cb: () => void) {
+        ready: (cb: () => void) => {
           cb();
         },
-        on () {},
-        close () {
+        on: () => {},
+        close: () => {
           throw new Error('close error');
         }
       })

package/src/feed-store.ts

@@ -7,7 +7,7 @@ import defaultHypercore from 'hypercore';
 
 import { synchronized, Event } from '@dxos/async';
 import { PublicKey } from '@dxos/crypto';
-import { Storage } from '@dxos/random-access-multi-storage';
+import { Directory } from '@dxos/random-access-multi-storage';
 
 import FeedDescriptor from './feed-descriptor';
 import type { Hypercore } from './hypercore-types';
@@ -45,7 +45,7 @@ export interface FeedStoreOptions {
  * into a persist repository storage.
  */
 export class FeedStore {
-  private _storage: Storage;
+  private _directory: Directory;
   private _valueEncoding: ValueEncoding | undefined;
   private _hypercore: Hypercore;
   private _descriptors: Map<string, FeedDescriptor>;
@@ -56,13 +56,13 @@ export class FeedStore {
   readonly feedOpenedEvent = new Event<FeedDescriptor>();
 
   /**
-   * @param storage RandomAccessStorage to use by default by the feeds.
+   * @param directory RandomAccessStorage to use by default by the feeds.
    * @param options Feedstore options.
    */
-  constructor (storage: Storage, options: FeedStoreOptions = {}) {
-    assert(storage, 'The storage is required.');
+  constructor (directory: Directory, options: FeedStoreOptions = {}) {
+    assert(directory, 'The storage is required.');
 
-    this._storage = storage;
+    this._directory = directory;
 
     const {
       valueEncoding,
@@ -79,7 +79,7 @@ export class FeedStore {
    * @type {RandomAccessStorage}
    */
   get storage () {
-    return this._storage;
+    return this._directory;
   }
 
   @synchronized
@@ -114,7 +114,7 @@ export class FeedStore {
     const { key, secretKey } = options;
 
     const descriptor = new FeedDescriptor({
-      storage: this._storage,
+      directory: this._directory,
       key,
       secretKey,
       valueEncoding: this._valueEncoding,
@@ -133,7 +133,7 @@ export class FeedStore {
   }
 }
 
-function patchBufferCodec (encoding: ValueEncoding): ValueEncoding {
+const patchBufferCodec = (encoding: ValueEncoding): ValueEncoding => {
   if (typeof encoding === 'string') {
     return encoding;
   }
@@ -141,4 +141,4 @@ function patchBufferCodec (encoding: ValueEncoding): ValueEncoding {
     encode: (x: any) => Buffer.from(encoding.encode(x)),
     decode: encoding.decode.bind(encoding)
   };
-}
+};

package/src/stream.ts

@@ -20,62 +20,54 @@ const error = debug('dxos:stream:error');
  * @returns {NodeJS.WritableStream}
  */
 // TODO(burdon): Move to @dxos/codec.
-export function createWritableFeedStream (feed: HypercoreFeed) {
-  return new Writable({
-    objectMode: true,
-    write (message, _, callback) {
-      feed.append(message, callback);
-    }
-  });
-}
+export const createWritableFeedStream = (feed: HypercoreFeed) => new Writable({
+  objectMode: true,
+  write: (message, _, callback) => {
+    feed.append(message, callback);
+  }
+});
 
 /**
  * Creates a readStream stream that can be used as a buffer into which messages can be pushed.
  */
-export function createReadable (): Readable {
-  return new Readable({
-    objectMode: true,
-    read () {}
-  });
-}
+export const createReadable = (): Readable => new Readable({
+  objectMode: true,
+  read: () => {}
+});
 
 /**
  * Creates a writeStream object stream.
  * @param callback
  */
-export function createWritable<T> (callback: (message: T) => Promise<void>): NodeJS.WritableStream {
-  return new Writable({
-    objectMode: true,
-    write: async (message: T, _, next) => {
-      try {
-        await callback(message);
-        next();
-      } catch (err: any) {
-        error(err);
-        next(err);
-      }
+export const createWritable = <T>(callback: (message: T) => Promise<void>): NodeJS.WritableStream => new Writable({
+  objectMode: true,
+  write: async (message: T, _, next) => {
+    try {
+      await callback(message);
+      next();
+    } catch (err: any) {
+      error(err);
+      next(err);
     }
-  });
-}
+  }
+});
 
 /**
  * Creates a transform object stream.
  * @param callback
  */
-export function createTransform<R, W> (callback: (message: R) => Promise<W | undefined>): Transform {
-  return new Transform({
-    objectMode: true,
-    transform: async (message: R, _, next) => {
-      try {
-        const response = await callback(message);
-        next(null, response);
-      } catch (err: any) {
-        error(err);
-        next(err);
-      }
+export const createTransform = <R, W>(callback: (message: R) => Promise<W | undefined>): Transform => new Transform({
+  objectMode: true,
+  transform: async (message: R, _, next) => {
+    try {
+      const response = await callback(message);
+      next(null, response);
+    } catch (err: any) {
+      error(err);
+      next(err);
     }
-  });
-}
+  }
+});
 
 /**
  * Wriable stream that collects objects (e.g., for testing).