@dxos/feed-store 2.33.5-dev.ea3876ba → 2.33.5-dev.fee8f5fe

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/feed-store",
-  "version": "2.33.5-dev.ea3876ba",
+  "version": "2.33.5-dev.fee8f5fe",
   "description": "A consistent store for your hypercore feeds.",
   "license": "MIT",
   "main": "dist/src/index.js",
@@ -10,8 +10,8 @@
     "dist"
   ],
   "dependencies": {
-    "@dxos/async": "2.33.5-dev.ea3876ba",
-    "@dxos/util": "2.33.5-dev.ea3876ba",
+    "@dxos/async": "2.33.5-dev.fee8f5fe",
+    "@dxos/util": "2.33.5-dev.fee8f5fe",
     "buffer-json-encoding": "^1.0.2",
     "debug": "^4.3.3",
     "end-of-stream": "^1.4.1",
@@ -24,10 +24,10 @@
   "devDependencies": {
     "@dxos/benchmark-suite": "^1.0.0-beta.1",
     "@dxos/browser-runner": "^1.0.0-beta.13",
-    "@dxos/crypto": "2.33.5-dev.ea3876ba",
-    "@dxos/eslint-plugin": "~1.0.27",
+    "@dxos/crypto": "2.33.5-dev.fee8f5fe",
+    "@dxos/eslint-plugin": "~1.0.33",
     "@dxos/protocols-toolchain": "2.33.4",
-    "@dxos/random-access-multi-storage": "2.33.5-dev.ea3876ba",
+    "@dxos/random-access-multi-storage": "2.33.5-dev.fee8f5fe",
     "@types/debug": "^4.1.7",
     "@types/end-of-stream": "^1.4.0",
     "@types/from2": "~2.3.1",
@@ -40,7 +40,7 @@
     "eslint": "^7.12.1",
     "hypertrie": "5.1.1",
     "tempy": "~1.0.1",
-    "typescript": "^4.5.2",
+    "typescript": "^4.7.2",
     "wait-for-expect": "^3.0.2"
   },
   "publishConfig": {

package/src/benchmark.ts

@@ -21,44 +21,40 @@ void (async () => {
     }
   };
 
-  const fs = new FeedStore(createStorage('.benchmark'), { valueEncoding: 'utf8' });
+  const fs = new FeedStore(createStorage('').directory('.benchmark'), { valueEncoding: 'utf8' });
   const suite = new Suite(fs, { maxFeeds, maxMessages });
 
-  suite.beforeAll(() => {
-    return Promise.all(range(maxFeeds).map(async i => {
-      const name = `feed/${i}`;
-      const { publicKey, secretKey } = createKeyPair();
-      const { feed } = await fs.openReadWriteFeed(PublicKey.from(publicKey), secretKey);
+  suite.beforeAll(() => Promise.all(range(maxFeeds).map(async i => {
+    const name = `feed/${i}`;
+    const { publicKey, secretKey } = createKeyPair();
+    const { feed } = await fs.openReadWriteFeed(PublicKey.from(publicKey), secretKey);
 
-      for (let i = 0; i < maxMessages; i++) {
-        await new Promise<void>((resolve, reject) => {
-          feed.append(`${name}/${i}`, (err) => {
-            if (err) {
-              return reject(err);
-            }
-            resolve();
-          });
+    for (let i = 0; i < maxMessages; i++) {
+      await new Promise<void>((resolve, reject) => {
+        feed.append(`${name}/${i}`, (err) => {
+          if (err) {
+            return reject(err);
+          }
+          resolve();
         });
-      }
+      });
+    }
 
-      return feed;
-    }));
-  });
+    return feed;
+  })));
 
   suite.test('getBatch', async () => {
     let count = 0;
 
-    await Promise.all(Array.from((fs as any)._descriptors.values()).map(({ feed }: any) => {
-      return new Promise<void>((resolve, reject) => {
-        feed.getBatch(0, maxMessages, (err: Error, result: any) => {
-          count += result.length;
-          if (err) {
-            return reject(err);
-          }
-          resolve();
-        });
+    await Promise.all(Array.from((fs as any)._descriptors.values()).map(({ feed }: any) => new Promise<void>((resolve, reject) => {
+      feed.getBatch(0, maxMessages, (err: Error, result: any) => {
+        count += result.length;
+        if (err) {
+          return reject(err);
+        }
+        resolve();
       });
-    }));
+    })));
 
     check(count);
   });

package/src/create-batch-stream.test.ts

@@ -13,15 +13,13 @@ import { FeedStore } from './feed-store';
 import { HypercoreFeed } from './hypercore-types';
 
 const createFeed = async () => {
-  const feedStore = new FeedStore(createStorage('', StorageType.RAM), { valueEncoding: 'utf-8' });
+  const feedStore = new FeedStore(createStorage('', StorageType.RAM).directory('feed'), { valueEncoding: 'utf-8' });
   const { publicKey, secretKey } = createKeyPair();
   const { feed } = await feedStore.openReadWriteFeed(PublicKey.from(publicKey), secretKey);
   return feed;
 };
 
-const append = (feed: HypercoreFeed, message: any) => {
-  return pify(feed.append.bind(feed))(message);
-};
+const append = (feed: HypercoreFeed, message: any) => pify(feed.append.bind(feed))(message);
 
 describe('Batch stream', () => {
   test('Single message', async () => {

package/src/create-batch-stream.ts

@@ -17,7 +17,7 @@ export interface CreateBatchStreamOptions {
   tail?: boolean
 }
 
-export function createBatchStream (feed: HypercoreFeed, opts: CreateBatchStreamOptions = {}) {
+export const createBatchStream = (feed: HypercoreFeed, opts: CreateBatchStreamOptions = {}) => {
   assert(!opts.batch || opts.batch > 0, 'batch must be major or equal to 1');
 
   let start = opts.start || 0;
@@ -34,9 +34,7 @@ export function createBatchStream (feed: HypercoreFeed, opts: CreateBatchStreamO
 
   let range = feed.download({ start, end, linear: true });
 
-  return streamFrom.obj(read).on('end', cleanup).on('close', cleanup);
-
-  function read (size: any, cb?: any) {
+  const read = (size: any, cb?: any) => {
     if (!feed.opened) {
       return open(size, cb);
     }
@@ -103,9 +101,9 @@ export function createBatchStream (feed: HypercoreFeed, opts: CreateBatchStreamO
 
       cb(null, messages.map(buildMessage));
     });
-  }
+  };
 
-  function buildMessage (data: object) {
+  const buildMessage = (data: object) => {
     const message = {
       key: feed.key,
       seq: seq++,
@@ -115,26 +113,26 @@ export function createBatchStream (feed: HypercoreFeed, opts: CreateBatchStreamO
     };
 
     return message;
-  }
+  };
 
-  function cleanup () {
+  const cleanup = () => {
     if (!range) {
       return;
     }
     feed.undownload(range);
     range = null;
-  }
+  };
 
-  function open (size: any, cb: (err: Error) => void) {
-    feed.ready(function (err: Error) {
+  const open = (size: any, cb: (err: Error) => void) => {
+    feed.ready((err: Error) => {
       if (err) {
         return cb(err);
       }
       read(size, cb);
     });
-  }
+  };
 
-  function setStart (value: number) {
+  const setStart = (value: number) => {
     const prevStart = start;
     start = value;
     range.start = start;
@@ -142,5 +140,7 @@ export function createBatchStream (feed: HypercoreFeed, opts: CreateBatchStreamO
       range.iterator.start = start;
     }
     return prevStart;
-  }
-}
+  };
+
+  return streamFrom.obj(read).on('end', cleanup).on('close', cleanup);
+};

package/src/feed-descriptor.test.ts

@@ -20,7 +20,7 @@ describe('FeedDescriptor', () => {
   beforeEach(async () => {
     const { publicKey, secretKey } = createKeyPair();
     fd = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       hypercore: defaultHypercore
@@ -41,7 +41,7 @@ describe('FeedDescriptor', () => {
     // When this behaviour was changed, suddenly `protocol-plugin-replicator` tests started hanging forever on network generation.
     const { publicKey } = createKeyPair();
     const key = PublicKey.from(publicKey);
-    const fd = new FeedDescriptor({ key, storage: createStorage('', StorageType.NODE), hypercore: defaultHypercore });
+    const fd = new FeedDescriptor({ key, directory: createStorage('', StorageType.NODE).directory('feed'), hypercore: defaultHypercore });
     expect(fd.key).toEqual(key);
     expect(fd.secretKey).toBeUndefined();
   });
@@ -50,7 +50,7 @@ describe('FeedDescriptor', () => {
     const { publicKey, secretKey } = createKeyPair();
 
     const fd = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       valueEncoding: 'json',
@@ -92,7 +92,7 @@ describe('FeedDescriptor', () => {
     // If we try to close a feed that is opening should wait for the open result.
     const { publicKey, secretKey } = createKeyPair();
     const fd2 = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       hypercore: defaultHypercore
@@ -108,7 +108,7 @@ describe('FeedDescriptor', () => {
 
     const { publicKey, secretKey } = createKeyPair();
     const fd = new FeedDescriptor({
-      storage: createStorage(root, StorageType.NODE),
+      directory: createStorage(root, StorageType.NODE).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       valueEncoding: 'utf-8',
@@ -135,7 +135,7 @@ describe('FeedDescriptor', () => {
   test('on open error should unlock the resource', async () => {
     const { publicKey, secretKey } = createKeyPair();
     const fd = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       hypercore: () => {
@@ -149,16 +149,16 @@ describe('FeedDescriptor', () => {
   test('on close error should unlock the resource', async () => {
     const { publicKey, secretKey } = createKeyPair();
     const fd = new FeedDescriptor({
-      storage: createStorage('', StorageType.RAM),
+      directory: createStorage('', StorageType.RAM).directory('feed'),
       key: PublicKey.from(publicKey),
       secretKey,
       hypercore: () => ({
         opened: true,
-        on () {},
-        ready (cb: () => void) {
+        on: () => {},
+        ready: (cb: () => void) => {
           cb();
         },
-        close () {
+        close: () => {
           throw new Error('close error');
         }
       } as any)

package/src/feed-descriptor.ts

@@ -5,20 +5,22 @@
 import assert from 'assert';
 import defaultHypercore from 'hypercore';
 import pify from 'pify';
+import { callbackify } from 'util';
 
 import { Lock } from '@dxos/async';
 import { PublicKey } from '@dxos/crypto';
-import type { File, Storage } from '@dxos/random-access-multi-storage';
+import type { Directory } from '@dxos/random-access-multi-storage';
 
 import type { HypercoreFeed, Hypercore } from './hypercore-types';
 import type { ValueEncoding } from './types';
 
 interface FeedDescriptorOptions {
-  storage: Storage,
+  directory: Directory,
   key: PublicKey,
   hypercore: Hypercore,
   secretKey?: Buffer,
   valueEncoding?: ValueEncoding
+  disableSigning?: boolean
 }
 
 /**
@@ -27,29 +29,32 @@ interface FeedDescriptorOptions {
  * Abstract handler for an Hypercore instance.
  */
 export class FeedDescriptor {
-  private readonly _storage: Storage;
+  private readonly _directory: Directory;
   private readonly _key: PublicKey;
   private readonly _secretKey?: Buffer;
   private readonly _valueEncoding?: ValueEncoding;
   private readonly _hypercore: Hypercore;
   private readonly _lock: Lock;
+  private readonly _disableSigning: boolean;
 
   private _feed: HypercoreFeed | null;
 
   constructor (options: FeedDescriptorOptions) {
     const {
-      storage,
+      directory,
       key,
       secretKey,
       valueEncoding,
-      hypercore = defaultHypercore
+      hypercore = defaultHypercore,
+      disableSigning = false
     } = options;
 
-    this._storage = storage;
+    this._directory = directory;
     this._valueEncoding = valueEncoding;
     this._hypercore = hypercore;
     this._key = key;
     this._secretKey = secretKey;
+    this._disableSigning = !!disableSigning;
 
     this._lock = new Lock();
 
@@ -115,9 +120,19 @@ export class FeedDescriptor {
    * Defines the real path where the Hypercore is going
    * to work with the RandomAccessStorage specified.
    */
-  private _createStorage (dir = ''): (name: string) => File {
+
+  private _createStorage (dir = ''): (name: string) => HypercoreFile {
     return (name) => {
-      return this._storage.createOrOpen(`${dir}/${name}`);
+      const file = this._directory.createOrOpen(`${dir}/${name}`);
+      // Separation between our internal File API and Hypercore's.
+      return {
+        read: callbackify(file.read.bind(file)),
+        write: callbackify(file.write.bind(file)),
+        del: callbackify(file.truncate.bind(file)),
+        stat: callbackify(file.stat.bind(file)),
+        close: callbackify(file.close.bind(file)),
+        destroy: callbackify(file.delete.bind(file))
+      } as HypercoreFile;
     };
   }
 
@@ -127,7 +142,8 @@ export class FeedDescriptor {
       this._key.asBuffer(),
       {
         secretKey: this._secretKey,
-        valueEncoding: this._valueEncoding
+        valueEncoding: this._valueEncoding,
+        crypto: this._disableSigning ? MOCK_CRYPTO : undefined
       }
     );
 
@@ -141,3 +157,24 @@ export class FeedDescriptor {
 }
 
 export default FeedDescriptor;
+
+const MOCK_CRYPTO = {
+  sign: (data: any, secretKey: any, cb: any) => {
+    cb(null, Buffer.from(''));
+  },
+  verify: (signature: any, data: any, key: any, cb: any) => {
+    cb(null, true);
+  }
+};
+
+/**
+ * File API that hypercore uses to read/write from storage.
+ */
+interface HypercoreFile {
+  read (offset: number, size: number, cb?: (err: Error | null, data?: Buffer) => void): void;
+  write (offset: number, data: Buffer, cb?: (err: Error | null) => void): void;
+  del (offset: number, size: number, cb?: (err: Error | null) => void): void;
+  stat (cb: (err: Error | null, data?: {size: number}) => void): void;
+  close (cb?: (err: Error | null) => void): void;
+  destroy (cb?: (err: Error | null) => void): void;
+}

package/src/feed-store.test.ts

@@ -6,7 +6,6 @@
 
 import assert from 'assert';
 import hypercore from 'hypercore';
-import hypertrie from 'hypertrie';
 import pify from 'pify';
 import tempy from 'tempy';
 
@@ -26,71 +25,43 @@ interface KeyPair {
 const feedNames = ['booksFeed', 'usersFeed', 'groupsFeed'];
 
 const createFeedStore = (storage: Storage, options = {}) => {
-  const feedStore = new FeedStore(storage, options);
+  const feedStore = new FeedStore(storage.directory('feed'), options);
   return feedStore;
 };
 
-async function createDefault () {
+const createDefault = async () => {
   const directory = tempy.directory();
 
   return {
     directory,
     feedStore: createFeedStore(createStorage(directory, StorageType.NODE), { valueEncoding: 'utf-8' })
   };
-}
+};
 
-async function defaultFeeds (feedStore: FeedStore, keys: Record<string, KeyPair>) : Promise<Record<string, FeedDescriptor>> {
-  return Object.fromEntries(await Promise.all(Object.entries<KeyPair>(keys).map(async ([feed, keyPair]) =>
-    [feed, await feedStore.openReadWriteFeed(keyPair.key, keyPair.secretKey)]
-  )));
-}
+const defaultFeeds = async (feedStore: FeedStore, keys: Record<string, KeyPair>): Promise<Record<string, FeedDescriptor>> => Object.fromEntries(await Promise.all(Object.entries<KeyPair>(keys).map(async ([feed, keyPair]) =>
+  [feed, await feedStore.openReadWriteFeed(keyPair.key, keyPair.secretKey)]
+)));
 
-function append (feed: HypercoreFeed, message: any) {
-  return pify(feed.append.bind(feed))(message);
-}
+const append = (feed: HypercoreFeed, message: any) => pify(feed.append.bind(feed))(message);
 
-function head (feed: HypercoreFeed) {
-  return pify(feed.head.bind(feed))();
-}
+const head = (feed: HypercoreFeed) => pify(feed.head.bind(feed))();
 
-const createKeyPairs = () => {
-  return Object.fromEntries<KeyPair>(feedNames.map(feed => {
-    const { publicKey, secretKey } = createKeyPair();
-    return [feed, { key: PublicKey.from(publicKey), secretKey }];
-  }));
-};
+const createKeyPairs = () => Object.fromEntries<KeyPair>(feedNames.map(feed => {
+  const { publicKey, secretKey } = createKeyPair();
+  return [feed, { key: PublicKey.from(publicKey), secretKey }];
+}));
 
 describe('FeedStore', () => {
   const keys = createKeyPairs();
 
   test('Config default', async () => {
-    const feedStore = await createFeedStore(createStorage('feed', StorageType.RAM));
+    const feedStore = await createFeedStore(createStorage('', StorageType.RAM));
     expect(feedStore).toBeInstanceOf(FeedStore);
 
-    const feedStore2 = new FeedStore(createStorage('feed', StorageType.RAM));
+    const feedStore2 = new FeedStore(createStorage('', StorageType.RAM).directory('feed'));
     expect(feedStore2).toBeInstanceOf(FeedStore);
   });
 
-  test('Config default + custom database + custom hypercore', async () => {
-    const customHypercore = jest.fn((...args) => {
-      return hypercore(args[0], args[1], args[2]);
-    });
-
-    const storage = createStorage('', StorageType.RAM);
-    const database = hypertrie(storage.createOrOpen.bind(storage), { valueEncoding: 'json' });
-    database.list = jest.fn((_, cb) => cb(null, []));
-
-    const feedStore = createFeedStore(createStorage('feed', StorageType.RAM), {
-      hypercore: customHypercore
-    });
-
-    expect(feedStore).toBeInstanceOf(FeedStore);
-
-    await feedStore.openReadOnlyFeed(PublicKey.random());
-
-    expect(customHypercore.mock.calls.length).toBe(1);
-  });
-
   test('Create feed', async () => {
     const { feedStore } = await createDefault();
     const { booksFeed: { feed: booksFeed } } = await defaultFeeds(feedStore, keys);
@@ -161,7 +132,7 @@ describe('FeedStore', () => {
   });
 
   test('Default codec: binary', async () => {
-    const feedStore = createFeedStore(createStorage('feed', StorageType.RAM));
+    const feedStore = createFeedStore(createStorage('', StorageType.RAM));
     expect(feedStore).toBeInstanceOf(FeedStore);
 
     const { publicKey, secretKey } = createKeyPair();
@@ -172,14 +143,14 @@ describe('FeedStore', () => {
   });
 
   test('on close error should unlock the descriptor', async () => {
-    const feedStore = createFeedStore(createStorage('feed', StorageType.RAM), {
+    const feedStore = createFeedStore(createStorage('', StorageType.RAM), {
       hypercore: () => ({
         opened: true,
-        ready (cb: () => void) {
+        ready: (cb: () => void) => {
           cb();
         },
-        on () {},
-        close () {
+        on: () => {},
+        close: () => {
           throw new Error('close error');
         }
       })

package/src/feed-store.ts

@@ -7,7 +7,7 @@ import defaultHypercore from 'hypercore';
 
 import { synchronized, Event } from '@dxos/async';
 import { PublicKey } from '@dxos/crypto';
-import { Storage } from '@dxos/random-access-multi-storage';
+import { Directory } from '@dxos/random-access-multi-storage';
 
 import FeedDescriptor from './feed-descriptor';
 import type { Hypercore } from './hypercore-types';
@@ -45,7 +45,7 @@ export interface FeedStoreOptions {
  * into a persist repository storage.
  */
 export class FeedStore {
-  private _storage: Storage;
+  private _directory: Directory;
   private _valueEncoding: ValueEncoding | undefined;
   private _hypercore: Hypercore;
   private _descriptors: Map<string, FeedDescriptor>;
@@ -56,13 +56,13 @@ export class FeedStore {
   readonly feedOpenedEvent = new Event<FeedDescriptor>();
 
   /**
-   * @param storage RandomAccessStorage to use by default by the feeds.
+   * @param directory RandomAccessStorage to use by default by the feeds.
    * @param options Feedstore options.
    */
-  constructor (storage: Storage, options: FeedStoreOptions = {}) {
-    assert(storage, 'The storage is required.');
+  constructor (directory: Directory, options: FeedStoreOptions = {}) {
+    assert(directory, 'The storage is required.');
 
-    this._storage = storage;
+    this._directory = directory;
 
     const {
       valueEncoding,
@@ -79,7 +79,7 @@ export class FeedStore {
    * @type {RandomAccessStorage}
    */
   get storage () {
-    return this._storage;
+    return this._directory;
   }
 
   @synchronized
@@ -114,7 +114,7 @@ export class FeedStore {
     const { key, secretKey } = options;
 
     const descriptor = new FeedDescriptor({
-      storage: this._storage,
+      directory: this._directory,
       key,
       secretKey,
       valueEncoding: this._valueEncoding,
@@ -133,7 +133,7 @@ export class FeedStore {
   }
 }
 
-function patchBufferCodec (encoding: ValueEncoding): ValueEncoding {
+const patchBufferCodec = (encoding: ValueEncoding): ValueEncoding => {
   if (typeof encoding === 'string') {
     return encoding;
   }
@@ -141,4 +141,4 @@ function patchBufferCodec (encoding: ValueEncoding): ValueEncoding {
     encode: (x: any) => Buffer.from(encoding.encode(x)),
     decode: encoding.decode.bind(encoding)
   };
-}
+};