@dxos/hypercore 0.1.0

package/src/hypercore-protocol.test.ts

@@ -0,0 +1,236 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+import { expect } from 'chai';
+import faker from 'faker';
+import ProtocolStream from 'hypercore-protocol';
+
+import { latch } from '@dxos/async';
+import { createKeyPair } from '@dxos/crypto';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+
+import { HypercoreFactory } from './hypercore-factory';
+
+// TODO(burdon): Test encoding.
+// TODO(burdon): Simulate multiple peers and broadcast.
+
+describe('ProtocolStream', function () {
+  it('protocol stream handshake completes', async function () {
+    const [handshake, setHandshake] = latch({ count: 2 });
+
+    const stream1 = new ProtocolStream(true, {
+      onhandshake: () => {
+        log('onhandshake', { key: PublicKey.from(stream1.publicKey) });
+        setHandshake();
+      }
+    });
+
+    const stream2 = new ProtocolStream(false, {
+      onhandshake: () => {
+        log('onhandshake', { key: PublicKey.from(stream2.publicKey) });
+        setHandshake();
+      }
+    });
+
+    const [closed, setClosed] = latch({ count: 2 });
+    const pipeline = stream1.pipe(stream2, setClosed).pipe(stream1, setClosed);
+
+    await handshake();
+    pipeline.destroy();
+    await closed();
+  });
+
+  it('protocol stream handshake completes with feeds', async function () {
+    //
+    // Pipeline and handshake.
+    //
+
+    const [handshake, setHandshake] = latch({ count: 2 });
+
+    const stream1 = new ProtocolStream(true, {
+      onhandshake: () => {
+        log('onhandshake', { key: PublicKey.from(stream1.publicKey) });
+        setHandshake();
+      }
+    });
+
+    const stream2 = new ProtocolStream(false, {
+      onhandshake: () => {
+        log('onhandshake', { key: PublicKey.from(stream2.publicKey) });
+        setHandshake();
+      }
+    });
+
+    const [streamClosed, setStreamClosed] = latch({ count: 2 });
+    const pipeline = stream1.pipe(stream2, setStreamClosed).pipe(stream1, setStreamClosed);
+
+    await handshake();
+
+    //
+    // Hypercore replication.
+    // https://github.com/hypercore-protocol/hypercore-protocol#const-channel--streamopenkey-handlers
+    //
+
+    const factory = new HypercoreFactory<string>();
+    const { publicKey, secretKey } = createKeyPair();
+    const core1 = factory.createFeed(publicKey, { secretKey });
+    const core2 = factory.createFeed(publicKey);
+
+    const [feedsClosed, setFeedClosed] = latch({ count: 2 });
+    core1.once('close', setFeedClosed);
+    core2.once('close', setFeedClosed);
+
+    core1.replicate(true, { stream: stream1, live: true });
+    core2.replicate(false, { stream: stream2, live: true });
+
+    //
+    // Extensions.
+    //
+
+    const [received, incMessage] = latch({ count: 2 });
+    const extension1 = stream1.registerExtension('stream-extension', {
+      onmessage: (msg: any) => {
+        log('onmessage', {
+          key: PublicKey.from(stream1.publicKey),
+          msg: msg.toString()
+        });
+        incMessage();
+      }
+    });
+
+    const extension2 = stream2.registerExtension('stream-extension', {
+      onmessage: (msg: any) => {
+        log('onmessage', {
+          key: PublicKey.from(stream2.publicKey),
+          msg: msg.toString()
+        });
+        incMessage();
+      }
+    });
+
+    extension1.send(Buffer.from('message-1'));
+    extension2.send(Buffer.from('message-2'));
+    await received();
+
+    //
+    // Replication.
+    //
+
+    const [synced, setSynced] = latch({ count: 1 });
+
+    {
+      const numBlocks = 10;
+      Array.from(Array(numBlocks)).forEach((_, i) => {
+        core1.append(`block-${i}`, () => {});
+      });
+
+      core2.on('sync', () => {
+        log('sync', {
+          core: { key: PublicKey.from(core2.key), length: core2.length }
+        });
+        expect(core2.length).to.eq(numBlocks);
+        setSynced();
+      });
+    }
+
+    await synced();
+
+    //
+    // Close.
+    //
+
+    // Gracefully shutdown.
+    pipeline.finalize();
+    await streamClosed();
+
+    // TODO(burdon): Destroy and finalize don't seem to close the feeds.
+    expect(core1.closed).to.be.false;
+    expect(core2.closed).to.be.false;
+
+    core1.close(setFeedClosed);
+    core2.close(setFeedClosed);
+    await feedsClosed();
+
+    expect(core1.closed).to.be.true;
+    expect(core2.closed).to.be.true;
+  });
+
+  it('multi-feed multiplexing', async function () {
+    //
+    // Handshake.
+    //
+
+    const [handshake, setHandshake] = latch({ count: 2 });
+    const stream1 = new ProtocolStream(true, { onhandshake: setHandshake });
+    const stream2 = new ProtocolStream(false, { onhandshake: setHandshake });
+
+    const [streamClosed, setStreamClosed] = latch({ count: 2 });
+    const pipeline = stream1.pipe(stream2, setStreamClosed).pipe(stream1, setStreamClosed);
+    await handshake();
+
+    //
+    // Feeds.
+    //
+
+    const factory = new HypercoreFactory<string>();
+
+    const numFeeds = 5;
+    const [feedsClosed, setFeedClosed] = latch({ count: 2 * numFeeds });
+
+    // Create set of feed pairs.
+    const feeds = Array.from(Array(numFeeds)).map(() => {
+      const { publicKey, secretKey } = createKeyPair();
+      const core1 = factory.createFeed(publicKey, { secretKey });
+      const core2 = factory.createFeed(publicKey);
+
+      core1.once('close', setFeedClosed);
+      core2.once('close', setFeedClosed);
+
+      core1.replicate(true, { stream: stream1, live: true });
+      core2.replicate(false, { stream: stream2, live: true });
+
+      return [core1, core2];
+    });
+
+    //
+    // Replicate.
+    //
+
+    let total = 0;
+    const numBlocks = 100;
+    const [synced, setSynced] = latch({ count: numFeeds });
+    feeds.forEach(([, core2]) => {
+      core2.on('sync', () => {
+        core2.audit((_, data) => {
+          total += data.valid;
+          log('synced', { data });
+          setSynced();
+        });
+      });
+    });
+
+    Array.from(Array(numBlocks)).forEach((_, i) => {
+      const [core1] = faker.random.arrayElement(feeds);
+      core1.append(`block-${i}`, () => {});
+    });
+
+    await synced();
+    expect(total).to.eq(numBlocks);
+
+    //
+    // Close.
+    //
+
+    pipeline.finalize();
+    await streamClosed();
+
+    feeds.forEach(([core1, core2]) => {
+      core1.close();
+      core2.close();
+    });
+
+    await feedsClosed();
+  });
+});

package/src/hypercore.test.ts

@@ -0,0 +1,98 @@
+//
+// Copyright 2019 DXOS.org
+//
+
+import { expect } from 'chai';
+import { AbstractValueEncoding } from 'hypercore';
+import util from 'node:util';
+
+import { Codec } from '@dxos/codec-protobuf';
+import { createKeyPair } from '@dxos/crypto';
+
+import { createCodecEncoding } from './crypto';
+import { HypercoreFactory } from './hypercore-factory';
+
+type TestItem = {
+  key: string;
+  value: string;
+};
+
+const codec: Codec<TestItem> = {
+  encode: (obj: TestItem) => Buffer.from(JSON.stringify(obj)),
+  decode: (buffer: Uint8Array) => JSON.parse(buffer.toString())
+};
+
+const valueEncoding: AbstractValueEncoding<TestItem> = createCodecEncoding(codec);
+
+describe('Hypercore', function () {
+  it('create, append, and close a feed', async function () {
+    const factory = new HypercoreFactory<string>();
+    const { publicKey, secretKey } = createKeyPair();
+    const core = factory.createFeed(publicKey, { secretKey });
+
+    {
+      // Check open is idempotent.
+      const open = util.promisify(core.open.bind(core));
+      await open();
+      await open();
+    }
+
+    {
+      // Append block.
+      expect(core.length).to.eq(0);
+      const append = util.promisify(core.append.bind(core));
+      const seq = await append('test');
+      expect(core.length).to.eq(1);
+      expect(seq).to.eq(0);
+    }
+
+    {
+      // Get block.
+      const get = util.promisify(core.get.bind(core));
+      const block: any = await get(0);
+      expect(block.toString()).to.eq('test');
+    }
+
+    {
+      // Check open is idempotent.
+      const close = util.promisify(core.close.bind(core));
+      await close();
+      await close();
+    }
+  });
+
+  it('encoding with typed hypercore', async function () {
+    const factory = new HypercoreFactory<TestItem>();
+    const { publicKey, secretKey } = createKeyPair();
+    const core = factory.createFeed(publicKey, { secretKey, valueEncoding });
+
+    {
+      const append = util.promisify(core.append.bind(core));
+
+      expect(core.length).to.eq(0);
+      const seq = await append({
+        key: 'test-1',
+        value: 'test'
+      });
+
+      expect(core.length).to.eq(1);
+      expect(seq).to.eq(0);
+    }
+
+    {
+      const head = util.promisify(core.head.bind(core));
+
+      const { key, value } = await head();
+      expect(key).to.eq('test-1');
+      expect(value).to.eq('test');
+    }
+
+    {
+      const get = util.promisify(core.get.bind(core));
+
+      const { key, value } = await get(0);
+      expect(key).to.eq('test-1');
+      expect(value).to.eq('test');
+    }
+  });
+});

package/src/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+export * from './crypto';
+export * from './defaults';
+export * from './hypercore-factory';
+export * from './iterator';

package/src/iterator.test.ts

@@ -0,0 +1,66 @@
+//
+// Copyright 2019 DXOS.org
+//
+
+import util from 'node:util';
+
+import { latch } from '@dxos/async';
+import { createKeyPair } from '@dxos/crypto';
+import { log } from '@dxos/log';
+
+import { HypercoreFactory } from './hypercore-factory';
+import { createAsyncIterator, createReadable } from './iterator';
+
+describe('AsyncIterator', function () {
+  it('iterates a feed until stopped', async function () {
+    const factory = new HypercoreFactory();
+    const { publicKey, secretKey } = createKeyPair();
+    const core = factory.createFeed(publicKey, { secretKey });
+
+    const numBlocks = 10;
+
+    // Write.
+    {
+      const append = util.promisify(core.append.bind(core));
+      for (let i = 0; i < numBlocks; i++) {
+        await append(`test-${i}`);
+      }
+    }
+
+    // Read until stopped.
+    {
+      // Read forever.
+      // NOTE: Read must be opened in live mode.
+      // NOTE: Uses wrapper to normalize stream; otherwise done isn't called after destroying the stream.
+      const stream = createReadable(core.createReadStream({ live: true }));
+
+      // Use util to create reliable iterators.
+      const iterator = createAsyncIterator(stream);
+
+      const [streamEnded, end] = latch();
+      const [readAll, inc] = latch({ count: numBlocks });
+      setTimeout(async () => {
+        while (true) {
+          try {
+            const { value, done } = await iterator.next();
+            if (done) {
+              end();
+              break;
+            }
+
+            log('received', { data: String(value) });
+            inc();
+          } catch (err) {
+            end();
+            break;
+          }
+        }
+      });
+
+      await readAll();
+
+      stream.destroy();
+      await streamEnded();
+    }
+  });
+});

package/src/iterator.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { Readable } from 'readable-stream';
+import { Readable as StreamXReadable } from 'streamx';
+
+/**
+ * Wraps streamx.Readable (hypercore.createReadStream) to a standard Readable stream.
+ *
+ * The read-stream package is mirror of the streams implementations in Node.js 18.9.0.
+ * This function is here to standardize the cast in case there are incompatibilities
+ * across different platforms.
+ *
+ * Hypercore createReadStream returns a `streamx` Readable, which does not close properly on destroy.
+ *
+ * https://github.com/nodejs/readable-stream
+ * https://nodejs.org/api/stream.html#readable-streams
+ * https://nodejs.org/dist/v18.9.0/docs/api/stream.html#readablewrapstream
+ */
+export const createReadable = (stream: StreamXReadable): Readable => {
+  return new Readable({ objectMode: true }).wrap(stream as any);
+};
+
+/**
+ * Converts streamx.Readable (hypercore.createReadStream) to an async iterator.
+ *
+ * https://github.com/tc39/proposal-async-iteration
+ * https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-3.html#async-iteration
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Iterators_and_Generators
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols
+ */
+export const createAsyncIterator = (stream: Readable): AsyncIterator<any> => {
+  return stream[Symbol.asyncIterator]();
+};

package/src/multiplexer.test.ts

@@ -0,0 +1,172 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { expect } from 'chai';
+import faker from 'faker';
+import { PassThrough, Transform } from 'streamx';
+
+import { latch } from '@dxos/async';
+import { createKeyPair } from '@dxos/crypto';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+
+import { HypercoreFactory } from './hypercore-factory';
+import { Multiplexer } from './multiplexer';
+import { noop, py } from './testing';
+
+describe('Multiplexing', function () {
+  it.only('multiplexes feeds', async function () {
+    const plex1 = new Multiplexer('A');
+    const plex2 = new Multiplexer('B');
+
+    await plex1.open();
+    await plex2.open();
+
+    const numFeeds = 1;
+    const factory = new HypercoreFactory();
+    const feeds = await Promise.all(
+      Array.from(Array(numFeeds)).map(async () => {
+        const { publicKey, secretKey } = createKeyPair();
+        const core1 = factory.createFeed(publicKey, { secretKey });
+        const core2 = factory.createFeed(publicKey);
+
+        await py(core1, core1.open)();
+        await py(core2, core2.open)();
+
+        plex1.addFeed(core1, true);
+        plex2.addFeed(core2);
+        return [core1, core2];
+      })
+    );
+
+    feeds.forEach(([core1, core2]) => {
+      expect(core1.stats.peers.length).to.eq(1);
+      expect(core2.stats.peers.length).to.eq(1);
+    });
+
+    const [closed, setClosed] = latch({ count: 2 });
+
+    // TODO(burdon): Test with additional PassThrough.
+    plex1.output.pipe(plex2.input, setClosed);
+    plex2.output.pipe(plex1.input, setClosed);
+
+    const numMessages = 1;
+    const written = new Set<string>();
+    Array.from(Array(numMessages)).forEach((_, i) => {
+      const [core] = faker.random.arrayElement(feeds);
+      written.add(core.key.toString());
+      core.append(`test-${i}: ${faker.lorem.sentence(10)}`, noop); // Long message.
+    });
+
+    {
+      const [synced, setSynced] = latch({ count: written.size });
+      feeds.forEach(([, core2]) => {
+        core2.on('sync', () => {
+          log('sync', { feedKey: PublicKey.from(core2.key) });
+          setSynced();
+        });
+      });
+
+      await synced();
+    }
+
+    await plex1.close();
+    await plex2.close();
+    await closed();
+
+    expect(plex1.isOpen).to.be.false;
+    expect(plex2.isOpen).to.be.false;
+
+    const count = feeds.reduce((total, [, core2]) => total + core2.length, 0);
+    expect(count).to.eq(numMessages);
+  });
+
+  it('pipelines', async function () {
+    const factory = new HypercoreFactory();
+    const { publicKey, secretKey } = createKeyPair();
+    const core1 = factory.createFeed(publicKey, { secretKey });
+    const core2 = factory.createFeed(publicKey);
+
+    await py(core1, core1.open)();
+    await py(core2, core2.open)();
+
+    const stream1 = core1.replicate(true, { live: true });
+    const stream2 = core2.replicate(false, { live: true });
+
+    //
+    // Pipeline
+    //
+
+    const createWireTap = (label: string) =>
+      new Transform({
+        transform: (data: Buffer, next: (err: Error | null, data: Buffer) => void) => {
+          log(label, { length: data.length });
+          next(null, data);
+        }
+      });
+
+    const [pipelineClosed, onStreamClose] = latch({ count: 2 });
+    const createPipeline = (type: number) => {
+      switch (type) {
+        case 1:
+          // prettier-ignore
+          return stream1
+            .pipe(stream2, onStreamClose)
+            .pipe(stream1, onStreamClose);
+
+        case 2:
+          // prettier-ignore
+          return stream1
+            .pipe(new PassThrough())
+            .pipe(stream2, onStreamClose)
+            .pipe(new PassThrough())
+            .pipe(stream1, onStreamClose);
+
+        case 3:
+          // prettier-ignore
+          return stream1
+            .pipe(createWireTap('send'))
+            .pipe(stream2, onStreamClose)
+            .pipe(createWireTap('recv'))
+            .pipe(stream1, onStreamClose);
+      }
+    };
+
+    const pipeline = createPipeline(3);
+    expect(core1.stats.peers).to.have.lengthOf(1);
+    expect(core2.stats.peers).to.have.lengthOf(1);
+
+    //
+    // Sync
+    //
+
+    const numMessages = 10;
+    const [synced, setSynced] = latch();
+    core2.on('sync', () => {
+      log('sync', { send: core1.stats.totals, recv: core1.stats.totals });
+      expect(core2.length).to.eq(10);
+      setSynced();
+    });
+
+    Array.from(Array(numMessages)).map((_, i) => core1.append(`test-${i}: ${faker.lorem.sentence()}`, noop));
+    await synced();
+
+    //
+    // Close
+    //
+
+    {
+      pipeline.finalize();
+      await pipelineClosed();
+
+      const [feedsClosed, done] = latch({ count: 2 });
+      core1.close(done);
+      core2.close(done);
+      await feedsClosed();
+
+      expect(core1.closed).to.be.true;
+      expect(core2.closed).to.be.true;
+    }
+  });
+});