@dxos/hypercore 0.1.0

package/src/multiplexer.ts

@@ -0,0 +1,122 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import assert from 'assert';
+import { Hypercore } from 'hypercore';
+import { ProtocolStream } from 'hypercore-protocol';
+import { PassThrough, Transform, Writable } from 'streamx';
+
+import { latch, Trigger } from '@dxos/async';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { ComplexMap } from '@dxos/util';
+
+/**
+ * Hypercore multiplexer.
+ */
+// TODO(burdon): Add encoding.
+export class Multiplexer {
+  private readonly _feeds = new ComplexMap<PublicKey, Hypercore>(PublicKey.hash);
+  private readonly _streams = new ComplexMap<PublicKey, ProtocolStream>(PublicKey.hash);
+
+  private _isOpen = false;
+  private _closedTrigger = new Trigger({ autoReset: true });
+
+  public readonly input = new PassThrough();
+  public readonly output = new PassThrough();
+
+  // prettier-ignore
+  constructor(
+    private readonly _id: string
+  ) {}
+
+  get isOpen() {
+    return this._isOpen;
+  }
+
+  get feeds() {
+    return Array.from(this._feeds.values());
+  }
+
+  addFeed(core: Hypercore, initiator = false) {
+    const stream = core.replicate(initiator, { live: true });
+
+    const feedKey = PublicKey.from(core.key);
+    this._feeds.set(feedKey, core);
+    this._streams.set(feedKey, stream);
+    log('added', { feedKey: PublicKey.from(feedKey) });
+
+    stream
+      .pipe(
+        new Transform({
+          transform: (data: Buffer, next: (err: Error | null, data: any) => void) => {
+            log('send', { id: this._id, feedKey: PublicKey.from(feedKey), payload: data.length });
+            next(null, {
+              feedKey: core.key,
+              payload: data
+            });
+          }
+        })
+      )
+      .pipe(this.output, () => {
+        log('pipeline closed', { id: this._id });
+        this._isOpen = false;
+        this._closedTrigger.wake();
+      });
+
+    return this;
+  }
+
+  async open() {
+    if (this._isOpen) {
+      return;
+    }
+
+    this.input.pipe(
+      new Writable({
+        write: (data: any, next: () => void) => {
+          const { feedKey, payload } = data;
+          const feedStream = this._streams.get(PublicKey.from(feedKey));
+          assert(feedStream, `invalid stream: ${PublicKey.from(feedKey).truncate()}`);
+          log('recv', { id: this._id, feedKey: PublicKey.from(feedKey), payload: (payload as Buffer).length });
+          feedStream.write(payload);
+          next();
+        }
+      })
+    );
+
+    log('opened');
+    this._isOpen = true;
+  }
+
+  // TODO(burdon): Test if re-entrant?
+  async close() {
+    if (!this._isOpen) {
+      return;
+    }
+
+    log('closing...');
+    const [closed, setClosed] = latch({ count: this._streams.size });
+    Array.from(this._streams.values()).forEach((stream) => {
+      stream.once('end', () => {
+        log('stream closed');
+        setClosed();
+      });
+
+      stream.finalize();
+    });
+
+    await closed();
+
+    // NOTE: Depending on environment, pipeline may close before or after the stream above.
+    if (this._isOpen) {
+      await this._closedTrigger.wait();
+      assert(!this._isOpen);
+    }
+
+    this._feeds.clear();
+    this._streams.clear();
+    log('closed');
+  }
+}

package/src/replicate.test.ts

@@ -0,0 +1,241 @@
+//
+// Copyright 2019 DXOS.org
+//
+
+import { expect } from 'chai';
+import faker from 'faker';
+
+import { latch } from '@dxos/async';
+import { createKeyPair } from '@dxos/crypto';
+import { log } from '@dxos/log';
+
+import { HypercoreFactory } from './hypercore-factory';
+import { createReadable } from './iterator';
+import { batch, createDataItem, TestDataItem } from './testing';
+
+const noop = () => {};
+
+describe('Replication', function () {
+  const factory1 = new HypercoreFactory();
+  const factory2 = new HypercoreFactory();
+
+  it('replicates feeds', async function () {
+    const { publicKey, secretKey } = createKeyPair();
+    const core1 = factory1.createFeed(publicKey, { secretKey });
+    const core2 = factory2.createFeed(publicKey);
+
+    // Open.
+    {
+      const [ready, done] = latch({ count: 2 });
+      core1.open(done);
+      core2.open(done);
+      await ready();
+    }
+
+    const numBlocks = 10;
+
+    // Sync.
+    {
+      const stream1 = core1.replicate(true);
+      const stream2 = core2.replicate(false);
+
+      const [streamsClosed, onClose] = latch({ count: 2 });
+      stream1.pipe(stream2, onClose).pipe(stream1, onClose);
+
+      expect(core1.stats.peers).to.have.lengthOf(1);
+      expect(core2.stats.peers).to.have.lengthOf(1);
+
+      // Wait for complete sync.
+      core2.on('sync', () => {
+        stream1.end();
+        stream2.end();
+      });
+
+      // Write.
+      for (let i = 0; i < numBlocks; i++) {
+        core1.append(`test-${i}`, noop);
+      }
+
+      await streamsClosed();
+    }
+
+    // Close.
+    {
+      const [closed, close] = latch({ count: 2 });
+      core1.close(close);
+      core1.close(close);
+
+      await closed();
+    }
+
+    expect(core1.stats.totals.uploadedBlocks).to.eq(numBlocks);
+    expect(core2.stats.totals.downloadedBlocks).to.eq(numBlocks);
+  });
+
+  it('replicates feeds in batches', async function () {
+    const numBlocks = 100;
+
+    // Replicating feeds must have the same public key.
+    const { publicKey, secretKey } = createKeyPair();
+    const core1 = factory1.createFeed(publicKey, { secretKey });
+    const core2 = factory2.createFeed(publicKey);
+
+    // Open.
+    {
+      const [ready, done] = latch({ count: 2 });
+      core1.open(done);
+      core2.open(done);
+      await ready();
+    }
+
+    const stream1 = core1.replicate(true);
+    const stream2 = core2.replicate(false);
+
+    // Start replication.
+    // NOTE: Replication won't start unless the public keys on both sides are the same.
+    const [streamsClosed, onClose] = latch({ count: 2 });
+    stream1.pipe(stream2, onClose).pipe(stream1, onClose);
+
+    expect(core1.stats.peers).to.have.lengthOf(1);
+    expect(core2.stats.peers).to.have.lengthOf(1);
+
+    // Replicate messages.
+    {
+      // Wait until all uploaded.
+      const [waitForDownloaded, incDownload] = latch({ count: numBlocks });
+
+      // Monitor uploads.
+      let uploaded = 0;
+      core1.on('upload', () => {
+        uploaded++;
+        log('up', { uploaded, length: core1.length });
+      });
+
+      // Monitor downloads.
+      let downloaded = 0;
+      core2.on('download', (seq: number, data: Buffer) => {
+        downloaded++;
+        const { id } = JSON.parse(data.toString()) as TestDataItem;
+        expect(id).to.eq(seq);
+        log('down', { downloaded, length: core2.length });
+        expect(downloaded).to.be.lessThanOrEqual(uploaded);
+        incDownload();
+      });
+
+      // Monitor sync points (multiple since delayed writes).
+      const sync = { started: 0, complete: 0 };
+      core2.on('sync', () => {
+        log('sync', { events: ++sync.started });
+        core2.download();
+      });
+
+      // Write batch of messages with delay.
+      batch((next, i, remaining) => {
+        const size = faker.datatype.number({
+          min: 1,
+          max: Math.min(10, remaining)
+        });
+        for (let j = 0; j < size; j++) {
+          core1.append(JSON.stringify(createDataItem(i + j)), noop);
+        }
+
+        // Random delay.
+        setTimeout(() => {
+          next(size);
+        }, faker.datatype.number({ min: 0, max: 100 }));
+      }, numBlocks);
+
+      // Done.
+      await waitForDownloaded();
+      expect(uploaded).to.eq(downloaded);
+    }
+
+    // Close streams.
+    {
+      stream1.end();
+      stream2.end();
+
+      await streamsClosed();
+      expect(stream1.destroyed).to.be.true;
+      expect(stream2.destroyed).to.be.true;
+    }
+
+    // Close feeds.
+    {
+      await core1.close();
+      await core2.close();
+
+      expect(core1.writable).to.be.false;
+      expect(core1.stats.totals.uploadedBlocks).to.eq(numBlocks);
+      expect(core2.stats.totals.downloadedBlocks).to.eq(numBlocks);
+    }
+  }).timeout(5_000);
+
+  it('replicates feeds with read stream', async function () {
+    const numBlocks = 10;
+
+    const { publicKey, secretKey } = createKeyPair();
+    const core1 = factory1.createFeed(publicKey, { secretKey });
+    const core2 = factory2.createFeed(publicKey);
+
+    // Open.
+    {
+      const [ready, done] = latch({ count: 2 });
+      core1.open(done);
+      core2.open(done);
+      await ready();
+    }
+
+    expect(core1.opened).to.be.true;
+    expect(core2.opened).to.be.true;
+
+    const stream1 = core1.replicate(true, { live: true });
+    const stream2 = core2.replicate(false, { live: true });
+
+    const [done, onClose] = latch({ count: 2 });
+
+    // Start replication.
+    {
+      stream1.pipe(stream2, onClose).pipe(stream1, onClose);
+
+      expect(core1.stats.peers).to.have.lengthOf(1);
+      expect(core2.stats.peers).to.have.lengthOf(1);
+
+      let sync = 0;
+      core2.on('sync', () => {
+        log('sync', { events: ++sync });
+      });
+    }
+
+    // Writer.
+    {
+      setTimeout(async () => {
+        for (const _ of Array.from(Array(numBlocks))) {
+          const block = {
+            text: faker.lorem.sentence()
+          };
+
+          core1.append(JSON.stringify(block), noop);
+        }
+      });
+    }
+
+    // Reader.
+    {
+      const [done, inc] = latch({ count: numBlocks });
+
+      setTimeout(async () => {
+        for await (const _ of createReadable(core2.createReadStream({ live: true }))) {
+          inc();
+        }
+      });
+
+      await done();
+
+      stream1.end();
+      stream2.end();
+    }
+
+    await done();
+  }).timeout(5_000);
+});

package/src/streams.test.ts

@@ -0,0 +1,75 @@
+//
+// Copyright 2019 DXOS.org
+//
+
+import util from 'node:util';
+import { Writable } from 'streamx';
+
+import { latch } from '@dxos/async';
+import { createKeyPair } from '@dxos/crypto';
+import { log } from '@dxos/log';
+
+import { HypercoreFactory } from './hypercore-factory';
+
+describe('Streams', function () {
+  it('reads from stream', async function () {
+    const factory = new HypercoreFactory<string>();
+    const { publicKey, secretKey } = createKeyPair();
+    const core = factory.createFeed(publicKey, { secretKey });
+
+    const numBlocks = 10;
+    const [closed, setClosed] = latch();
+    const [processed, incProcessed] = latch({ count: numBlocks });
+
+    const stream = core.createReadStream({ live: true });
+    const consumer: Writable = stream.pipe(
+      new Writable({
+        write: (data: any, next: () => void) => {
+          log('received', { data: String(data) });
+          incProcessed();
+          next();
+        }
+      })
+    );
+
+    {
+      const append = util.promisify(core.append.bind(core));
+      for (const i of Array.from(Array(numBlocks)).keys()) {
+        await append(`message-${i}`);
+      }
+    }
+
+    await processed();
+
+    {
+      consumer.once('close', () => {
+        log('closed');
+        setClosed();
+      });
+
+      consumer.destroy();
+      await closed();
+    }
+  });
+
+  it('feed closed while stream is open', async function () {
+    const factory = new HypercoreFactory<string>();
+    const { publicKey, secretKey } = createKeyPair();
+    const core = factory.createFeed(publicKey, { secretKey });
+
+    const [closed, setClosed] = latch({ count: 1 });
+    {
+      const stream = core.createReadStream({ live: true });
+      stream.once('close', () => {
+        log('closed');
+        setClosed();
+      });
+
+      stream.destroy();
+    }
+    await closed();
+
+    const close = util.promisify(core.close.bind(core));
+    await close();
+  });
+});

package/src/testing.ts

@@ -0,0 +1,40 @@
+//
+// Copyright 2019 DXOS.org
+//
+
+import faker from 'faker';
+import util from 'node:util';
+
+export const noop = () => {};
+
+export const py = (obj: any, fn: Function) => util.promisify(fn.bind(obj));
+
+// TODO(burdon): Replace with proto def.
+export type TestDataItem = { id: number; text: string };
+
+export const createDataItem = (n: number): TestDataItem => ({
+  id: n,
+  text: faker.lorem.sentence()
+});
+
+type BatchCallback = (next: (num: number) => void, index: number, remaining: number) => void;
+
+// TODO(burdon): Factor out.
+export const batch = (cb: BatchCallback, total: number) => {
+  let i = 0;
+  const f = () => {
+    const remaining = total - i;
+    cb(
+      (num = 1) => {
+        i += num;
+        if (i < total) {
+          f();
+        }
+      },
+      i,
+      remaining
+    );
+  };
+
+  f();
+};