@dxos/hypercore 0.1.0

package/dist/src/replicate.test.js

@@ -0,0 +1,197 @@
+"use strict";
+//
+// Copyright 2019 DXOS.org
+//
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const chai_1 = require("chai");
+const faker_1 = __importDefault(require("faker"));
+const async_1 = require("@dxos/async");
+const crypto_1 = require("@dxos/crypto");
+const log_1 = require("@dxos/log");
+const hypercore_factory_1 = require("./hypercore-factory");
+const iterator_1 = require("./iterator");
+const testing_1 = require("./testing");
+const noop = () => { };
+describe('Replication', function () {
+    const factory1 = new hypercore_factory_1.HypercoreFactory();
+    const factory2 = new hypercore_factory_1.HypercoreFactory();
+    it('replicates feeds', async function () {
+        const { publicKey, secretKey } = (0, crypto_1.createKeyPair)();
+        const core1 = factory1.createFeed(publicKey, { secretKey });
+        const core2 = factory2.createFeed(publicKey);
+        // Open.
+        {
+            const [ready, done] = (0, async_1.latch)({ count: 2 });
+            core1.open(done);
+            core2.open(done);
+            await ready();
+        }
+        const numBlocks = 10;
+        // Sync.
+        {
+            const stream1 = core1.replicate(true);
+            const stream2 = core2.replicate(false);
+            const [streamsClosed, onClose] = (0, async_1.latch)({ count: 2 });
+            stream1.pipe(stream2, onClose).pipe(stream1, onClose);
+            (0, chai_1.expect)(core1.stats.peers).to.have.lengthOf(1);
+            (0, chai_1.expect)(core2.stats.peers).to.have.lengthOf(1);
+            // Wait for complete sync.
+            core2.on('sync', () => {
+                stream1.end();
+                stream2.end();
+            });
+            // Write.
+            for (let i = 0; i < numBlocks; i++) {
+                core1.append(`test-${i}`, noop);
+            }
+            await streamsClosed();
+        }
+        // Close.
+        {
+            const [closed, close] = (0, async_1.latch)({ count: 2 });
+            core1.close(close);
+            core1.close(close);
+            await closed();
+        }
+        (0, chai_1.expect)(core1.stats.totals.uploadedBlocks).to.eq(numBlocks);
+        (0, chai_1.expect)(core2.stats.totals.downloadedBlocks).to.eq(numBlocks);
+    });
+    it('replicates feeds in batches', async function () {
+        const numBlocks = 100;
+        // Replicating feeds must have the same public key.
+        const { publicKey, secretKey } = (0, crypto_1.createKeyPair)();
+        const core1 = factory1.createFeed(publicKey, { secretKey });
+        const core2 = factory2.createFeed(publicKey);
+        // Open.
+        {
+            const [ready, done] = (0, async_1.latch)({ count: 2 });
+            core1.open(done);
+            core2.open(done);
+            await ready();
+        }
+        const stream1 = core1.replicate(true);
+        const stream2 = core2.replicate(false);
+        // Start replication.
+        // NOTE: Replication won't start unless the public keys on both sides are the same.
+        const [streamsClosed, onClose] = (0, async_1.latch)({ count: 2 });
+        stream1.pipe(stream2, onClose).pipe(stream1, onClose);
+        (0, chai_1.expect)(core1.stats.peers).to.have.lengthOf(1);
+        (0, chai_1.expect)(core2.stats.peers).to.have.lengthOf(1);
+        // Replicate messages.
+        {
+            // Wait until all uploaded.
+            const [waitForDownloaded, incDownload] = (0, async_1.latch)({ count: numBlocks });
+            // Monitor uploads.
+            let uploaded = 0;
+            core1.on('upload', () => {
+                uploaded++;
+                (0, log_1.log)('up', { uploaded, length: core1.length }, { file: "packages/common/hypercore/src/replicate.test.ts", line: 111 });
+            });
+            // Monitor downloads.
+            let downloaded = 0;
+            core2.on('download', (seq, data) => {
+                downloaded++;
+                const { id } = JSON.parse(data.toString());
+                (0, chai_1.expect)(id).to.eq(seq);
+                (0, log_1.log)('down', { downloaded, length: core2.length }, { file: "packages/common/hypercore/src/replicate.test.ts", line: 120 });
+                (0, chai_1.expect)(downloaded).to.be.lessThanOrEqual(uploaded);
+                incDownload();
+            });
+            // Monitor sync points (multiple since delayed writes).
+            const sync = { started: 0, complete: 0 };
+            core2.on('sync', () => {
+                (0, log_1.log)('sync', { events: ++sync.started }, { file: "packages/common/hypercore/src/replicate.test.ts", line: 128 });
+                core2.download();
+            });
+            // Write batch of messages with delay.
+            (0, testing_1.batch)((next, i, remaining) => {
+                const size = faker_1.default.datatype.number({
+                    min: 1,
+                    max: Math.min(10, remaining)
+                });
+                for (let j = 0; j < size; j++) {
+                    core1.append(JSON.stringify((0, testing_1.createDataItem)(i + j)), noop);
+                }
+                // Random delay.
+                setTimeout(() => {
+                    next(size);
+                }, faker_1.default.datatype.number({ min: 0, max: 100 }));
+            }, numBlocks);
+            // Done.
+            await waitForDownloaded();
+            (0, chai_1.expect)(uploaded).to.eq(downloaded);
+        }
+        // Close streams.
+        {
+            stream1.end();
+            stream2.end();
+            await streamsClosed();
+            (0, chai_1.expect)(stream1.destroyed).to.be.true;
+            (0, chai_1.expect)(stream2.destroyed).to.be.true;
+        }
+        // Close feeds.
+        {
+            await core1.close();
+            await core2.close();
+            (0, chai_1.expect)(core1.writable).to.be.false;
+            (0, chai_1.expect)(core1.stats.totals.uploadedBlocks).to.eq(numBlocks);
+            (0, chai_1.expect)(core2.stats.totals.downloadedBlocks).to.eq(numBlocks);
+        }
+    }).timeout(5000);
+    it('replicates feeds with read stream', async function () {
+        const numBlocks = 10;
+        const { publicKey, secretKey } = (0, crypto_1.createKeyPair)();
+        const core1 = factory1.createFeed(publicKey, { secretKey });
+        const core2 = factory2.createFeed(publicKey);
+        // Open.
+        {
+            const [ready, done] = (0, async_1.latch)({ count: 2 });
+            core1.open(done);
+            core2.open(done);
+            await ready();
+        }
+        (0, chai_1.expect)(core1.opened).to.be.true;
+        (0, chai_1.expect)(core2.opened).to.be.true;
+        const stream1 = core1.replicate(true, { live: true });
+        const stream2 = core2.replicate(false, { live: true });
+        const [done, onClose] = (0, async_1.latch)({ count: 2 });
+        // Start replication.
+        {
+            stream1.pipe(stream2, onClose).pipe(stream1, onClose);
+            (0, chai_1.expect)(core1.stats.peers).to.have.lengthOf(1);
+            (0, chai_1.expect)(core2.stats.peers).to.have.lengthOf(1);
+            let sync = 0;
+            core2.on('sync', () => {
+                (0, log_1.log)('sync', { events: ++sync }, { file: "packages/common/hypercore/src/replicate.test.ts", line: 206 });
+            });
+        }
+        // Writer.
+        {
+            setTimeout(async () => {
+                for (const _ of Array.from(Array(numBlocks))) {
+                    const block = {
+                        text: faker_1.default.lorem.sentence()
+                    };
+                    core1.append(JSON.stringify(block), noop);
+                }
+            });
+        }
+        // Reader.
+        {
+            const [done, inc] = (0, async_1.latch)({ count: numBlocks });
+            setTimeout(async () => {
+                for await (const _ of (0, iterator_1.createReadable)(core2.createReadStream({ live: true }))) {
+                    inc();
+                }
+            });
+            await done();
+            stream1.end();
+            stream2.end();
+        }
+        await done();
+    }).timeout(5000);
+});
+//# sourceMappingURL=replicate.test.js.map

package/dist/src/replicate.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replicate.test.js","sourceRoot":"","sources":["../../src/replicate.test.ts"],"names":[],"mappings":";AAAA,EAAE;AACF,0BAA0B;AAC1B,EAAE;;;;;AAEF,+BAA8B;AAC9B,kDAA0B;AAE1B,uCAAoC;AACpC,yCAA6C;AAC7C,mCAAgC;AAEhC,2DAAuD;AACvD,yCAA4C;AAC5C,uCAAgE;AAEhE,MAAM,IAAI,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;AAEtB,QAAQ,CAAC,aAAa,EAAE;IACtB,MAAM,QAAQ,GAAG,IAAI,oCAAgB,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,oCAAgB,EAAE,CAAC;IAExC,EAAE,CAAC,kBAAkB,EAAE,KAAK;QAC1B,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAA,sBAAa,GAAE,CAAC;QACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE7C,QAAQ;QACR;YACE,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,MAAM,KAAK,EAAE,CAAC;SACf;QAED,MAAM,SAAS,GAAG,EAAE,CAAC;QAErB,QAAQ;QACR;YACE,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAEvC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAEtD,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC9C,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAE9C,0BAA0B;YAC1B,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACpB,OAAO,CAAC,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,CAAC,CAAC,CAAC;YAEH,SAAS;YACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;gBAClC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;aACjC;YAED,MAAM,aAAa,EAAE,CAAC;SACvB;QAED,SAAS;QACT;YACE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5C,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAEnB,MAAM,MAAM,EAAE,CAAC;SAChB;QAED,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3D,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK;QACrC,MAAM,SAAS,GAAG,GAAG,CAAC;QAEtB,mDAAmD;QACnD,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAA,sBAAa,GAAE,CAAC;QACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE7C,QAAQ;QACR;YACE,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,MAAM,KAAK,EAAE,CAAC;SACf;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAEvC,qBAAqB;QACrB,mFAAmF;QACnF,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAEtD,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAE9C,sBAAsB;QACtB;YACE,2BAA2B;YAC3B,MAAM,CAAC,iBAAiB,EAAE,WAAW,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YAErE,mBAAmB;YACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;gBACtB,QAAQ,EAAE,CAAC;gBACX,IAAA,SAAG,EAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,yEAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YAEH,qBAAqB;YACrB,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;gBACjD,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAiB,CAAC;gBAC3D,IAAA,aAAM,EAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;gBACtB,IAAA,SAAG,EAAC,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,yEAAC,CAAC;gBAClD,IAAA,aAAM,EAAC,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;gBACnD,WAAW,EAAE,CAAC;YAChB,CAAC,CAAC,CAAC;YAEH,uDAAuD;YACvD,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YACzC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACpB,IAAA,SAAG,EAAC,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,yEAAC,CAAC;gBACxC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,sCAAsC;YACtC,IAAA,eAAK,EAAC,CAAC,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE;gBAC3B,MAAM,IAAI,GAAG,eAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;oBACjC,GAAG,EAAE,CAAC;oBACN,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC;iBAC7B,CAAC,CAAC;gBACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE;oBAC7B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAA,wBAAc,EAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;iBAC3D;gBAED,gBAAgB;gBAChB,UAAU,CAAC,GAAG,EAAE;oBACd,IAAI,CAAC,IAAI,CAAC,CAAC;gBACb,CAAC,EAAE,eAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC,EAAE,SAAS,CAAC,CAAC;YAEd,QAAQ;YACR,MAAM,iBAAiB,EAAE,CAAC;YAC1B,IAAA,aAAM,EAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;SACpC;QAED,iBAAiB;QACjB;YACE,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,MAAM,aAAa,EAAE,CAAC;YACtB,IAAA,aAAM,EAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;YACrC,IAAA,aAAM,EAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;SACtC;QAED,eAAe;QACf;YACE,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YAEpB,IAAA,aAAM,EAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC;YACnC,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;YAC3D,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;SAC9D;IACH,CAAC,CAAC,CAAC,OAAO,CAAC,IAAK,CAAC,CAAC;IAElB,EAAE,CAAC,mCAAmC,EAAE,KAAK;QAC3C,MAAM,SAAS,GAAG,EAAE,CAAC;QAErB,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAA,sBAAa,GAAE,CAAC;QACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAE7C,QAAQ;QACR;YACE,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,MAAM,KAAK,EAAE,CAAC;SACf;QAED,IAAA,aAAM,EAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;QAChC,IAAA,aAAM,EAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC;QAEhC,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvD,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAE5C,qBAAqB;QACrB;YACE,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAEtD,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC9C,IAAA,aAAM,EAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAE9C,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACpB,IAAA,SAAG,EAAC,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,yEAAC,CAAC;YAClC,CAAC,CAAC,CAAC;SACJ;QAED,UAAU;QACV;YACE,UAAU,CAAC,KAAK,IAAI,EAAE;gBACpB,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE;oBAC5C,MAAM,KAAK,GAAG;wBACZ,IAAI,EAAE,eAAK,CAAC,KAAK,CAAC,QAAQ,EAAE;qBAC7B,CAAC;oBAEF,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;iBAC3C;YACH,CAAC,CAAC,CAAC;SACJ;QAED,UAAU;QACV;YACE,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YAEhD,UAAU,CAAC,KAAK,IAAI,EAAE;gBACpB,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,IAAA,yBAAc,EAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;oBAC5E,GAAG,EAAE,CAAC;iBACP;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,EAAE,CAAC;YAEb,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,EAAE,CAAC;SACf;QAED,MAAM,IAAI,EAAE,CAAC;IACf,CAAC,CAAC,CAAC,OAAO,CAAC,IAAK,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC"}

package/dist/src/streams.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streams.test.d.ts.map

package/dist/src/streams.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streams.test.d.ts","sourceRoot":"","sources":["../../src/streams.test.ts"],"names":[],"mappings":""}

package/dist/src/streams.test.js

@@ -0,0 +1,65 @@
+"use strict";
+//
+// Copyright 2019 DXOS.org
+//
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_util_1 = __importDefault(require("node:util"));
+const streamx_1 = require("streamx");
+const async_1 = require("@dxos/async");
+const crypto_1 = require("@dxos/crypto");
+const log_1 = require("@dxos/log");
+const hypercore_factory_1 = require("./hypercore-factory");
+describe('Streams', function () {
+    it('reads from stream', async function () {
+        const factory = new hypercore_factory_1.HypercoreFactory();
+        const { publicKey, secretKey } = (0, crypto_1.createKeyPair)();
+        const core = factory.createFeed(publicKey, { secretKey });
+        const numBlocks = 10;
+        const [closed, setClosed] = (0, async_1.latch)();
+        const [processed, incProcessed] = (0, async_1.latch)({ count: numBlocks });
+        const stream = core.createReadStream({ live: true });
+        const consumer = stream.pipe(new streamx_1.Writable({
+            write: (data, next) => {
+                (0, log_1.log)('received', { data: String(data) }, { file: "packages/common/hypercore/src/streams.test.ts", line: 28 });
+                incProcessed();
+                next();
+            }
+        }));
+        {
+            const append = node_util_1.default.promisify(core.append.bind(core));
+            for (const i of Array.from(Array(numBlocks)).keys()) {
+                await append(`message-${i}`);
+            }
+        }
+        await processed();
+        {
+            consumer.once('close', () => {
+                (0, log_1.log)('closed', {}, { file: "packages/common/hypercore/src/streams.test.ts", line: 46 });
+                setClosed();
+            });
+            consumer.destroy();
+            await closed();
+        }
+    });
+    it('feed closed while stream is open', async function () {
+        const factory = new hypercore_factory_1.HypercoreFactory();
+        const { publicKey, secretKey } = (0, crypto_1.createKeyPair)();
+        const core = factory.createFeed(publicKey, { secretKey });
+        const [closed, setClosed] = (0, async_1.latch)({ count: 1 });
+        {
+            const stream = core.createReadStream({ live: true });
+            stream.once('close', () => {
+                (0, log_1.log)('closed', {}, { file: "packages/common/hypercore/src/streams.test.ts", line: 64 });
+                setClosed();
+            });
+            stream.destroy();
+        }
+        await closed();
+        const close = node_util_1.default.promisify(core.close.bind(core));
+        await close();
+    });
+});
+//# sourceMappingURL=streams.test.js.map

package/dist/src/streams.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streams.test.js","sourceRoot":"","sources":["../../src/streams.test.ts"],"names":[],"mappings":";AAAA,EAAE;AACF,0BAA0B;AAC1B,EAAE;;;;;AAEF,0DAA6B;AAC7B,qCAAmC;AAEnC,uCAAoC;AACpC,yCAA6C;AAC7C,mCAAgC;AAEhC,2DAAuD;AAEvD,QAAQ,CAAC,SAAS,EAAE;IAClB,EAAE,CAAC,mBAAmB,EAAE,KAAK;QAC3B,MAAM,OAAO,GAAG,IAAI,oCAAgB,EAAU,CAAC;QAC/C,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAA,sBAAa,GAAE,CAAC;QACjD,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAE1D,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,IAAA,aAAK,GAAE,CAAC;QACpC,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAa,MAAM,CAAC,IAAI,CACpC,IAAI,kBAAQ,CAAC;YACX,KAAK,EAAE,CAAC,IAAS,EAAE,IAAgB,EAAE,EAAE;gBACrC,IAAA,SAAG,EAAC,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,sEAAC,CAAC;gBACxC,YAAY,EAAE,CAAC;gBACf,IAAI,EAAE,CAAC;YACT,CAAC;SACF,CAAC,CACH,CAAC;QAEF;YACE,MAAM,MAAM,GAAG,mBAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACtD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE;gBACnD,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;aAC9B;SACF;QAED,MAAM,SAAS,EAAE,CAAC;QAElB;YACE,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE;gBAC1B,IAAA,SAAG,EAAC,QAAQ,0EAAC,CAAC;gBACd,SAAS,EAAE,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,MAAM,EAAE,CAAC;SAChB;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK;QAC1C,MAAM,OAAO,GAAG,IAAI,oCAAgB,EAAU,CAAC;QAC/C,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAA,sBAAa,GAAE,CAAC;QACjD,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAE1D,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,IAAA,aAAK,EAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAChD;YACE,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE;gBACxB,IAAA,SAAG,EAAC,QAAQ,0EAAC,CAAC;gBACd,SAAS,EAAE,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,OAAO,EAAE,CAAC;SAClB;QACD,MAAM,MAAM,EAAE,CAAC;QAEf,MAAM,KAAK,GAAG,mBAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,MAAM,KAAK,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/testing.d.ts

@@ -0,0 +1,11 @@
+export declare const noop: () => void;
+export declare const py: (obj: any, fn: Function) => Function;
+export declare type TestDataItem = {
+    id: number;
+    text: string;
+};
+export declare const createDataItem: (n: number) => TestDataItem;
+declare type BatchCallback = (next: (num: number) => void, index: number, remaining: number) => void;
+export declare const batch: (cb: BatchCallback, total: number) => void;
+export {};
+//# sourceMappingURL=testing.d.ts.map

package/dist/src/testing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../../src/testing.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,IAAI,YAAW,CAAC;AAE7B,eAAO,MAAM,EAAE,QAAS,GAAG,MAAM,QAAQ,aAAiC,CAAC;AAG3E,oBAAY,YAAY,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAExD,eAAO,MAAM,cAAc,MAAO,MAAM,KAAG,YAGzC,CAAC;AAEH,aAAK,aAAa,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;AAG7F,eAAO,MAAM,KAAK,OAAQ,aAAa,SAAS,MAAM,SAiBrD,CAAC"}

package/dist/src/testing.js

@@ -0,0 +1,36 @@
+"use strict";
+//
+// Copyright 2019 DXOS.org
+//
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.batch = exports.createDataItem = exports.py = exports.noop = void 0;
+const faker_1 = __importDefault(require("faker"));
+const node_util_1 = __importDefault(require("node:util"));
+const noop = () => { };
+exports.noop = noop;
+const py = (obj, fn) => node_util_1.default.promisify(fn.bind(obj));
+exports.py = py;
+const createDataItem = (n) => ({
+    id: n,
+    text: faker_1.default.lorem.sentence()
+});
+exports.createDataItem = createDataItem;
+// TODO(burdon): Factor out.
+const batch = (cb, total) => {
+    let i = 0;
+    const f = () => {
+        const remaining = total - i;
+        cb((num = 1) => {
+            i += num;
+            if (i < total) {
+                f();
+            }
+        }, i, remaining);
+    };
+    f();
+};
+exports.batch = batch;
+//# sourceMappingURL=testing.js.map

package/dist/src/testing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.js","sourceRoot":"","sources":["../../src/testing.ts"],"names":[],"mappings":";AAAA,EAAE;AACF,0BAA0B;AAC1B,EAAE;;;;;;AAEF,kDAA0B;AAC1B,0DAA6B;AAEtB,MAAM,IAAI,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;AAAhB,QAAA,IAAI,QAAY;AAEtB,MAAM,EAAE,GAAG,CAAC,GAAQ,EAAE,EAAY,EAAE,EAAE,CAAC,mBAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAA9D,QAAA,EAAE,MAA4D;AAKpE,MAAM,cAAc,GAAG,CAAC,CAAS,EAAgB,EAAE,CAAC,CAAC;IAC1D,EAAE,EAAE,CAAC;IACL,IAAI,EAAE,eAAK,CAAC,KAAK,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAHU,QAAA,cAAc,kBAGxB;AAIH,4BAA4B;AACrB,MAAM,KAAK,GAAG,CAAC,EAAiB,EAAE,KAAa,EAAE,EAAE;IACxD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,CAAC,GAAG,GAAG,EAAE;QACb,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC;QAC5B,EAAE,CACA,CAAC,GAAG,GAAG,CAAC,EAAE,EAAE;YACV,CAAC,IAAI,GAAG,CAAC;YACT,IAAI,CAAC,GAAG,KAAK,EAAE;gBACb,CAAC,EAAE,CAAC;aACL;QACH,CAAC,EACD,CAAC,EACD,SAAS,CACV,CAAC;IACJ,CAAC,CAAC;IAEF,CAAC,EAAE,CAAC;AACN,CAAC,CAAC;AAjBW,QAAA,KAAK,SAiBhB"}

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@dxos/hypercore",
+  "version": "0.1.0",
+  "description": "Hypercore wrapper and utils.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "license": "MIT",
+  "author": "DXOS.org",
+  "main": "./dist/src/index.js",
+  "browser": {
+    "node:assert": "assert",
+    "node:util": "util"
+  },
+  "types": "./dist/src/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@dxos/async": "0.1.0",
+    "@dxos/codec-protobuf": "0.1.0",
+    "@dxos/crypto": "0.1.0",
+    "@dxos/debug": "0.1.0",
+    "@dxos/keys": "0.1.0",
+    "@dxos/random-access-storage": "0.1.0",
+    "assert": "^2.0.0",
+    "hypercore": "^9.12.0",
+    "hypercore-protocol": "^8.0.7",
+    "readable-stream": "^3.6.0",
+    "streamx": "^2.12.5",
+    "util": "^0.12.4"
+  },
+  "devDependencies": {
+    "@dxos/log": "0.1.0",
+    "@dxos/util": "0.1.0",
+    "@types/debug": "^4.1.7",
+    "@types/faker": "^5.5.9",
+    "@types/randombytes": "^2.0.0",
+    "@types/readable-stream": "^2.3.9",
+    "debug": "^4.3.3",
+    "faker": "^5.5.3",
+    "random-access-memory": "^4.1.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "check": "true"
+  }
+}

package/src/crypto.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import assert from 'assert';
+import { AbstractValueEncoding, Crypto } from 'hypercore';
+import { callbackify } from 'node:util';
+
+import { Codec } from '@dxos/codec-protobuf';
+import { Signer, verifySignature } from '@dxos/crypto';
+import { PublicKey } from '@dxos/keys';
+
+/**
+ * Create encoding (e.g., from protobuf codec).
+ */
+export const createCodecEncoding = <T>(codec: Codec<T>): AbstractValueEncoding<T> => ({
+  encode: (obj: T) => Buffer.from(codec.encode(obj)),
+  decode: (buffer: Buffer) => codec.decode(buffer)
+});
+
+/**
+ * Create a custom hypercore crypto signer.
+ */
+// TODO(burdon): Create test without adding deps.
+export const createCrypto = (signer: Signer, publicKey: PublicKey): Crypto => {
+  assert(signer);
+  assert(publicKey);
+
+  return {
+    sign: (message, secretKey, cb) => {
+      callbackify(signer.sign.bind(signer!))(publicKey, message, (err, result) => {
+        if (err) {
+          cb(err, null);
+          return;
+        }
+
+        cb(null, Buffer.from(result));
+      });
+    },
+
+    verify: async (message, signature, key, cb) => {
+      // NOTE: Uses the public key passed into function.
+      callbackify(verifySignature)(publicKey, message, signature, cb);
+    }
+  };
+};

package/src/defaults.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import type { HypercoreOptions, ReadStreamOptions, ReplicationOptions, WriteStreamOptions } from 'hypercore';
+
+/**
+ * https://github.com/hypercore-protocol/hypercore/tree/v9.12.0#var-feed--hypercorestorage-key-options
+ */
+export const defaultFeedOptions: HypercoreOptions = {
+  createIfMissing: true,
+  valueEncoding: 'binary'
+};
+
+/**
+ * https://github.com/hypercore-protocol/hypercore/tree/v9.12.0#var-stream--feedcreatereadstreamoptions
+ */
+export const defaultReadStreamOptions: ReadStreamOptions = {
+  start: 0,
+  end: Infinity,
+  snapshot: true,
+  tail: false,
+  live: false,
+  timeout: 0,
+  wait: true,
+  batch: 1
+};
+
+/**
+ * https://github.com/hypercore-protocol/hypercore/tree/v9.12.0#var-stream--feedcreatewritestreamopts
+ */
+export const defaultWriteStreamOptions: WriteStreamOptions = {
+  maxBlockSize: Infinity
+};
+
+/**
+ * https://github.com/hypercore-protocol/hypercore/tree/v9.12.0#var-stream--feedreplicateisinitiator-options
+ */
+export const defaultReplicateOptions: ReplicationOptions = {
+  live: false,
+  ack: false,
+  download: true,
+  upload: true,
+  encrypted: true,
+  noise: true
+};

package/src/hypercore-factory.test.ts

@@ -0,0 +1,53 @@
+//
+// Copyright 2019 DXOS.org
+//
+
+import { expect } from 'chai';
+import faker from 'faker';
+
+import { createKeyPair } from '@dxos/crypto';
+
+import { HypercoreFactory } from './hypercore-factory';
+import { createDataItem, py } from './testing';
+
+describe('HypercoreFactory', function () {
+  it('appends to, and read from, multiple feeds', async function () {
+    const factory = new HypercoreFactory();
+
+    const numFeeds = 10;
+    const numBlocks = 100;
+
+    // Create feeds.
+    const feeds = await Promise.all(
+      Array.from({ length: numFeeds }).map(async () => {
+        const { publicKey, secretKey } = createKeyPair();
+        return await factory.openFeed(publicKey, { secretKey });
+      })
+    );
+
+    // Write data.
+    {
+      const data = Array.from({ length: numBlocks }).map((_, i) => createDataItem(i));
+      for await (const datum of data) {
+        const feed = faker.random.arrayElement(feeds);
+        await py(feed, feed.append)(Buffer.from(JSON.stringify(datum)));
+      }
+    }
+
+    // Test.
+    {
+      const total = feeds.reduce((result, feed) => result + feed.length, 0);
+      expect(total).to.eq(numBlocks);
+
+      const feed = faker.random.arrayElement(feeds);
+      const block1 = await py(feed, feed.head)();
+      const block2 = await py(feed, feed.get)(feed.length - 1);
+      expect(block1.toString()).to.eq(block2.toString());
+
+      const blocks = await py(feed, feed.getBatch)(0, feed.length);
+      expect(blocks.length).to.eq(feed.length);
+    }
+
+    await Promise.all(feeds.map((feed) => feed.close()));
+  });
+});

package/src/hypercore-factory.ts

@@ -0,0 +1,44 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import assert from 'assert';
+import hypercore from 'hypercore';
+import type { Hypercore, HypercoreOptions } from 'hypercore';
+
+import { createStorage, Directory, StorageType } from '@dxos/random-access-storage';
+
+import { py } from './testing';
+
+/**
+ * Creates feeds with default properties.
+ */
+export class HypercoreFactory<T> {
+  // prettier-ignore
+  constructor(
+    private readonly _root: Directory = createStorage({ type: StorageType.RAM }).createDirectory(),
+    private readonly _options?: HypercoreOptions
+  ) {
+    assert(this._root);
+  }
+
+  /**
+   * Creates a feed using a storage factory prefixed with the feed's key.
+   * NOTE: We have to use our `random-access-storage` implementation since the native ones
+   * do not behave uniformly across platforms.
+   */
+  createFeed(publicKey: Buffer, options?: HypercoreOptions): Hypercore<T> {
+    const directory = this._root.createDirectory(publicKey.toString());
+    const storage = (filename: string) => directory.getOrCreateFile(filename).native;
+    return hypercore(storage, publicKey, Object.assign({}, this._options, options));
+  }
+
+  /**
+   * Creates and opens a feed.
+   */
+  async openFeed(publicKey: Buffer, options?: HypercoreOptions): Promise<Hypercore<T>> {
+    const feed = this.createFeed(publicKey, options);
+    await py(feed, feed.open)(); // TODO(burdon): Sometimes strange bug if done inside function.
+    return feed;
+  }
+}