@epfml/discojs-node 2.0.0

package/dist/core/client/decentralized/clear_text.js

@@ -0,0 +1,96 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClearText = void 0;
+var tslib_1 = require("tslib");
+var immutable_1 = require("immutable");
+var __1 = require("../..");
+var base_1 = require("./base");
+var messages_1 = require("../messages");
+var event_connection_1 = require("../event_connection");
+/**
+ * Decentralized client that does not utilize secure aggregation, but sends model updates in clear text
+ */
+var ClearText = /** @class */ (function (_super) {
+    (0, tslib_1.__extends)(ClearText, _super);
+    function ClearText() {
+        return _super !== null && _super.apply(this, arguments) || this;
+    }
+    ClearText.prototype.sendAndReceiveWeights = function (peers, noisyWeights, round, trainingInformant) {
+        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
+            var weights, ret;
+            var _this = this;
+            return (0, tslib_1.__generator)(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.receivedWeights) {
+                            throw new Error('no promise setup for receiving weights');
+                        }
+                        // PHASE 1 COMMUNICATION --> create weights message and send to all peers (only one phase of communication for clear_text)
+                        // send weights asynchronously
+                        __1.serialization.weights.encode(noisyWeights).then(function (encodedWeights) {
+                            // create weights message and send to all peers
+                            peers.forEach(function (peer, id) {
+                                return _this.sendMessagetoPeer(peer, {
+                                    type: messages_1.type.Weights,
+                                    peer: id,
+                                    weights: encodedWeights
+                                });
+                            });
+                        }).catch(function () {
+                            throw new Error('error while sending weights');
+                        });
+                        return [4 /*yield*/, this.receivedWeights];
+                    case 1:
+                        weights = _a.sent();
+                        trainingInformant.update({
+                            currentNumberOfParticipants: weights.size + 1
+                        });
+                        ret = weights.push(noisyWeights);
+                        this.receivedWeights = undefined;
+                        return [2 /*return*/, ret];
+                }
+            });
+        });
+    };
+    ClearText.prototype.receiveWeights = function (peers) {
+        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
+            var waitWeights, receivedWeights;
+            var _this = this;
+            return (0, tslib_1.__generator)(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        console.debug('beginning of receiveWeights');
+                        waitWeights = Array.from(peers.values()).map(function (peer) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () {
+                            return (0, tslib_1.__generator)(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0: return [4 /*yield*/, (0, event_connection_1.waitMessage)(peer, messages_1.type.Weights)];
+                                    case 1: return [2 /*return*/, _a.sent()];
+                                }
+                            });
+                        }); });
+                        receivedWeights = (0, immutable_1.List)();
+                        return [4 /*yield*/, Promise.allSettled(waitWeights)];
+                    case 1: return [4 /*yield*/, (_a.sent()).forEach(function (message) {
+                            if (message.status === 'fulfilled') {
+                                receivedWeights = receivedWeights.push(__1.serialization.weights.decode(message.value.weights));
+                            }
+                        })];
+                    case 2:
+                        _a.sent();
+                        if (receivedWeights.size < peers.size) {
+                            throw new Error('not enough peer weights received');
+                        }
+                        return [2 /*return*/, receivedWeights];
+                }
+            });
+        });
+    };
+    /*
+      handles received messages from signaling server
+   */
+    ClearText.prototype.clientHandle = function (peers) {
+        this.receivedWeights = this.receiveWeights(peers);
+    };
+    return ClearText;
+}(base_1.Base));
+exports.ClearText = ClearText;

package/dist/core/client/decentralized/index.d.ts

@@ -0,0 +1,4 @@
+export { ClearText } from './clear_text';
+export { SecAgg } from './sec_agg';
+export * as messages from './messages';
+export { PeerID } from './types';

package/dist/core/client/decentralized/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.messages = exports.SecAgg = exports.ClearText = void 0;
+var tslib_1 = require("tslib");
+var clear_text_1 = require("./clear_text");
+Object.defineProperty(exports, "ClearText", { enumerable: true, get: function () { return clear_text_1.ClearText; } });
+var sec_agg_1 = require("./sec_agg");
+Object.defineProperty(exports, "SecAgg", { enumerable: true, get: function () { return sec_agg_1.SecAgg; } });
+exports.messages = (0, tslib_1.__importStar)(require("./messages"));

package/dist/core/client/decentralized/messages.d.ts

@@ -0,0 +1,41 @@
+import { SignalData } from 'simple-peer';
+import { weights } from '../../serialization';
+import { PeerID as PeerIDType } from './types';
+import { type, clientConnected } from '../messages';
+export interface PeerID {
+    type: type.PeerID;
+    id: PeerIDType;
+}
+export interface SignalForPeer {
+    type: type.SignalForPeer;
+    peer: PeerIDType;
+    signal: SignalData;
+}
+export interface PeerIsReady {
+    type: type.PeerIsReady;
+}
+export interface PeersForRound {
+    type: type.PeersForRound;
+    peers: PeerIDType[];
+}
+export interface Weights {
+    type: type.Weights;
+    peer: PeerIDType;
+    weights: weights.Encoded;
+}
+export interface Shares {
+    type: type.Shares;
+    peer: PeerIDType;
+    weights: weights.Encoded;
+}
+export interface PartialSums {
+    type: type.PartialSums;
+    peer: PeerIDType;
+    partials: weights.Encoded;
+}
+export declare type MessageFromServer = PeerID | SignalForPeer | PeersForRound;
+export declare type MessageToServer = clientConnected | SignalForPeer | PeerIsReady;
+export declare type PeerMessage = Weights | Shares | PartialSums;
+export declare function isMessageFromServer(o: unknown): o is MessageFromServer;
+export declare function isMessageToServer(o: unknown): o is MessageToServer;
+export declare function isPeerMessage(o: unknown): o is PeerMessage;

package/dist/core/client/decentralized/messages.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isPeerMessage = exports.isMessageToServer = exports.isMessageFromServer = void 0;
+var serialization_1 = require("../../serialization");
+var types_1 = require("./types");
+var messages_1 = require("../messages");
+function isMessageFromServer(o) {
+    if (!(0, messages_1.hasMessageType)(o)) {
+        return false;
+    }
+    switch (o.type) {
+        case messages_1.type.PeerID:
+            return 'id' in o && (0, types_1.isPeerID)(o.id);
+        case messages_1.type.SignalForPeer:
+            return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
+                'signal' in o; // TODO check signal content?
+        case messages_1.type.PeersForRound:
+            return 'peers' in o && Array.isArray(o.peers) && o.peers.every(types_1.isPeerID);
+    }
+    return false;
+}
+exports.isMessageFromServer = isMessageFromServer;
+function isMessageToServer(o) {
+    if (!(0, messages_1.hasMessageType)(o)) {
+        return false;
+    }
+    switch (o.type) {
+        case messages_1.type.clientConnected:
+            return true;
+        case messages_1.type.SignalForPeer:
+            return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
+                'signal' in o; // TODO check signal content?
+        case messages_1.type.PeerIsReady:
+            return true;
+    }
+    return false;
+}
+exports.isMessageToServer = isMessageToServer;
+function isPeerMessage(o) {
+    if (!(0, messages_1.hasMessageType)(o)) {
+        return false;
+    }
+    switch (o.type) {
+        case messages_1.type.Weights:
+        case messages_1.type.Shares:
+            return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
+                'weights' in o && serialization_1.weights.isEncoded(o.weights);
+        case messages_1.type.PartialSums:
+            return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
+                'partials' in o && serialization_1.weights.isEncoded(o.partials);
+    }
+    return false;
+}
+exports.isPeerMessage = isPeerMessage;

package/dist/core/client/decentralized/peer.d.ts

@@ -0,0 +1,26 @@
+/// <reference types="node" />
+import SimplePeer, { SignalData } from 'simple-peer';
+import { PeerID } from './types';
+interface Events {
+    'close': () => void;
+    'connect': () => void;
+    'signal': (signal: SignalData) => void;
+    'data': (data: Buffer) => void;
+}
+export declare class Peer {
+    readonly id: PeerID;
+    private readonly peer;
+    private bufferSize?;
+    private sendCounter;
+    private sendQueue;
+    private receiving;
+    constructor(id: PeerID, opts?: SimplePeer.Options);
+    send(msg: Buffer): void;
+    private flush;
+    get maxChunkSize(): number;
+    private chunk;
+    destroy(): void;
+    signal(signal: SimplePeer.SignalData): void;
+    on<K extends keyof Events>(event: K, listener: Events[K]): void;
+}
+export {};

package/dist/core/client/decentralized/peer.js

@@ -0,0 +1,210 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Peer = void 0;
+var tslib_1 = require("tslib");
+var immutable_1 = require("immutable");
+var simple_peer_1 = (0, tslib_1.__importDefault)(require("simple-peer"));
+// message id + (chunk counter == 0) + chunk count
+var FIRST_HEADER_SIZE = 2 + 1 + 1;
+// message id + chunk counter
+var HEADER_SIZE = 2 + 1;
+// at which interval to poll
+var TICK = 10;
+// Peer wraps a SimplePeer, adding message fragmentation
+//
+// WebRTC implementations have various maximum message size
+// but with huge models, our messages might be bigger.
+// We split messages by chunks and reconstruct theses
+// on the other side.
+//
+// As the WebRTC's DataChannel is not a stream, we need
+// reorder messages, so we use a header on each chunk
+// with a message id and chunk counter. The first chunk
+// (chunk counter == 0), also add the total number of chunk.
+//
+// see feross/simple-peer#393 for more info
+var Peer = /** @class */ (function () {
+    function Peer(id, opts) {
+        this.sendCounter = 0;
+        this.sendQueue = (0, immutable_1.List)();
+        this.receiving = (0, immutable_1.Map)();
+        this.id = id;
+        this.peer = new simple_peer_1.default(opts);
+    }
+    Peer.prototype.send = function (msg) {
+        console.debug('sending message of size', msg.length);
+        var chunks = this.chunk(msg);
+        this.sendQueue = this.sendQueue.concat(chunks);
+        this.flush();
+    };
+    Peer.prototype.flush = function () {
+        var _this = this;
+        if (this.bufferSize === undefined) {
+            throw new Error('flush without known buffer size');
+        }
+        var chunk = this.sendQueue.first();
+        if (chunk === undefined) {
+            return; // nothing to flush
+        }
+        var remainingBufferSize = this.bufferSize - this.peer.bufferSize;
+        if (chunk.length > remainingBufferSize) {
+            setTimeout(function () { return _this.flush(); }, TICK);
+            return;
+        }
+        console.debug('sending chunk of size', chunk.length);
+        this.sendQueue = this.sendQueue.shift();
+        this.peer.send(chunk);
+        // and loop
+        this.flush();
+    };
+    Object.defineProperty(Peer.prototype, "maxChunkSize", {
+        get: function () {
+            if (this.bufferSize === undefined) {
+                throw new Error('chunk without known buffer size');
+            }
+            // in the perfect world of bug-free implementations
+            // we would return this.bufferSize
+            // sadly, we are not there yet
+            //
+            // based on MDN, taking 16K seems to be a pretty safe
+            // and widely supported buffer size
+            return 16 * (1 << 10);
+        },
+        enumerable: false,
+        configurable: true
+    });
+    Peer.prototype.chunk = function (b) {
+        var _this = this;
+        var messageID = this.sendCounter;
+        this.sendCounter++;
+        if (this.sendCounter > 0xFFFF) {
+            throw new Error('too much messages sent to this peer');
+        }
+        // special case as Range(1, 0) yields a value
+        var tail = immutable_1.Seq.Indexed([]);
+        if (b.length > this.maxChunkSize) {
+            tail = (0, immutable_1.Range)(this.maxChunkSize - FIRST_HEADER_SIZE, b.length, this.maxChunkSize - HEADER_SIZE).map(function (offset) { return b.subarray(offset, offset + _this.maxChunkSize - HEADER_SIZE); });
+        }
+        var totalChunkCount = 1 + tail.count();
+        if (totalChunkCount > 0xFF) {
+            throw new Error('too big message to even chunk it');
+        }
+        var firstChunk = Buffer.alloc((b.length > this.maxChunkSize - FIRST_HEADER_SIZE)
+            ? this.maxChunkSize
+            : FIRST_HEADER_SIZE + b.length);
+        firstChunk.writeUint16BE(messageID);
+        firstChunk.writeUint8(0, 2);
+        firstChunk.writeUint8(totalChunkCount, 3);
+        b.copy(firstChunk, FIRST_HEADER_SIZE, 0, this.maxChunkSize - FIRST_HEADER_SIZE);
+        return immutable_1.Seq.Indexed([firstChunk])
+            .concat((0, immutable_1.Range)(1).zip(tail)
+            .map(function (_a) {
+            var _b = (0, tslib_1.__read)(_a, 2), id = _b[0], raw = _b[1];
+            var chunk = Buffer.alloc(HEADER_SIZE + raw.length);
+            chunk.writeUint16BE(messageID);
+            chunk.writeUint8(id, 2);
+            raw.copy(chunk, HEADER_SIZE, 0);
+            return chunk;
+        }));
+    };
+    Peer.prototype.destroy = function () {
+        this.peer.destroy();
+    };
+    Peer.prototype.signal = function (signal) {
+        // extract max buffer size
+        if (signal.type === 'offer' || signal.type === 'answer') {
+            if (signal.sdp === undefined) {
+                throw new Error('signal answer|offer without session description');
+            }
+            if (this.bufferSize !== undefined) {
+                throw new Error('buffer size set twice');
+            }
+            var match = signal.sdp.match(/a=max-message-size:(\d+)/);
+            if (match === null) {
+                // TODO default value instead?
+                throw new Error('no max-message-size found in signal');
+            }
+            var max = parseInt(match[1], 10);
+            if (isNaN(max)) {
+                throw new Error("unable to parse max-message-size as int: " + match[1]);
+            }
+            this.bufferSize = max;
+        }
+        this.peer.signal(signal);
+    };
+    Peer.prototype.on = function (event, listener) {
+        var _this = this;
+        if (event !== 'data') {
+            this.peer.on(event, listener);
+            return;
+        }
+        this.peer.on('data', function (data) {
+            if (!Buffer.isBuffer(data) || data.length < HEADER_SIZE) {
+                throw new Error('received invalid message type');
+            }
+            var messageID = data.readUint16BE();
+            var chunkID = data.readUint8(2);
+            var received = _this.receiving.get(messageID, {
+                total: undefined,
+                chunks: (0, immutable_1.Map)()
+            });
+            var total = received.total;
+            var chunks = received.chunks;
+            if (chunks.has(chunkID)) {
+                throw new Error("chunk " + messageID + ":" + chunkID + " already received");
+            }
+            var chunk;
+            if (chunkID !== 0) {
+                chunk = Buffer.alloc(data.length - HEADER_SIZE);
+                data.copy(chunk, 0, HEADER_SIZE);
+            }
+            else {
+                if (data.length < FIRST_HEADER_SIZE) {
+                    throw new Error('received invalid message type');
+                }
+                if (total !== undefined) {
+                    throw new Error('first header received twice');
+                }
+                var readTotal_1 = data.readUint8(3);
+                total = readTotal_1;
+                chunk = Buffer.alloc(data.length - FIRST_HEADER_SIZE);
+                data.copy(chunk, 0, FIRST_HEADER_SIZE);
+                if (chunks.keySeq().some(function (id) { return id > readTotal_1; })) {
+                    throw new Error('received total of chunk but got now-out-of-bound chunks');
+                }
+            }
+            _this.receiving = _this.receiving.set(messageID, {
+                total: total,
+                chunks: chunks.set(chunkID, chunk)
+            });
+            console.debug("got chunk " + messageID + ":" + chunkID + "/" + (total !== null && total !== void 0 ? total : 'unknown') + " of size " + chunk.length);
+            var readyMessages = _this.receiving
+                .filter(function (_a) {
+                var total = _a.total, chunks = _a.chunks;
+                return total !== undefined && chunks.size === total;
+            })
+                .sort()
+                .map(function (_a) {
+                var chunks = _a.chunks;
+                return chunks.entrySeq().toList().sortBy(function (_a) {
+                    var _b = (0, tslib_1.__read)(_a, 2), id = _b[0], _ = _b[1];
+                    return id;
+                });
+            })
+                .map(function (chunks) { return Buffer.concat(chunks.map(function (_a) {
+                var _b = (0, tslib_1.__read)(_a, 2), _ = _b[0], b = _b[1];
+                return b;
+            }).toArray()); });
+            _this.receiving = _this.receiving.deleteAll(readyMessages.keys());
+            readyMessages
+                .forEach(function (message) {
+                console.debug(_this.peer.address().port, 'recved message of size', message.length);
+                // TODO debug
+                // @ts-expect-error
+                listener(message);
+            });
+        });
+    };
+    return Peer;
+}());
+exports.Peer = Peer;

package/dist/core/client/decentralized/peer_pool.d.ts

@@ -0,0 +1,14 @@
+import { Map, Set } from 'immutable';
+import { SignalData } from 'simple-peer';
+import { PeerID } from './types';
+import { PeerConnection, EventConnection } from '../event_connection';
+export declare class PeerPool {
+    private readonly id;
+    private readonly wrtc?;
+    private peers;
+    private constructor();
+    static init(id: PeerID): Promise<PeerPool>;
+    shutdown(): void;
+    signal(peerID: PeerID, signal: SignalData): void;
+    getPeers(peersToConnect: Set<PeerID>, signallingServer: EventConnection, clientHandle: (connections: Map<PeerID, PeerConnection>) => void): Promise<Map<PeerID, PeerConnection>>;
+}

package/dist/core/client/decentralized/peer_pool.js

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PeerPool = void 0;
+var tslib_1 = require("tslib");
+var immutable_1 = require("immutable");
+var peer_1 = require("./peer");
+var event_connection_1 = require("../event_connection");
+// TODO cleanup old peers
+var PeerPool = /** @class */ (function () {
+    function PeerPool(id, wrtc) {
+        this.id = id;
+        this.wrtc = wrtc;
+        this.peers = (0, immutable_1.Map)();
+    }
+    PeerPool.init = function (id) {
+        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
+            var wrtc, path, e_1;
+            return (0, tslib_1.__generator)(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        _a.trys.push([0, 2, , 3]);
+                        path = require.resolve('@koush/wrtc', { paths: ['.'] });
+                        return [4 /*yield*/, Promise.resolve().then(function () { return (0, tslib_1.__importStar)(require(path)); })];
+                    case 1:
+                        wrtc = _a.sent();
+                        return [3 /*break*/, 3];
+                    case 2:
+                        e_1 = _a.sent();
+                        return [3 /*break*/, 3];
+                    case 3: return [2 /*return*/, new PeerPool(id, wrtc)];
+                }
+            });
+        });
+    };
+    PeerPool.prototype.shutdown = function () {
+        console.debug(this.id, 'shutdown their peers');
+        this.peers.forEach(function (peer) { return peer.disconnect(); });
+        this.peers = (0, immutable_1.Map)();
+    };
+    PeerPool.prototype.signal = function (peerID, signal) {
+        console.debug(this.id, 'signals for', peerID);
+        var peer = this.peers.get(peerID);
+        if (peer === undefined) {
+            throw new Error("received signal for unknown peer: " + peerID);
+        }
+        peer.signal(signal);
+    };
+    PeerPool.prototype.getPeers = function (peersToConnect, signallingServer, 
+    // TODO as event?
+    clientHandle) {
+        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
+            var newPeers, newPeersConnections;
+            var _this = this;
+            return (0, tslib_1.__generator)(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (peersToConnect.contains(this.id)) {
+                            throw new Error('peers to connect contains our id');
+                        }
+                        console.debug(this.id, 'is connecting peers:', peersToConnect.toJS());
+                        newPeers = (0, immutable_1.Map)(peersToConnect
+                            .filter(function (id) { return !_this.peers.has(id); })
+                            .map(function (id) { return [id, id < _this.id]; })
+                            .map(function (_a) {
+                            var _b = (0, tslib_1.__read)(_a, 2), id = _b[0], initiator = _b[1];
+                            var p = new peer_1.Peer(id, { initiator: initiator, wrtc: _this.wrtc });
+                            // onNewPeer(id, p)
+                            return [id, p];
+                        }));
+                        console.debug(this.id, 'asked to connect new peers:', newPeers.keySeq().toJS());
+                        newPeersConnections = newPeers.map(function (peer, id) { return new event_connection_1.PeerConnection(_this.id, peer, signallingServer); });
+                        // adding peers to pool before connecting them because they must be set to call signal on them
+                        this.peers = this.peers.merge(newPeersConnections);
+                        clientHandle(this.peers);
+                        return [4 /*yield*/, Promise.all(Array.from(newPeersConnections.values()).map(function (connection) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () { return (0, tslib_1.__generator)(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0: return [4 /*yield*/, connection.connect()];
+                                    case 1: return [2 /*return*/, _a.sent()];
+                                }
+                            }); }); }))];
+                    case 1:
+                        _a.sent();
+                        console.debug(this.id, 'knowns connected peers:', this.peers.keySeq().toJS());
+                        return [2 /*return*/, this.peers
+                                .filter(function (_, id) { return peersToConnect.has(id); })];
+                }
+            });
+        });
+    };
+    return PeerPool;
+}());
+exports.PeerPool = PeerPool;

package/dist/core/client/decentralized/sec_agg.d.ts

@@ -0,0 +1,22 @@
+import { List, Map } from 'immutable';
+import { Task, TrainingInformant, WeightsContainer } from '../..';
+import { Base } from './base';
+import { PeerID } from './types';
+import { PeerConnection } from '../event_connection';
+/**
+ * Decentralized client that utilizes secure aggregation so client updates remain private
+ */
+export declare class SecAgg extends Base {
+    readonly url: URL;
+    readonly task: Task;
+    private readonly maxShareValue;
+    private receivedShares?;
+    private receivedPartialSums?;
+    constructor(url: URL, task: Task);
+    private sendShares;
+    private sendPartialSums;
+    sendAndReceiveWeights(peers: Map<PeerID, PeerConnection>, noisyWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<List<WeightsContainer>>;
+    private receiveShares;
+    private receivePartials;
+    clientHandle(peers: Map<PeerID, PeerConnection>): void;
+}