@epfml/discojs 0.1.0 → 2.0.0

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "@epfml/discojs",
-  "version": "0.1.0",
+  "version": "2.0.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
-    "build": "tsc",
-    "test": "mocha"
+    "build": "cp ./src/core/index.browser.ts ./src/core/index.ts && tsc",
+    "build-win": "copy ..\\discojs-core\\src\\index.node.ts ..\\discojs-core\\src\\index.ts && tsc",
+    "test": "cp ./src/core/index.node.ts ./src/core/index.ts && mocha",
+    "lint": "cp ./src/core/index.browser.ts ./src/core/index.ts && npx eslint --max-warnings 0 --ignore-pattern '*.spec.ts' --ignore-pattern 'src/core' ."
   },
   "repository": {
     "type": "git",
@@ -16,32 +18,17 @@
   },
   "homepage": "https://github.com/epfml/disco#readme",
   "dependencies": {
-    "axios": "0.27",
-    "immutable": "4",
-    "isomorphic-ws": "4",
-    "lodash": "4",
     "msgpack-lite": "0.1",
-    "simple-peer": "9",
+    "immutable": "4",
+    "@tensorflow/tfjs": "4",
     "tslib": "2",
+    "isomorphic-ws": "4",
     "url": "0.11",
-    "uuid": "8",
-    "ws": "8"
-  },
-  "devDependencies": {
-    "@tensorflow/tfjs-node": "3",
-    "@types/chai": "4",
-    "@types/lodash": "4",
-    "@types/mocha": "9",
+    "@koush/wrtc": "0.5",
+    "axios": "0.27",
     "@types/msgpack-lite": "0.1",
-    "@types/simple-peer": "9",
-    "@types/uuid": "8",
-    "@typescript-eslint/eslint-plugin": "4",
-    "@typescript-eslint/parser": "4",
-    "chai": "4",
-    "eslint": "7",
-    "eslint-config-standard-with-typescript": "21",
-    "mocha": "9",
-    "ts-node": "10",
-    "typescript": "<4.5.0"
+    "uuid": "8",
+    "ws": "8",
+    "simple-peer": "9"
   }
 }

package/dist/aggregation.d.ts

@@ -1,5 +0,0 @@
-import { List } from 'immutable';
-import { Weights } from './types';
-export declare function sumWeights(peersWeights: List<Weights>): Weights;
-export declare function subtractWeights(peersWeights: List<Weights>): Weights;
-export declare function averageWeights(peersWeights: List<Weights>): Weights;

package/dist/aggregation.js

@@ -1,33 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.averageWeights = exports.subtractWeights = exports.sumWeights = void 0;
-var tslib_1 = require("tslib");
-var tf = (0, tslib_1.__importStar)(require("@tensorflow/tfjs"));
-function applyArithmeticToWeights(peersWeights, tfjsArithmeticFunction) {
-    var _a;
-    console.log('Aggregating a list of', peersWeights.size, 'weight vectors.');
-    var firstWeightSize = (_a = peersWeights.first()) === null || _a === void 0 ? void 0 : _a.length;
-    if (firstWeightSize === undefined) {
-        throw new Error('no weights to average');
-    }
-    if (!peersWeights.rest().every(function (ws) { return ws.length === firstWeightSize; })) {
-        throw new Error('weights dimensions are different for some of the summands');
-    }
-    var peersAverageWeights = peersWeights.reduce(function (accum, weights) {
-        return accum.map(function (w, i) { return tfjsArithmeticFunction(w, weights[i]); });
-    });
-    return peersAverageWeights;
-}
-function sumWeights(peersWeights) {
-    return applyArithmeticToWeights(peersWeights, tf.add);
-}
-exports.sumWeights = sumWeights;
-function subtractWeights(peersWeights) {
-    return applyArithmeticToWeights(peersWeights, tf.sub);
-}
-exports.subtractWeights = subtractWeights;
-function averageWeights(peersWeights) {
-    var numberOfPeers = peersWeights.size;
-    return sumWeights(peersWeights).map(function (w) { return w.div(numberOfPeers); });
-}
-exports.averageWeights = averageWeights;

package/dist/client/decentralized/base.d.ts

@@ -1,43 +0,0 @@
-/// <reference types="node" />
-import { List } from 'immutable';
-import isomorphic from 'isomorphic-ws';
-import { URL } from 'url';
-import { Task } from '@/task';
-import { TrainingInformant, Weights } from '../..';
-import { Base as ClientBase } from '../base';
-import * as messages from './messages';
-import { PeerID } from './types';
-/**
- * Abstract class for decentralized clients, executes onRoundEndCommunication as well as connecting
- * to the signaling server
- */
-export declare abstract class Base extends ClientBase {
-    readonly url: URL;
-    readonly task: Task;
-    protected minimumReadyPeers: number;
-    protected maxShareValue: number;
-    constructor(url: URL, task: Task);
-    protected server?: isomorphic.WebSocket;
-    protected peers: PeerID[];
-    protected peersLocked: boolean;
-    protected ID: number;
-    protected pauseUntil(condition: () => boolean): Promise<void>;
-    protected sendMessagetoPeer(message: unknown): void;
-    protected sendReadyMessage(round: number): void;
-    private instanceOfMessageGeneral;
-    private instanceOfServerClientIDMessage;
-    private instanceOfServerReadyClients;
-    protected connectServer(url: URL): Promise<isomorphic.WebSocket>;
-    /**
-     * Initialize the connection to the peers and to the other nodes.
-     */
-    connect(): Promise<void>;
-    /**
-     * Disconnection process when user quits the task.
-     */
-    disconnect(): Promise<void>;
-    onTrainEndCommunication(_: Weights, trainingInformant: TrainingInformant): Promise<void>;
-    onRoundEndCommunication(updatedWeights: Weights, staleWeights: Weights, round: number, trainingInformant: TrainingInformant): Promise<Weights>;
-    abstract sendAndReceiveWeights(noisyWeights: Weights, round: number, trainingInformant: TrainingInformant): Promise<List<Weights>>;
-    abstract clientHandle(msg: messages.messageGeneral): void;
-}

package/dist/client/decentralized/base.js

@@ -1,243 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Base = void 0;
-var tslib_1 = require("tslib");
-var isomorphic_ws_1 = (0, tslib_1.__importDefault)(require("isomorphic-ws"));
-var msgpack_lite_1 = (0, tslib_1.__importDefault)(require("msgpack-lite"));
-var url_1 = require("url");
-var __1 = require("../..");
-var base_1 = require("../base");
-var messages = (0, tslib_1.__importStar)(require("./messages"));
-// Time to wait between network checks in milliseconds.
-var TICK = 100;
-// Time to wait for the others in milliseconds.
-var MAX_WAIT_PER_ROUND = 10000;
-/**
- * Abstract class for decentralized clients, executes onRoundEndCommunication as well as connecting
- * to the signaling server
- */
-var Base = /** @class */ (function (_super) {
-    (0, tslib_1.__extends)(Base, _super);
-    function Base(url, task) {
-        var _a, _b, _c, _d;
-        var _this = _super.call(this, url, task) || this;
-        _this.url = url;
-        _this.task = task;
-        // list of peerIDs who the client will send messages to
-        _this.peers = [];
-        _this.peersLocked = false;
-        // the ID of the client, set arbitrarily to 0 but gets set an actual value once it cues the signaling server
-        // that it is ready to connect
-        _this.ID = 0;
-        _this.minimumReadyPeers = (_b = (_a = _this.task.trainingInformation) === null || _a === void 0 ? void 0 : _a.minimumReadyPeers) !== null && _b !== void 0 ? _b : 3;
-        _this.maxShareValue = (_d = (_c = _this.task.trainingInformation) === null || _c === void 0 ? void 0 : _c.maxShareValue) !== null && _d !== void 0 ? _d : 100;
-        return _this;
-    }
-    /*
-  function to check if a given boolean condition is true, checks continuously until maxWait time is reached
-   */
-    Base.prototype.pauseUntil = function (condition) {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            return (0, tslib_1.__generator)(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, new Promise(function (resolve, reject) {
-                            var timeWas = new Date().getTime();
-                            var wait = setInterval(function () {
-                                if (condition()) {
-                                    console.log('resolved after', new Date().getTime() - timeWas, 'ms');
-                                    clearInterval(wait);
-                                    resolve();
-                                }
-                                else if (new Date().getTime() - timeWas > MAX_WAIT_PER_ROUND) { // Timeout
-                                    console.log('rejected after', new Date().getTime() - timeWas, 'ms');
-                                    clearInterval(wait);
-                                    reject(new Error('timeout'));
-                                }
-                            }, TICK);
-                        })];
-                    case 1: return [2 /*return*/, _a.sent()];
-                }
-            });
-        });
-    };
-    /*
-    function behavior will change with peer 2 peer, sends message to its destination, currently through server
-     */
-    Base.prototype.sendMessagetoPeer = function (message) {
-        if (this.server === undefined) {
-            throw new Error("Undefined Server, can't send message");
-        }
-        this.server.send(message);
-    };
-    /*
-    send message to server that client is ready
-     */
-    Base.prototype.sendReadyMessage = function (round) {
-        // Broadcast our readiness
-        var msg = { type: messages.messageType.clientReadyMessage, round: round };
-        var encodedMsg = msgpack_lite_1.default.encode(msg);
-        if (this.server === undefined) {
-            throw new Error('server undefined, could not connect peers');
-        }
-        this.server.send(encodedMsg);
-    };
-    /*
-    checks if message is of type messageGeneral. If it is, the specific message.type can be identified.
-     */
-    Base.prototype.instanceOfMessageGeneral = function (msg) {
-        return typeof msg === 'object' && msg !== null && 'type' in msg;
-    };
-    /*
-    checks if message contains the client's ID number
-     */
-    Base.prototype.instanceOfServerClientIDMessage = function (msg) {
-        return msg.type === messages.messageType.serverClientIDMessage;
-    };
-    /*
-    checks if message contains the list of peerIDs that are ready to share updates
-     */
-    Base.prototype.instanceOfServerReadyClients = function (msg) {
-        return msg.type === messages.messageType.serverReadyClients;
-    };
-    /*
-    creation of the websocket for the server, connection of client to that webSocket,
-    deals with message reception from decentralized client perspective (messages received by client)
-     */
-    Base.prototype.connectServer = function (url) {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var ws;
-            var _this = this;
-            return (0, tslib_1.__generator)(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        ws = new isomorphic_ws_1.default.WebSocket(url);
-                        ws.binaryType = 'arraybuffer';
-                        ws.onmessage = function (event) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () {
-                            var msg;
-                            return (0, tslib_1.__generator)(this, function (_a) {
-                                if (!(event.data instanceof ArrayBuffer)) {
-                                    throw new Error('server did not send an ArrayBuffer');
-                                }
-                                msg = msgpack_lite_1.default.decode(new Uint8Array(event.data));
-                                // check message type to choose correct action
-                                if (this.instanceOfMessageGeneral(msg)) {
-                                    if (this.instanceOfServerClientIDMessage(msg)) {
-                                        // updated ID
-                                        this.ID = msg.peerID;
-                                    }
-                                    else if (this.instanceOfServerReadyClients(msg)) {
-                                        // updated connected peers
-                                        if (!this.peersLocked) {
-                                            this.peers = msg.peerList;
-                                            this.peersLocked = true;
-                                        }
-                                    }
-                                    else {
-                                        this.clientHandle(msg);
-                                    }
-                                }
-                                return [2 /*return*/];
-                            });
-                        }); };
-                        return [4 /*yield*/, new Promise(function (resolve, reject) {
-                                ws.onerror = function (err) {
-                                    return reject(new Error("connecting server: " + err.message));
-                                }; // eslint-disable-line @typescript-eslint/restrict-template-expressions
-                                ws.onopen = function () { return resolve(ws); };
-                            })];
-                    case 1: return [2 /*return*/, _a.sent()];
-                }
-            });
-        });
-    };
-    /**
-     * Initialize the connection to the peers and to the other nodes.
-     */
-    Base.prototype.connect = function () {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var serverURL, _a;
-            return (0, tslib_1.__generator)(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        serverURL = new url_1.URL('', this.url.href);
-                        switch (this.url.protocol) {
-                            case 'http:':
-                                serverURL.protocol = 'ws:';
-                                break;
-                            case 'https:':
-                                serverURL.protocol = 'wss:';
-                                break;
-                            default:
-                                throw new Error("unknown protocol: " + this.url.protocol);
-                        }
-                        serverURL.pathname += "deai/" + this.task.taskID;
-                        _a = this;
-                        return [4 /*yield*/, this.connectServer(serverURL)];
-                    case 1:
-                        _a.server = _b.sent();
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    /**
-     * Disconnection process when user quits the task.
-     */
-    Base.prototype.disconnect = function () {
-        var _a;
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            return (0, tslib_1.__generator)(this, function (_b) {
-                // this.peers.forEach((peer) => peer.destroy())
-                // this.peers = Map()
-                (_a = this.server) === null || _a === void 0 ? void 0 : _a.close();
-                this.server = undefined;
-                return [2 /*return*/];
-            });
-        });
-    };
-    Base.prototype.onTrainEndCommunication = function (_, trainingInformant) {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            return (0, tslib_1.__generator)(this, function (_a) {
-                // TODO: enter seeding mode?
-                trainingInformant.addMessage('Training finished.');
-                return [2 /*return*/];
-            });
-        });
-    };
-    Base.prototype.onRoundEndCommunication = function (updatedWeights, staleWeights, round, trainingInformant) {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var noisyWeights, finalWeights, Error_1;
-            var _this = this;
-            return (0, tslib_1.__generator)(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        _a.trys.push([0, 3, , 4]);
-                        // reset peer list at each round of training to make sure client waits for updated peerList from server
-                        this.peers = [];
-                        this.peersLocked = false;
-                        // centralized phase of communication --> client tells server that they have finished a local round and are ready to aggregate
-                        this.sendReadyMessage(round);
-                        // wait for peers to be connected before sending any update information
-                        return [4 /*yield*/, this.pauseUntil(function () { return _this.peers.length >= _this.minimumReadyPeers; })
-                            // Apply DP to updates that will be sent
-                        ];
-                    case 1:
-                        // wait for peers to be connected before sending any update information
-                        _a.sent();
-                        noisyWeights = __1.privacy.addDifferentialPrivacy(updatedWeights, staleWeights, this.task);
-                        return [4 /*yield*/, this.sendAndReceiveWeights(noisyWeights, round, trainingInformant)];
-                    case 2:
-                        finalWeights = _a.sent();
-                        return [2 /*return*/, __1.aggregation.averageWeights(finalWeights)];
-                    case 3:
-                        Error_1 = _a.sent();
-                        console.log('Timeout Error Reported, training will continue');
-                        return [2 /*return*/, updatedWeights];
-                    case 4: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    return Base;
-}(base_1.Base));
-exports.Base = Base;

package/dist/client/decentralized/clear_text.d.ts

@@ -1,13 +0,0 @@
-import { List } from 'immutable';
-import { TrainingInformant, Weights } from '../..';
-import { Base } from './base';
-import * as messages from './messages';
-/**
- * Decentralized client that does not utilize secure aggregation, but sends model updates in clear text
- */
-export declare class ClearText extends Base {
-    protected receivedWeights: List<Weights>;
-    sendAndReceiveWeights(noisyWeights: Weights, round: number, trainingInformant: TrainingInformant): Promise<List<Weights>>;
-    private instanceOfClientWeightsMessageServer;
-    clientHandle(msg: messages.messageGeneral): void;
-}

package/dist/client/decentralized/clear_text.js

@@ -1,78 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ClearText = void 0;
-var tslib_1 = require("tslib");
-var immutable_1 = require("immutable");
-var msgpack_lite_1 = (0, tslib_1.__importDefault)(require("msgpack-lite"));
-var __1 = require("../..");
-var base_1 = require("./base");
-var messages = (0, tslib_1.__importStar)(require("./messages"));
-/**
- * Decentralized client that does not utilize secure aggregation, but sends model updates in clear text
- */
-var ClearText = /** @class */ (function (_super) {
-    (0, tslib_1.__extends)(ClearText, _super);
-    function ClearText() {
-        var _this = _super !== null && _super.apply(this, arguments) || this;
-        // list of weights received from other clients
-        _this.receivedWeights = (0, immutable_1.List)();
-        return _this;
-    }
-    ClearText.prototype.sendAndReceiveWeights = function (noisyWeights, round, trainingInformant) {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var weightsToSend, i, msg, encodedMsg;
-            var _this = this;
-            return (0, tslib_1.__generator)(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        // reset received fields at beginning of each round
-                        this.receivedWeights = this.receivedWeights.clear();
-                        return [4 /*yield*/, __1.serialization.weights.encode(noisyWeights)];
-                    case 1:
-                        weightsToSend = _a.sent();
-                        if (this.server === undefined) {
-                            throw new Error('server undefined so we cannot send weights through it');
-                        }
-                        // PHASE 1 COMMUNICATION --> create weights message and send to all peers (only one phase of communication for clear_text)
-                        for (i = 0; i < this.peers.length; i++) {
-                            msg = {
-                                type: messages.messageType.clientWeightsMessageServer,
-                                peerID: this.ID,
-                                weights: weightsToSend,
-                                destination: this.peers[i]
-                            };
-                            encodedMsg = msgpack_lite_1.default.encode(msg);
-                            this.sendMessagetoPeer(encodedMsg);
-                        }
-                        // wait to receive all weights from peers
-                        return [4 /*yield*/, this.pauseUntil(function () { return _this.receivedWeights.size >= _this.peers.length; })];
-                    case 2:
-                        // wait to receive all weights from peers
-                        _a.sent();
-                        return [2 /*return*/, this.receivedWeights];
-                }
-            });
-        });
-    };
-    /*
-    checks if message contains weights from a peer
-     */
-    ClearText.prototype.instanceOfClientWeightsMessageServer = function (msg) {
-        return msg.type === messages.messageType.clientWeightsMessageServer;
-    };
-    /*
-  handles received messages from signaling server
-   */
-    ClearText.prototype.clientHandle = function (msg) {
-        if (this.instanceOfClientWeightsMessageServer(msg)) {
-            // update received weights by one weights reception
-            var weights = __1.serialization.weights.decode(msg.weights);
-            this.receivedWeights = this.receivedWeights.push(weights);
-        }
-        else {
-            throw new Error('Unexpected Message Type');
-        }
-    };
-    return ClearText;
-}(base_1.Base));
-exports.ClearText = ClearText;

package/dist/client/decentralized/messages.d.ts

@@ -1,37 +0,0 @@
-import { weights } from '../../serialization';
-import { PeerID } from './types';
-export declare enum messageType {
-    serverClientIDMessage = 0,
-    clientReadyMessage = 1,
-    serverReadyClients = 2,
-    clientWeightsMessageServer = 3,
-    clientSharesMessageServer = 4,
-    clientPartialSumsMessageServer = 5
-}
-export interface messageGeneral {
-    type: messageType;
-}
-export interface serverClientIDMessage extends messageGeneral {
-    peerID: PeerID;
-}
-export interface clientReadyMessage extends messageGeneral {
-    round: number;
-}
-export interface clientWeightsMessageServer extends messageGeneral {
-    peerID: PeerID;
-    weights: weights.Encoded;
-    destination: PeerID;
-}
-export interface clientSharesMessageServer extends messageGeneral {
-    peerID: PeerID;
-    weights: weights.Encoded;
-    destination: PeerID;
-}
-export interface clientPartialSumsMessageServer extends messageGeneral {
-    peerID: PeerID;
-    partials: weights.Encoded;
-    destination: PeerID;
-}
-export interface serverReadyClients extends messageGeneral {
-    peerList: PeerID[];
-}

package/dist/client/decentralized/messages.js

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.messageType = void 0;
-var messageType;
-(function (messageType) {
-    // Phase 0 communication (just between server and client)
-    messageType[messageType["serverClientIDMessage"] = 0] = "serverClientIDMessage";
-    messageType[messageType["clientReadyMessage"] = 1] = "clientReadyMessage";
-    messageType[messageType["serverReadyClients"] = 2] = "serverReadyClients";
-    // Phase 1 communication (between client and peers)
-    messageType[messageType["clientWeightsMessageServer"] = 3] = "clientWeightsMessageServer";
-    messageType[messageType["clientSharesMessageServer"] = 4] = "clientSharesMessageServer";
-    // Phase 2 communication (between client and peers)
-    messageType[messageType["clientPartialSumsMessageServer"] = 5] = "clientPartialSumsMessageServer";
-})(messageType = exports.messageType || (exports.messageType = {}));

package/dist/client/decentralized/sec_agg.d.ts

@@ -1,18 +0,0 @@
-import { List } from 'immutable';
-import { TrainingInformant, Weights } from '../..';
-import { Base } from './base';
-import * as messages from './messages';
-/**
- * Decentralized client that utilizes secure aggregation so client updates remain private
- */
-export declare class SecAgg extends Base {
-    private receivedShares;
-    private receivedPartialSums;
-    private mySum;
-    private sendShares;
-    private sendPartialSums;
-    sendAndReceiveWeights(noisyWeights: Weights, round: number, trainingInformant: TrainingInformant): Promise<List<Weights>>;
-    private instanceOfClientSharesMessageServer;
-    private instanceOfClientPartialSumsMessageServer;
-    clientHandle(msg: messages.messageGeneral): void;
-}