@epfml/discojs 2.0.0 → 2.1.2-p20240506085037.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aggregator/base.d.ts +180 -0
- package/dist/aggregator/base.js +236 -0
- package/dist/aggregator/get.d.ts +16 -0
- package/dist/aggregator/get.js +31 -0
- package/dist/aggregator/index.d.ts +7 -0
- package/dist/aggregator/index.js +4 -0
- package/dist/aggregator/mean.d.ts +23 -0
- package/dist/aggregator/mean.js +69 -0
- package/dist/aggregator/secure.d.ts +27 -0
- package/dist/aggregator/secure.js +91 -0
- package/dist/async_informant.d.ts +15 -0
- package/dist/async_informant.js +42 -0
- package/dist/client/base.d.ts +76 -0
- package/dist/client/base.js +88 -0
- package/dist/client/decentralized/base.d.ts +32 -0
- package/dist/client/decentralized/base.js +192 -0
- package/dist/client/decentralized/index.d.ts +2 -0
- package/dist/client/decentralized/index.js +2 -0
- package/dist/client/decentralized/messages.d.ts +28 -0
- package/dist/client/decentralized/messages.js +44 -0
- package/dist/client/decentralized/peer.d.ts +40 -0
- package/dist/client/decentralized/peer.js +189 -0
- package/dist/client/decentralized/peer_pool.d.ts +12 -0
- package/dist/client/decentralized/peer_pool.js +44 -0
- package/dist/client/event_connection.d.ts +34 -0
- package/dist/client/event_connection.js +105 -0
- package/dist/client/federated/base.d.ts +54 -0
- package/dist/client/federated/base.js +151 -0
- package/dist/client/federated/index.d.ts +2 -0
- package/dist/client/federated/index.js +2 -0
- package/dist/client/federated/messages.d.ts +30 -0
- package/dist/client/federated/messages.js +24 -0
- package/dist/client/index.d.ts +8 -0
- package/dist/client/index.js +8 -0
- package/dist/client/local.d.ts +3 -0
- package/dist/client/local.js +3 -0
- package/dist/client/messages.d.ts +30 -0
- package/dist/client/messages.js +26 -0
- package/dist/client/types.d.ts +2 -0
- package/dist/client/types.js +4 -0
- package/dist/client/utils.d.ts +2 -0
- package/dist/client/utils.js +7 -0
- package/dist/dataset/data/data.d.ts +48 -0
- package/dist/dataset/data/data.js +72 -0
- package/dist/dataset/data/data_split.d.ts +8 -0
- package/dist/dataset/data/data_split.js +1 -0
- package/dist/dataset/data/image_data.d.ts +11 -0
- package/dist/dataset/data/image_data.js +38 -0
- package/dist/dataset/data/index.d.ts +6 -0
- package/dist/dataset/data/index.js +5 -0
- package/dist/dataset/data/preprocessing/base.d.ts +16 -0
- package/dist/dataset/data/preprocessing/base.js +1 -0
- package/dist/dataset/data/preprocessing/image_preprocessing.d.ts +13 -0
- package/dist/dataset/data/preprocessing/image_preprocessing.js +40 -0
- package/dist/dataset/data/preprocessing/index.d.ts +4 -0
- package/dist/dataset/data/preprocessing/index.js +3 -0
- package/dist/dataset/data/preprocessing/tabular_preprocessing.d.ts +13 -0
- package/dist/dataset/data/preprocessing/tabular_preprocessing.js +45 -0
- package/dist/dataset/data/preprocessing/text_preprocessing.d.ts +13 -0
- package/dist/dataset/data/preprocessing/text_preprocessing.js +85 -0
- package/dist/dataset/data/tabular_data.d.ts +11 -0
- package/dist/dataset/data/tabular_data.js +25 -0
- package/dist/dataset/data/text_data.d.ts +11 -0
- package/dist/dataset/data/text_data.js +14 -0
- package/dist/{core/dataset → dataset}/data_loader/data_loader.d.ts +3 -5
- package/dist/dataset/data_loader/data_loader.js +2 -0
- package/dist/dataset/data_loader/image_loader.d.ts +20 -3
- package/dist/dataset/data_loader/image_loader.js +98 -23
- package/dist/dataset/data_loader/index.d.ts +5 -2
- package/dist/dataset/data_loader/index.js +4 -7
- package/dist/dataset/data_loader/tabular_loader.d.ts +34 -3
- package/dist/dataset/data_loader/tabular_loader.js +75 -15
- package/dist/dataset/data_loader/text_loader.d.ts +14 -0
- package/dist/dataset/data_loader/text_loader.js +25 -0
- package/dist/dataset/dataset.d.ts +5 -0
- package/dist/dataset/dataset.js +1 -0
- package/dist/dataset/dataset_builder.d.ts +60 -0
- package/dist/dataset/dataset_builder.js +142 -0
- package/dist/dataset/index.d.ts +5 -0
- package/dist/dataset/index.js +3 -0
- package/dist/default_tasks/cifar10/index.d.ts +2 -0
- package/dist/default_tasks/cifar10/index.js +60 -0
- package/dist/default_tasks/cifar10/model.d.ts +434 -0
- package/dist/default_tasks/cifar10/model.js +2385 -0
- package/dist/default_tasks/geotags/index.d.ts +2 -0
- package/dist/default_tasks/geotags/index.js +65 -0
- package/dist/default_tasks/geotags/model.d.ts +593 -0
- package/dist/default_tasks/geotags/model.js +4715 -0
- package/dist/default_tasks/index.d.ts +8 -0
- package/dist/default_tasks/index.js +8 -0
- package/dist/default_tasks/lus_covid.d.ts +2 -0
- package/dist/default_tasks/lus_covid.js +89 -0
- package/dist/default_tasks/mnist.d.ts +2 -0
- package/dist/default_tasks/mnist.js +61 -0
- package/dist/default_tasks/simple_face/index.d.ts +2 -0
- package/dist/default_tasks/simple_face/index.js +48 -0
- package/dist/default_tasks/simple_face/model.d.ts +513 -0
- package/dist/default_tasks/simple_face/model.js +4301 -0
- package/dist/default_tasks/skin_mnist.d.ts +2 -0
- package/dist/default_tasks/skin_mnist.js +80 -0
- package/dist/default_tasks/titanic.d.ts +2 -0
- package/dist/default_tasks/titanic.js +88 -0
- package/dist/default_tasks/wikitext.d.ts +2 -0
- package/dist/default_tasks/wikitext.js +38 -0
- package/dist/index.d.ts +18 -2
- package/dist/index.js +18 -6
- package/dist/{core/informant → informant}/graph_informant.d.ts +1 -1
- package/dist/informant/graph_informant.js +20 -0
- package/dist/informant/index.d.ts +1 -0
- package/dist/informant/index.js +1 -0
- package/dist/{core/logging → logging}/console_logger.d.ts +2 -2
- package/dist/logging/console_logger.js +22 -0
- package/dist/logging/index.d.ts +2 -0
- package/dist/logging/index.js +1 -0
- package/dist/{core/logging → logging}/logger.d.ts +3 -3
- package/dist/logging/logger.js +1 -0
- package/dist/memory/base.d.ts +119 -0
- package/dist/memory/base.js +9 -0
- package/dist/memory/empty.d.ts +20 -0
- package/dist/memory/empty.js +43 -0
- package/dist/memory/index.d.ts +3 -1
- package/dist/memory/index.js +3 -5
- package/dist/memory/model_type.d.ts +9 -0
- package/dist/memory/model_type.js +10 -0
- package/dist/{core/privacy.d.ts → privacy.d.ts} +1 -1
- package/dist/{core/privacy.js → privacy.js} +11 -16
- package/dist/serialization/index.d.ts +2 -0
- package/dist/serialization/index.js +2 -0
- package/dist/serialization/model.d.ts +5 -0
- package/dist/serialization/model.js +67 -0
- package/dist/{core/serialization → serialization}/weights.d.ts +2 -2
- package/dist/serialization/weights.js +37 -0
- package/dist/task/data_example.js +14 -0
- package/dist/task/digest.d.ts +5 -0
- package/dist/task/digest.js +14 -0
- package/dist/{core/task → task}/display_information.d.ts +5 -3
- package/dist/task/display_information.js +46 -0
- package/dist/task/index.d.ts +7 -0
- package/dist/task/index.js +5 -0
- package/dist/task/label_type.d.ts +9 -0
- package/dist/task/label_type.js +28 -0
- package/dist/task/summary.js +13 -0
- package/dist/task/task.d.ts +12 -0
- package/dist/task/task.js +22 -0
- package/dist/task/task_handler.d.ts +5 -0
- package/dist/task/task_handler.js +20 -0
- package/dist/task/task_provider.d.ts +5 -0
- package/dist/task/task_provider.js +1 -0
- package/dist/{core/task → task}/training_information.d.ts +9 -10
- package/dist/task/training_information.js +88 -0
- package/dist/training/disco.d.ts +40 -0
- package/dist/training/disco.js +107 -0
- package/dist/training/index.d.ts +2 -0
- package/dist/training/index.js +1 -0
- package/dist/training/trainer/distributed_trainer.d.ts +20 -0
- package/dist/training/trainer/distributed_trainer.js +36 -0
- package/dist/training/trainer/local_trainer.d.ts +12 -0
- package/dist/training/trainer/local_trainer.js +19 -0
- package/dist/training/trainer/trainer.d.ts +33 -0
- package/dist/training/trainer/trainer.js +52 -0
- package/dist/{core/training → training}/trainer/trainer_builder.d.ts +5 -7
- package/dist/training/trainer/trainer_builder.js +43 -0
- package/dist/types.d.ts +8 -0
- package/dist/types.js +1 -0
- package/dist/utils/event_emitter.d.ts +40 -0
- package/dist/utils/event_emitter.js +57 -0
- package/dist/validation/index.d.ts +1 -0
- package/dist/validation/index.js +1 -0
- package/dist/validation/validator.d.ts +28 -0
- package/dist/validation/validator.js +132 -0
- package/dist/weights/aggregation.d.ts +21 -0
- package/dist/weights/aggregation.js +44 -0
- package/dist/weights/index.d.ts +2 -0
- package/dist/weights/index.js +2 -0
- package/dist/weights/weights_container.d.ts +68 -0
- package/dist/weights/weights_container.js +96 -0
- package/package.json +25 -16
- package/README.md +0 -53
- package/dist/core/async_buffer.d.ts +0 -41
- package/dist/core/async_buffer.js +0 -97
- package/dist/core/async_informant.d.ts +0 -20
- package/dist/core/async_informant.js +0 -69
- package/dist/core/client/base.d.ts +0 -33
- package/dist/core/client/base.js +0 -35
- package/dist/core/client/decentralized/base.d.ts +0 -32
- package/dist/core/client/decentralized/base.js +0 -212
- package/dist/core/client/decentralized/clear_text.d.ts +0 -14
- package/dist/core/client/decentralized/clear_text.js +0 -96
- package/dist/core/client/decentralized/index.d.ts +0 -4
- package/dist/core/client/decentralized/index.js +0 -9
- package/dist/core/client/decentralized/messages.d.ts +0 -41
- package/dist/core/client/decentralized/messages.js +0 -54
- package/dist/core/client/decentralized/peer.d.ts +0 -26
- package/dist/core/client/decentralized/peer.js +0 -210
- package/dist/core/client/decentralized/peer_pool.d.ts +0 -14
- package/dist/core/client/decentralized/peer_pool.js +0 -92
- package/dist/core/client/decentralized/sec_agg.d.ts +0 -22
- package/dist/core/client/decentralized/sec_agg.js +0 -190
- package/dist/core/client/decentralized/secret_shares.d.ts +0 -3
- package/dist/core/client/decentralized/secret_shares.js +0 -39
- package/dist/core/client/decentralized/types.d.ts +0 -2
- package/dist/core/client/decentralized/types.js +0 -7
- package/dist/core/client/event_connection.d.ts +0 -37
- package/dist/core/client/event_connection.js +0 -158
- package/dist/core/client/federated/client.d.ts +0 -37
- package/dist/core/client/federated/client.js +0 -273
- package/dist/core/client/federated/index.d.ts +0 -2
- package/dist/core/client/federated/index.js +0 -7
- package/dist/core/client/federated/messages.d.ts +0 -38
- package/dist/core/client/federated/messages.js +0 -25
- package/dist/core/client/index.d.ts +0 -5
- package/dist/core/client/index.js +0 -11
- package/dist/core/client/local.d.ts +0 -8
- package/dist/core/client/local.js +0 -36
- package/dist/core/client/messages.d.ts +0 -28
- package/dist/core/client/messages.js +0 -33
- package/dist/core/client/utils.d.ts +0 -2
- package/dist/core/client/utils.js +0 -19
- package/dist/core/dataset/data/data.d.ts +0 -11
- package/dist/core/dataset/data/data.js +0 -20
- package/dist/core/dataset/data/data_split.d.ts +0 -5
- package/dist/core/dataset/data/data_split.js +0 -2
- package/dist/core/dataset/data/image_data.d.ts +0 -8
- package/dist/core/dataset/data/image_data.js +0 -64
- package/dist/core/dataset/data/index.d.ts +0 -5
- package/dist/core/dataset/data/index.js +0 -11
- package/dist/core/dataset/data/preprocessing.d.ts +0 -13
- package/dist/core/dataset/data/preprocessing.js +0 -33
- package/dist/core/dataset/data/tabular_data.d.ts +0 -8
- package/dist/core/dataset/data/tabular_data.js +0 -40
- package/dist/core/dataset/data_loader/data_loader.js +0 -10
- package/dist/core/dataset/data_loader/image_loader.d.ts +0 -17
- package/dist/core/dataset/data_loader/image_loader.js +0 -141
- package/dist/core/dataset/data_loader/index.d.ts +0 -3
- package/dist/core/dataset/data_loader/index.js +0 -9
- package/dist/core/dataset/data_loader/tabular_loader.d.ts +0 -29
- package/dist/core/dataset/data_loader/tabular_loader.js +0 -101
- package/dist/core/dataset/dataset.d.ts +0 -2
- package/dist/core/dataset/dataset.js +0 -2
- package/dist/core/dataset/dataset_builder.d.ts +0 -18
- package/dist/core/dataset/dataset_builder.js +0 -96
- package/dist/core/dataset/index.d.ts +0 -4
- package/dist/core/dataset/index.js +0 -14
- package/dist/core/index.d.ts +0 -18
- package/dist/core/index.js +0 -41
- package/dist/core/informant/graph_informant.js +0 -23
- package/dist/core/informant/index.d.ts +0 -3
- package/dist/core/informant/index.js +0 -9
- package/dist/core/informant/training_informant/base.d.ts +0 -31
- package/dist/core/informant/training_informant/base.js +0 -83
- package/dist/core/informant/training_informant/decentralized.d.ts +0 -5
- package/dist/core/informant/training_informant/decentralized.js +0 -22
- package/dist/core/informant/training_informant/federated.d.ts +0 -14
- package/dist/core/informant/training_informant/federated.js +0 -32
- package/dist/core/informant/training_informant/index.d.ts +0 -4
- package/dist/core/informant/training_informant/index.js +0 -11
- package/dist/core/informant/training_informant/local.d.ts +0 -6
- package/dist/core/informant/training_informant/local.js +0 -20
- package/dist/core/logging/console_logger.js +0 -33
- package/dist/core/logging/index.d.ts +0 -3
- package/dist/core/logging/index.js +0 -9
- package/dist/core/logging/logger.js +0 -9
- package/dist/core/logging/trainer_logger.d.ts +0 -24
- package/dist/core/logging/trainer_logger.js +0 -59
- package/dist/core/memory/base.d.ts +0 -22
- package/dist/core/memory/base.js +0 -9
- package/dist/core/memory/empty.d.ts +0 -14
- package/dist/core/memory/empty.js +0 -75
- package/dist/core/memory/index.d.ts +0 -3
- package/dist/core/memory/index.js +0 -9
- package/dist/core/memory/model_type.d.ts +0 -4
- package/dist/core/memory/model_type.js +0 -9
- package/dist/core/serialization/index.d.ts +0 -2
- package/dist/core/serialization/index.js +0 -6
- package/dist/core/serialization/model.d.ts +0 -5
- package/dist/core/serialization/model.js +0 -55
- package/dist/core/serialization/weights.js +0 -64
- package/dist/core/task/data_example.js +0 -24
- package/dist/core/task/display_information.js +0 -49
- package/dist/core/task/index.d.ts +0 -3
- package/dist/core/task/index.js +0 -8
- package/dist/core/task/model_compile_data.d.ts +0 -6
- package/dist/core/task/model_compile_data.js +0 -22
- package/dist/core/task/summary.js +0 -19
- package/dist/core/task/task.d.ts +0 -10
- package/dist/core/task/task.js +0 -31
- package/dist/core/task/training_information.js +0 -66
- package/dist/core/tasks/cifar10.d.ts +0 -3
- package/dist/core/tasks/cifar10.js +0 -65
- package/dist/core/tasks/geotags.d.ts +0 -3
- package/dist/core/tasks/geotags.js +0 -67
- package/dist/core/tasks/index.d.ts +0 -6
- package/dist/core/tasks/index.js +0 -10
- package/dist/core/tasks/lus_covid.d.ts +0 -3
- package/dist/core/tasks/lus_covid.js +0 -87
- package/dist/core/tasks/mnist.d.ts +0 -3
- package/dist/core/tasks/mnist.js +0 -60
- package/dist/core/tasks/simple_face.d.ts +0 -2
- package/dist/core/tasks/simple_face.js +0 -41
- package/dist/core/tasks/titanic.d.ts +0 -3
- package/dist/core/tasks/titanic.js +0 -88
- package/dist/core/training/disco.d.ts +0 -23
- package/dist/core/training/disco.js +0 -130
- package/dist/core/training/index.d.ts +0 -2
- package/dist/core/training/index.js +0 -7
- package/dist/core/training/trainer/distributed_trainer.d.ts +0 -20
- package/dist/core/training/trainer/distributed_trainer.js +0 -65
- package/dist/core/training/trainer/local_trainer.d.ts +0 -11
- package/dist/core/training/trainer/local_trainer.js +0 -34
- package/dist/core/training/trainer/round_tracker.d.ts +0 -30
- package/dist/core/training/trainer/round_tracker.js +0 -47
- package/dist/core/training/trainer/trainer.d.ts +0 -65
- package/dist/core/training/trainer/trainer.js +0 -160
- package/dist/core/training/trainer/trainer_builder.js +0 -95
- package/dist/core/training/training_schemes.d.ts +0 -5
- package/dist/core/training/training_schemes.js +0 -10
- package/dist/core/types.d.ts +0 -4
- package/dist/core/types.js +0 -2
- package/dist/core/validation/index.d.ts +0 -1
- package/dist/core/validation/index.js +0 -5
- package/dist/core/validation/validator.d.ts +0 -17
- package/dist/core/validation/validator.js +0 -104
- package/dist/core/weights/aggregation.d.ts +0 -8
- package/dist/core/weights/aggregation.js +0 -96
- package/dist/core/weights/index.d.ts +0 -2
- package/dist/core/weights/index.js +0 -7
- package/dist/core/weights/weights_container.d.ts +0 -19
- package/dist/core/weights/weights_container.js +0 -64
- package/dist/imports.d.ts +0 -2
- package/dist/imports.js +0 -7
- package/dist/memory/memory.d.ts +0 -26
- package/dist/memory/memory.js +0 -160
- package/dist/{core/task → task}/data_example.d.ts +1 -1
- package/dist/{core/task → task}/summary.d.ts +1 -1
|
@@ -1,212 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Base = void 0;
|
|
4
|
-
var tslib_1 = require("tslib");
|
|
5
|
-
var immutable_1 = require("immutable");
|
|
6
|
-
var nodeUrl = (0, tslib_1.__importStar)(require("url"));
|
|
7
|
-
var __1 = require("../..");
|
|
8
|
-
var base_1 = require("../base");
|
|
9
|
-
var peer_pool_1 = require("./peer_pool");
|
|
10
|
-
var messages = (0, tslib_1.__importStar)(require("./messages"));
|
|
11
|
-
var messages_1 = require("../messages");
|
|
12
|
-
var event_connection_1 = require("../event_connection");
|
|
13
|
-
var utils_1 = require("../utils");
|
|
14
|
-
/**
|
|
15
|
-
* Abstract class for decentralized clients, executes onRoundEndCommunication as well as connecting
|
|
16
|
-
* to the signaling server
|
|
17
|
-
*/
|
|
18
|
-
var Base = /** @class */ (function (_super) {
|
|
19
|
-
(0, tslib_1.__extends)(Base, _super);
|
|
20
|
-
function Base(url, task) {
|
|
21
|
-
var _a, _b;
|
|
22
|
-
var _this = _super.call(this, url, task) || this;
|
|
23
|
-
_this.url = url;
|
|
24
|
-
_this.task = task;
|
|
25
|
-
_this.minimumReadyPeers = (_b = (_a = _this.task.trainingInformation) === null || _a === void 0 ? void 0 : _a.minimumReadyPeers) !== null && _b !== void 0 ? _b : 3;
|
|
26
|
-
return _this;
|
|
27
|
-
}
|
|
28
|
-
// send message to server that client is ready
|
|
29
|
-
Base.prototype.waitForPeers = function (round) {
|
|
30
|
-
var _a;
|
|
31
|
-
return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
|
|
32
|
-
var msg, receivedMessage, peers, ret;
|
|
33
|
-
var _this = this;
|
|
34
|
-
return (0, tslib_1.__generator)(this, function (_b) {
|
|
35
|
-
switch (_b.label) {
|
|
36
|
-
case 0:
|
|
37
|
-
console.debug(this.ID, 'is ready for round', round);
|
|
38
|
-
// clean old round
|
|
39
|
-
this.peers = undefined;
|
|
40
|
-
msg = { type: messages_1.type.PeerIsReady };
|
|
41
|
-
if (this.server === undefined) {
|
|
42
|
-
throw new Error('server undefined, could not connect peers');
|
|
43
|
-
}
|
|
44
|
-
this.server.send(msg);
|
|
45
|
-
return [4 /*yield*/, (0, event_connection_1.waitMessageWithTimeout)(this.server, messages_1.type.PeersForRound, utils_1.MAX_WAIT_PER_ROUND)];
|
|
46
|
-
case 1:
|
|
47
|
-
receivedMessage = _b.sent();
|
|
48
|
-
peers = (0, immutable_1.Set)(receivedMessage.peers);
|
|
49
|
-
if (this.ID !== undefined && peers.has(this.ID)) {
|
|
50
|
-
throw new Error('received peer list contains our own id');
|
|
51
|
-
}
|
|
52
|
-
if (this.peers !== undefined) {
|
|
53
|
-
throw new Error('got new peer list from server but was already received for this round');
|
|
54
|
-
}
|
|
55
|
-
if (peers.size + 1 < this.minimumReadyPeers) {
|
|
56
|
-
throw new Error('new peer list do not contain enough ready peers');
|
|
57
|
-
}
|
|
58
|
-
this.peers = peers;
|
|
59
|
-
console.debug(this.ID, 'got peers for round:', peers.toJS());
|
|
60
|
-
if (this.pool === undefined) {
|
|
61
|
-
throw new Error('waiting for peers but peer pool is undefined');
|
|
62
|
-
}
|
|
63
|
-
return [4 /*yield*/, this.pool];
|
|
64
|
-
case 2: return [4 /*yield*/, (_b.sent()).getPeers((0, immutable_1.Set)((_a = this.peers) !== null && _a !== void 0 ? _a : []), this.server, function (p) { return _this.clientHandle(p); })];
|
|
65
|
-
case 3:
|
|
66
|
-
ret = _b.sent();
|
|
67
|
-
console.debug(this.ID, "got peers for round " + round + ":", ret.keySeq().toJS());
|
|
68
|
-
return [2 /*return*/, ret];
|
|
69
|
-
}
|
|
70
|
-
});
|
|
71
|
-
});
|
|
72
|
-
};
|
|
73
|
-
// TODO inline? have a serialization mod
|
|
74
|
-
Base.prototype.sendMessagetoPeer = function (peer, msg) {
|
|
75
|
-
console.debug(this.ID, 'sends message to peer', msg.peer, msg);
|
|
76
|
-
peer.send(msg);
|
|
77
|
-
};
|
|
78
|
-
/*
|
|
79
|
-
creation of the websocket for the server, connection of client to that webSocket,
|
|
80
|
-
deals with message reception from decentralized client perspective (messages received by client)
|
|
81
|
-
*/
|
|
82
|
-
Base.prototype.connectServer = function (url) {
|
|
83
|
-
return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
|
|
84
|
-
var server;
|
|
85
|
-
var _this = this;
|
|
86
|
-
return (0, tslib_1.__generator)(this, function (_a) {
|
|
87
|
-
switch (_a.label) {
|
|
88
|
-
case 0: return [4 /*yield*/, event_connection_1.WebSocketServer.connect(url, messages.isMessageFromServer, messages.isMessageToServer)];
|
|
89
|
-
case 1:
|
|
90
|
-
server = _a.sent();
|
|
91
|
-
server.on(messages_1.type.SignalForPeer, function (event) {
|
|
92
|
-
console.debug(_this.ID, 'got signal from', event.peer);
|
|
93
|
-
if (_this.pool === undefined) {
|
|
94
|
-
throw new Error('got signal but peer pool is undefined');
|
|
95
|
-
}
|
|
96
|
-
void _this.pool.then(function (pool) { return pool.signal(event.peer, event.signal); });
|
|
97
|
-
});
|
|
98
|
-
return [2 /*return*/, server];
|
|
99
|
-
}
|
|
100
|
-
});
|
|
101
|
-
});
|
|
102
|
-
};
|
|
103
|
-
/**
|
|
104
|
-
* Initialize the connection to the peers and to the other nodes.
|
|
105
|
-
*/
|
|
106
|
-
Base.prototype.connect = function () {
|
|
107
|
-
return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
|
|
108
|
-
var URL, serverURL, _a, msg, peerIdMsg;
|
|
109
|
-
return (0, tslib_1.__generator)(this, function (_b) {
|
|
110
|
-
switch (_b.label) {
|
|
111
|
-
case 0:
|
|
112
|
-
URL = typeof window !== 'undefined' ? window.URL : nodeUrl.URL;
|
|
113
|
-
serverURL = new URL('', this.url.href);
|
|
114
|
-
switch (this.url.protocol) {
|
|
115
|
-
case 'http:':
|
|
116
|
-
serverURL.protocol = 'ws:';
|
|
117
|
-
break;
|
|
118
|
-
case 'https:':
|
|
119
|
-
serverURL.protocol = 'wss:';
|
|
120
|
-
break;
|
|
121
|
-
default:
|
|
122
|
-
throw new Error("unknown protocol: " + this.url.protocol);
|
|
123
|
-
}
|
|
124
|
-
serverURL.pathname += "deai/" + this.task.taskID;
|
|
125
|
-
_a = this;
|
|
126
|
-
return [4 /*yield*/, this.connectServer(serverURL)];
|
|
127
|
-
case 1:
|
|
128
|
-
_a.server = _b.sent();
|
|
129
|
-
msg = {
|
|
130
|
-
type: messages_1.type.clientConnected
|
|
131
|
-
};
|
|
132
|
-
this.server.send(msg);
|
|
133
|
-
return [4 /*yield*/, (0, event_connection_1.waitMessage)(this.server, messages_1.type.PeerID)];
|
|
134
|
-
case 2:
|
|
135
|
-
peerIdMsg = _b.sent();
|
|
136
|
-
console.debug(peerIdMsg.id, 'got own id from server');
|
|
137
|
-
if (this.ID !== undefined) {
|
|
138
|
-
throw new Error('got ID from server but was already received');
|
|
139
|
-
}
|
|
140
|
-
this.ID = peerIdMsg.id;
|
|
141
|
-
this.pool = peer_pool_1.PeerPool.init(peerIdMsg.id);
|
|
142
|
-
this.connected = true; // Is this still needed?
|
|
143
|
-
console.debug(this.ID, 'client connected to server');
|
|
144
|
-
return [2 /*return*/];
|
|
145
|
-
}
|
|
146
|
-
});
|
|
147
|
-
});
|
|
148
|
-
};
|
|
149
|
-
// disconnect from server & peers
|
|
150
|
-
Base.prototype.disconnect = function () {
|
|
151
|
-
var _a, _b;
|
|
152
|
-
return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
|
|
153
|
-
return (0, tslib_1.__generator)(this, function (_c) {
|
|
154
|
-
switch (_c.label) {
|
|
155
|
-
case 0:
|
|
156
|
-
console.debug(this.ID, 'disconnect');
|
|
157
|
-
return [4 /*yield*/, this.pool];
|
|
158
|
-
case 1:
|
|
159
|
-
(_a = (_c.sent())) === null || _a === void 0 ? void 0 : _a.shutdown();
|
|
160
|
-
this.pool = undefined;
|
|
161
|
-
(_b = this.server) === null || _b === void 0 ? void 0 : _b.disconnect();
|
|
162
|
-
this.server = undefined;
|
|
163
|
-
this.connected = false;
|
|
164
|
-
return [2 /*return*/];
|
|
165
|
-
}
|
|
166
|
-
});
|
|
167
|
-
});
|
|
168
|
-
};
|
|
169
|
-
Base.prototype.onTrainEndCommunication = function (_, trainingInformant) {
|
|
170
|
-
return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
|
|
171
|
-
return (0, tslib_1.__generator)(this, function (_a) {
|
|
172
|
-
// TODO: enter seeding mode?
|
|
173
|
-
trainingInformant.addMessage('Training finished.');
|
|
174
|
-
return [2 /*return*/];
|
|
175
|
-
});
|
|
176
|
-
});
|
|
177
|
-
};
|
|
178
|
-
Base.prototype.onRoundEndCommunication = function (updatedWeights, staleWeights, round, trainingInformant) {
|
|
179
|
-
return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
|
|
180
|
-
var peers, noisyWeights, finalWeights, e_1, msg;
|
|
181
|
-
return (0, tslib_1.__generator)(this, function (_a) {
|
|
182
|
-
switch (_a.label) {
|
|
183
|
-
case 0:
|
|
184
|
-
_a.trys.push([0, 3, , 4]);
|
|
185
|
-
return [4 /*yield*/, this.waitForPeers(round)
|
|
186
|
-
// centralized phase of communication --> client tells server that they have finished a local round and are ready to aggregate
|
|
187
|
-
// Apply clipping and DP to updates that will be sent
|
|
188
|
-
];
|
|
189
|
-
case 1:
|
|
190
|
-
peers = _a.sent();
|
|
191
|
-
noisyWeights = __1.privacy.addDifferentialPrivacy(updatedWeights, staleWeights, this.task);
|
|
192
|
-
return [4 /*yield*/, this.sendAndReceiveWeights(peers, noisyWeights, round, trainingInformant)];
|
|
193
|
-
case 2:
|
|
194
|
-
finalWeights = _a.sent();
|
|
195
|
-
console.debug(this.ID, 'sent and received', finalWeights.size, 'weights at round', round);
|
|
196
|
-
return [2 /*return*/, __1.aggregation.avg(finalWeights)];
|
|
197
|
-
case 3:
|
|
198
|
-
e_1 = _a.sent();
|
|
199
|
-
msg = "errored on round " + round;
|
|
200
|
-
if (e_1 instanceof Error) {
|
|
201
|
-
msg += ": " + e_1.message;
|
|
202
|
-
}
|
|
203
|
-
console.warn(this.ID, msg);
|
|
204
|
-
return [2 /*return*/, updatedWeights];
|
|
205
|
-
case 4: return [2 /*return*/];
|
|
206
|
-
}
|
|
207
|
-
});
|
|
208
|
-
});
|
|
209
|
-
};
|
|
210
|
-
return Base;
|
|
211
|
-
}(base_1.Base));
|
|
212
|
-
exports.Base = Base;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { List, Map } from 'immutable';
|
|
2
|
-
import { TrainingInformant, WeightsContainer } from '../..';
|
|
3
|
-
import { Base } from './base';
|
|
4
|
-
import { PeerID } from './types';
|
|
5
|
-
import { PeerConnection } from '../event_connection';
|
|
6
|
-
/**
|
|
7
|
-
* Decentralized client that does not utilize secure aggregation, but sends model updates in clear text
|
|
8
|
-
*/
|
|
9
|
-
export declare class ClearText extends Base {
|
|
10
|
-
private receivedWeights?;
|
|
11
|
-
sendAndReceiveWeights(peers: Map<PeerID, PeerConnection>, noisyWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<List<WeightsContainer>>;
|
|
12
|
-
private receiveWeights;
|
|
13
|
-
clientHandle(peers: Map<PeerID, PeerConnection>): void;
|
|
14
|
-
}
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ClearText = void 0;
|
|
4
|
-
var tslib_1 = require("tslib");
|
|
5
|
-
var immutable_1 = require("immutable");
|
|
6
|
-
var __1 = require("../..");
|
|
7
|
-
var base_1 = require("./base");
|
|
8
|
-
var messages_1 = require("../messages");
|
|
9
|
-
var event_connection_1 = require("../event_connection");
|
|
10
|
-
/**
|
|
11
|
-
* Decentralized client that does not utilize secure aggregation, but sends model updates in clear text
|
|
12
|
-
*/
|
|
13
|
-
var ClearText = /** @class */ (function (_super) {
|
|
14
|
-
(0, tslib_1.__extends)(ClearText, _super);
|
|
15
|
-
function ClearText() {
|
|
16
|
-
return _super !== null && _super.apply(this, arguments) || this;
|
|
17
|
-
}
|
|
18
|
-
ClearText.prototype.sendAndReceiveWeights = function (peers, noisyWeights, round, trainingInformant) {
|
|
19
|
-
return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
|
|
20
|
-
var weights, ret;
|
|
21
|
-
var _this = this;
|
|
22
|
-
return (0, tslib_1.__generator)(this, function (_a) {
|
|
23
|
-
switch (_a.label) {
|
|
24
|
-
case 0:
|
|
25
|
-
if (!this.receivedWeights) {
|
|
26
|
-
throw new Error('no promise setup for receiving weights');
|
|
27
|
-
}
|
|
28
|
-
// PHASE 1 COMMUNICATION --> create weights message and send to all peers (only one phase of communication for clear_text)
|
|
29
|
-
// send weights asynchronously
|
|
30
|
-
__1.serialization.weights.encode(noisyWeights).then(function (encodedWeights) {
|
|
31
|
-
// create weights message and send to all peers
|
|
32
|
-
peers.forEach(function (peer, id) {
|
|
33
|
-
return _this.sendMessagetoPeer(peer, {
|
|
34
|
-
type: messages_1.type.Weights,
|
|
35
|
-
peer: id,
|
|
36
|
-
weights: encodedWeights
|
|
37
|
-
});
|
|
38
|
-
});
|
|
39
|
-
}).catch(function () {
|
|
40
|
-
throw new Error('error while sending weights');
|
|
41
|
-
});
|
|
42
|
-
return [4 /*yield*/, this.receivedWeights];
|
|
43
|
-
case 1:
|
|
44
|
-
weights = _a.sent();
|
|
45
|
-
trainingInformant.update({
|
|
46
|
-
currentNumberOfParticipants: weights.size + 1
|
|
47
|
-
});
|
|
48
|
-
ret = weights.push(noisyWeights);
|
|
49
|
-
this.receivedWeights = undefined;
|
|
50
|
-
return [2 /*return*/, ret];
|
|
51
|
-
}
|
|
52
|
-
});
|
|
53
|
-
});
|
|
54
|
-
};
|
|
55
|
-
ClearText.prototype.receiveWeights = function (peers) {
|
|
56
|
-
return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
|
|
57
|
-
var waitWeights, receivedWeights;
|
|
58
|
-
var _this = this;
|
|
59
|
-
return (0, tslib_1.__generator)(this, function (_a) {
|
|
60
|
-
switch (_a.label) {
|
|
61
|
-
case 0:
|
|
62
|
-
console.debug('beginning of receiveWeights');
|
|
63
|
-
waitWeights = Array.from(peers.values()).map(function (peer) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () {
|
|
64
|
-
return (0, tslib_1.__generator)(this, function (_a) {
|
|
65
|
-
switch (_a.label) {
|
|
66
|
-
case 0: return [4 /*yield*/, (0, event_connection_1.waitMessage)(peer, messages_1.type.Weights)];
|
|
67
|
-
case 1: return [2 /*return*/, _a.sent()];
|
|
68
|
-
}
|
|
69
|
-
});
|
|
70
|
-
}); });
|
|
71
|
-
receivedWeights = (0, immutable_1.List)();
|
|
72
|
-
return [4 /*yield*/, Promise.allSettled(waitWeights)];
|
|
73
|
-
case 1: return [4 /*yield*/, (_a.sent()).forEach(function (message) {
|
|
74
|
-
if (message.status === 'fulfilled') {
|
|
75
|
-
receivedWeights = receivedWeights.push(__1.serialization.weights.decode(message.value.weights));
|
|
76
|
-
}
|
|
77
|
-
})];
|
|
78
|
-
case 2:
|
|
79
|
-
_a.sent();
|
|
80
|
-
if (receivedWeights.size < peers.size) {
|
|
81
|
-
throw new Error('not enough peer weights received');
|
|
82
|
-
}
|
|
83
|
-
return [2 /*return*/, receivedWeights];
|
|
84
|
-
}
|
|
85
|
-
});
|
|
86
|
-
});
|
|
87
|
-
};
|
|
88
|
-
/*
|
|
89
|
-
handles received messages from signaling server
|
|
90
|
-
*/
|
|
91
|
-
ClearText.prototype.clientHandle = function (peers) {
|
|
92
|
-
this.receivedWeights = this.receiveWeights(peers);
|
|
93
|
-
};
|
|
94
|
-
return ClearText;
|
|
95
|
-
}(base_1.Base));
|
|
96
|
-
exports.ClearText = ClearText;
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.messages = exports.SecAgg = exports.ClearText = void 0;
|
|
4
|
-
var tslib_1 = require("tslib");
|
|
5
|
-
var clear_text_1 = require("./clear_text");
|
|
6
|
-
Object.defineProperty(exports, "ClearText", { enumerable: true, get: function () { return clear_text_1.ClearText; } });
|
|
7
|
-
var sec_agg_1 = require("./sec_agg");
|
|
8
|
-
Object.defineProperty(exports, "SecAgg", { enumerable: true, get: function () { return sec_agg_1.SecAgg; } });
|
|
9
|
-
exports.messages = (0, tslib_1.__importStar)(require("./messages"));
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
import { SignalData } from 'simple-peer';
|
|
2
|
-
import { weights } from '../../serialization';
|
|
3
|
-
import { PeerID as PeerIDType } from './types';
|
|
4
|
-
import { type, clientConnected } from '../messages';
|
|
5
|
-
export interface PeerID {
|
|
6
|
-
type: type.PeerID;
|
|
7
|
-
id: PeerIDType;
|
|
8
|
-
}
|
|
9
|
-
export interface SignalForPeer {
|
|
10
|
-
type: type.SignalForPeer;
|
|
11
|
-
peer: PeerIDType;
|
|
12
|
-
signal: SignalData;
|
|
13
|
-
}
|
|
14
|
-
export interface PeerIsReady {
|
|
15
|
-
type: type.PeerIsReady;
|
|
16
|
-
}
|
|
17
|
-
export interface PeersForRound {
|
|
18
|
-
type: type.PeersForRound;
|
|
19
|
-
peers: PeerIDType[];
|
|
20
|
-
}
|
|
21
|
-
export interface Weights {
|
|
22
|
-
type: type.Weights;
|
|
23
|
-
peer: PeerIDType;
|
|
24
|
-
weights: weights.Encoded;
|
|
25
|
-
}
|
|
26
|
-
export interface Shares {
|
|
27
|
-
type: type.Shares;
|
|
28
|
-
peer: PeerIDType;
|
|
29
|
-
weights: weights.Encoded;
|
|
30
|
-
}
|
|
31
|
-
export interface PartialSums {
|
|
32
|
-
type: type.PartialSums;
|
|
33
|
-
peer: PeerIDType;
|
|
34
|
-
partials: weights.Encoded;
|
|
35
|
-
}
|
|
36
|
-
export declare type MessageFromServer = PeerID | SignalForPeer | PeersForRound;
|
|
37
|
-
export declare type MessageToServer = clientConnected | SignalForPeer | PeerIsReady;
|
|
38
|
-
export declare type PeerMessage = Weights | Shares | PartialSums;
|
|
39
|
-
export declare function isMessageFromServer(o: unknown): o is MessageFromServer;
|
|
40
|
-
export declare function isMessageToServer(o: unknown): o is MessageToServer;
|
|
41
|
-
export declare function isPeerMessage(o: unknown): o is PeerMessage;
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.isPeerMessage = exports.isMessageToServer = exports.isMessageFromServer = void 0;
|
|
4
|
-
var serialization_1 = require("../../serialization");
|
|
5
|
-
var types_1 = require("./types");
|
|
6
|
-
var messages_1 = require("../messages");
|
|
7
|
-
function isMessageFromServer(o) {
|
|
8
|
-
if (!(0, messages_1.hasMessageType)(o)) {
|
|
9
|
-
return false;
|
|
10
|
-
}
|
|
11
|
-
switch (o.type) {
|
|
12
|
-
case messages_1.type.PeerID:
|
|
13
|
-
return 'id' in o && (0, types_1.isPeerID)(o.id);
|
|
14
|
-
case messages_1.type.SignalForPeer:
|
|
15
|
-
return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
|
|
16
|
-
'signal' in o; // TODO check signal content?
|
|
17
|
-
case messages_1.type.PeersForRound:
|
|
18
|
-
return 'peers' in o && Array.isArray(o.peers) && o.peers.every(types_1.isPeerID);
|
|
19
|
-
}
|
|
20
|
-
return false;
|
|
21
|
-
}
|
|
22
|
-
exports.isMessageFromServer = isMessageFromServer;
|
|
23
|
-
function isMessageToServer(o) {
|
|
24
|
-
if (!(0, messages_1.hasMessageType)(o)) {
|
|
25
|
-
return false;
|
|
26
|
-
}
|
|
27
|
-
switch (o.type) {
|
|
28
|
-
case messages_1.type.clientConnected:
|
|
29
|
-
return true;
|
|
30
|
-
case messages_1.type.SignalForPeer:
|
|
31
|
-
return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
|
|
32
|
-
'signal' in o; // TODO check signal content?
|
|
33
|
-
case messages_1.type.PeerIsReady:
|
|
34
|
-
return true;
|
|
35
|
-
}
|
|
36
|
-
return false;
|
|
37
|
-
}
|
|
38
|
-
exports.isMessageToServer = isMessageToServer;
|
|
39
|
-
function isPeerMessage(o) {
|
|
40
|
-
if (!(0, messages_1.hasMessageType)(o)) {
|
|
41
|
-
return false;
|
|
42
|
-
}
|
|
43
|
-
switch (o.type) {
|
|
44
|
-
case messages_1.type.Weights:
|
|
45
|
-
case messages_1.type.Shares:
|
|
46
|
-
return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
|
|
47
|
-
'weights' in o && serialization_1.weights.isEncoded(o.weights);
|
|
48
|
-
case messages_1.type.PartialSums:
|
|
49
|
-
return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
|
|
50
|
-
'partials' in o && serialization_1.weights.isEncoded(o.partials);
|
|
51
|
-
}
|
|
52
|
-
return false;
|
|
53
|
-
}
|
|
54
|
-
exports.isPeerMessage = isPeerMessage;
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import SimplePeer, { SignalData } from 'simple-peer';
|
|
3
|
-
import { PeerID } from './types';
|
|
4
|
-
interface Events {
|
|
5
|
-
'close': () => void;
|
|
6
|
-
'connect': () => void;
|
|
7
|
-
'signal': (signal: SignalData) => void;
|
|
8
|
-
'data': (data: Buffer) => void;
|
|
9
|
-
}
|
|
10
|
-
export declare class Peer {
|
|
11
|
-
readonly id: PeerID;
|
|
12
|
-
private readonly peer;
|
|
13
|
-
private bufferSize?;
|
|
14
|
-
private sendCounter;
|
|
15
|
-
private sendQueue;
|
|
16
|
-
private receiving;
|
|
17
|
-
constructor(id: PeerID, opts?: SimplePeer.Options);
|
|
18
|
-
send(msg: Buffer): void;
|
|
19
|
-
private flush;
|
|
20
|
-
get maxChunkSize(): number;
|
|
21
|
-
private chunk;
|
|
22
|
-
destroy(): void;
|
|
23
|
-
signal(signal: SimplePeer.SignalData): void;
|
|
24
|
-
on<K extends keyof Events>(event: K, listener: Events[K]): void;
|
|
25
|
-
}
|
|
26
|
-
export {};
|
|
@@ -1,210 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Peer = void 0;
|
|
4
|
-
var tslib_1 = require("tslib");
|
|
5
|
-
var immutable_1 = require("immutable");
|
|
6
|
-
var simple_peer_1 = (0, tslib_1.__importDefault)(require("simple-peer"));
|
|
7
|
-
// message id + (chunk counter == 0) + chunk count
|
|
8
|
-
var FIRST_HEADER_SIZE = 2 + 1 + 1;
|
|
9
|
-
// message id + chunk counter
|
|
10
|
-
var HEADER_SIZE = 2 + 1;
|
|
11
|
-
// at which interval to poll
|
|
12
|
-
var TICK = 10;
|
|
13
|
-
// Peer wraps a SimplePeer, adding message fragmentation
|
|
14
|
-
//
|
|
15
|
-
// WebRTC implementations have various maximum message size
|
|
16
|
-
// but with huge models, our messages might be bigger.
|
|
17
|
-
// We split messages by chunks and reconstruct theses
|
|
18
|
-
// on the other side.
|
|
19
|
-
//
|
|
20
|
-
// As the WebRTC's DataChannel is not a stream, we need
|
|
21
|
-
// reorder messages, so we use a header on each chunk
|
|
22
|
-
// with a message id and chunk counter. The first chunk
|
|
23
|
-
// (chunk counter == 0), also add the total number of chunk.
|
|
24
|
-
//
|
|
25
|
-
// see feross/simple-peer#393 for more info
|
|
26
|
-
var Peer = /** @class */ (function () {
|
|
27
|
-
function Peer(id, opts) {
|
|
28
|
-
this.sendCounter = 0;
|
|
29
|
-
this.sendQueue = (0, immutable_1.List)();
|
|
30
|
-
this.receiving = (0, immutable_1.Map)();
|
|
31
|
-
this.id = id;
|
|
32
|
-
this.peer = new simple_peer_1.default(opts);
|
|
33
|
-
}
|
|
34
|
-
Peer.prototype.send = function (msg) {
|
|
35
|
-
console.debug('sending message of size', msg.length);
|
|
36
|
-
var chunks = this.chunk(msg);
|
|
37
|
-
this.sendQueue = this.sendQueue.concat(chunks);
|
|
38
|
-
this.flush();
|
|
39
|
-
};
|
|
40
|
-
Peer.prototype.flush = function () {
|
|
41
|
-
var _this = this;
|
|
42
|
-
if (this.bufferSize === undefined) {
|
|
43
|
-
throw new Error('flush without known buffer size');
|
|
44
|
-
}
|
|
45
|
-
var chunk = this.sendQueue.first();
|
|
46
|
-
if (chunk === undefined) {
|
|
47
|
-
return; // nothing to flush
|
|
48
|
-
}
|
|
49
|
-
var remainingBufferSize = this.bufferSize - this.peer.bufferSize;
|
|
50
|
-
if (chunk.length > remainingBufferSize) {
|
|
51
|
-
setTimeout(function () { return _this.flush(); }, TICK);
|
|
52
|
-
return;
|
|
53
|
-
}
|
|
54
|
-
console.debug('sending chunk of size', chunk.length);
|
|
55
|
-
this.sendQueue = this.sendQueue.shift();
|
|
56
|
-
this.peer.send(chunk);
|
|
57
|
-
// and loop
|
|
58
|
-
this.flush();
|
|
59
|
-
};
|
|
60
|
-
Object.defineProperty(Peer.prototype, "maxChunkSize", {
|
|
61
|
-
get: function () {
|
|
62
|
-
if (this.bufferSize === undefined) {
|
|
63
|
-
throw new Error('chunk without known buffer size');
|
|
64
|
-
}
|
|
65
|
-
// in the perfect world of bug-free implementations
|
|
66
|
-
// we would return this.bufferSize
|
|
67
|
-
// sadly, we are not there yet
|
|
68
|
-
//
|
|
69
|
-
// based on MDN, taking 16K seems to be a pretty safe
|
|
70
|
-
// and widely supported buffer size
|
|
71
|
-
return 16 * (1 << 10);
|
|
72
|
-
},
|
|
73
|
-
enumerable: false,
|
|
74
|
-
configurable: true
|
|
75
|
-
});
|
|
76
|
-
Peer.prototype.chunk = function (b) {
|
|
77
|
-
var _this = this;
|
|
78
|
-
var messageID = this.sendCounter;
|
|
79
|
-
this.sendCounter++;
|
|
80
|
-
if (this.sendCounter > 0xFFFF) {
|
|
81
|
-
throw new Error('too much messages sent to this peer');
|
|
82
|
-
}
|
|
83
|
-
// special case as Range(1, 0) yields a value
|
|
84
|
-
var tail = immutable_1.Seq.Indexed([]);
|
|
85
|
-
if (b.length > this.maxChunkSize) {
|
|
86
|
-
tail = (0, immutable_1.Range)(this.maxChunkSize - FIRST_HEADER_SIZE, b.length, this.maxChunkSize - HEADER_SIZE).map(function (offset) { return b.subarray(offset, offset + _this.maxChunkSize - HEADER_SIZE); });
|
|
87
|
-
}
|
|
88
|
-
var totalChunkCount = 1 + tail.count();
|
|
89
|
-
if (totalChunkCount > 0xFF) {
|
|
90
|
-
throw new Error('too big message to even chunk it');
|
|
91
|
-
}
|
|
92
|
-
var firstChunk = Buffer.alloc((b.length > this.maxChunkSize - FIRST_HEADER_SIZE)
|
|
93
|
-
? this.maxChunkSize
|
|
94
|
-
: FIRST_HEADER_SIZE + b.length);
|
|
95
|
-
firstChunk.writeUint16BE(messageID);
|
|
96
|
-
firstChunk.writeUint8(0, 2);
|
|
97
|
-
firstChunk.writeUint8(totalChunkCount, 3);
|
|
98
|
-
b.copy(firstChunk, FIRST_HEADER_SIZE, 0, this.maxChunkSize - FIRST_HEADER_SIZE);
|
|
99
|
-
return immutable_1.Seq.Indexed([firstChunk])
|
|
100
|
-
.concat((0, immutable_1.Range)(1).zip(tail)
|
|
101
|
-
.map(function (_a) {
|
|
102
|
-
var _b = (0, tslib_1.__read)(_a, 2), id = _b[0], raw = _b[1];
|
|
103
|
-
var chunk = Buffer.alloc(HEADER_SIZE + raw.length);
|
|
104
|
-
chunk.writeUint16BE(messageID);
|
|
105
|
-
chunk.writeUint8(id, 2);
|
|
106
|
-
raw.copy(chunk, HEADER_SIZE, 0);
|
|
107
|
-
return chunk;
|
|
108
|
-
}));
|
|
109
|
-
};
|
|
110
|
-
Peer.prototype.destroy = function () {
|
|
111
|
-
this.peer.destroy();
|
|
112
|
-
};
|
|
113
|
-
Peer.prototype.signal = function (signal) {
|
|
114
|
-
// extract max buffer size
|
|
115
|
-
if (signal.type === 'offer' || signal.type === 'answer') {
|
|
116
|
-
if (signal.sdp === undefined) {
|
|
117
|
-
throw new Error('signal answer|offer without session description');
|
|
118
|
-
}
|
|
119
|
-
if (this.bufferSize !== undefined) {
|
|
120
|
-
throw new Error('buffer size set twice');
|
|
121
|
-
}
|
|
122
|
-
var match = signal.sdp.match(/a=max-message-size:(\d+)/);
|
|
123
|
-
if (match === null) {
|
|
124
|
-
// TODO default value instead?
|
|
125
|
-
throw new Error('no max-message-size found in signal');
|
|
126
|
-
}
|
|
127
|
-
var max = parseInt(match[1], 10);
|
|
128
|
-
if (isNaN(max)) {
|
|
129
|
-
throw new Error("unable to parse max-message-size as int: " + match[1]);
|
|
130
|
-
}
|
|
131
|
-
this.bufferSize = max;
|
|
132
|
-
}
|
|
133
|
-
this.peer.signal(signal);
|
|
134
|
-
};
|
|
135
|
-
Peer.prototype.on = function (event, listener) {
|
|
136
|
-
var _this = this;
|
|
137
|
-
if (event !== 'data') {
|
|
138
|
-
this.peer.on(event, listener);
|
|
139
|
-
return;
|
|
140
|
-
}
|
|
141
|
-
this.peer.on('data', function (data) {
|
|
142
|
-
if (!Buffer.isBuffer(data) || data.length < HEADER_SIZE) {
|
|
143
|
-
throw new Error('received invalid message type');
|
|
144
|
-
}
|
|
145
|
-
var messageID = data.readUint16BE();
|
|
146
|
-
var chunkID = data.readUint8(2);
|
|
147
|
-
var received = _this.receiving.get(messageID, {
|
|
148
|
-
total: undefined,
|
|
149
|
-
chunks: (0, immutable_1.Map)()
|
|
150
|
-
});
|
|
151
|
-
var total = received.total;
|
|
152
|
-
var chunks = received.chunks;
|
|
153
|
-
if (chunks.has(chunkID)) {
|
|
154
|
-
throw new Error("chunk " + messageID + ":" + chunkID + " already received");
|
|
155
|
-
}
|
|
156
|
-
var chunk;
|
|
157
|
-
if (chunkID !== 0) {
|
|
158
|
-
chunk = Buffer.alloc(data.length - HEADER_SIZE);
|
|
159
|
-
data.copy(chunk, 0, HEADER_SIZE);
|
|
160
|
-
}
|
|
161
|
-
else {
|
|
162
|
-
if (data.length < FIRST_HEADER_SIZE) {
|
|
163
|
-
throw new Error('received invalid message type');
|
|
164
|
-
}
|
|
165
|
-
if (total !== undefined) {
|
|
166
|
-
throw new Error('first header received twice');
|
|
167
|
-
}
|
|
168
|
-
var readTotal_1 = data.readUint8(3);
|
|
169
|
-
total = readTotal_1;
|
|
170
|
-
chunk = Buffer.alloc(data.length - FIRST_HEADER_SIZE);
|
|
171
|
-
data.copy(chunk, 0, FIRST_HEADER_SIZE);
|
|
172
|
-
if (chunks.keySeq().some(function (id) { return id > readTotal_1; })) {
|
|
173
|
-
throw new Error('received total of chunk but got now-out-of-bound chunks');
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
_this.receiving = _this.receiving.set(messageID, {
|
|
177
|
-
total: total,
|
|
178
|
-
chunks: chunks.set(chunkID, chunk)
|
|
179
|
-
});
|
|
180
|
-
console.debug("got chunk " + messageID + ":" + chunkID + "/" + (total !== null && total !== void 0 ? total : 'unknown') + " of size " + chunk.length);
|
|
181
|
-
var readyMessages = _this.receiving
|
|
182
|
-
.filter(function (_a) {
|
|
183
|
-
var total = _a.total, chunks = _a.chunks;
|
|
184
|
-
return total !== undefined && chunks.size === total;
|
|
185
|
-
})
|
|
186
|
-
.sort()
|
|
187
|
-
.map(function (_a) {
|
|
188
|
-
var chunks = _a.chunks;
|
|
189
|
-
return chunks.entrySeq().toList().sortBy(function (_a) {
|
|
190
|
-
var _b = (0, tslib_1.__read)(_a, 2), id = _b[0], _ = _b[1];
|
|
191
|
-
return id;
|
|
192
|
-
});
|
|
193
|
-
})
|
|
194
|
-
.map(function (chunks) { return Buffer.concat(chunks.map(function (_a) {
|
|
195
|
-
var _b = (0, tslib_1.__read)(_a, 2), _ = _b[0], b = _b[1];
|
|
196
|
-
return b;
|
|
197
|
-
}).toArray()); });
|
|
198
|
-
_this.receiving = _this.receiving.deleteAll(readyMessages.keys());
|
|
199
|
-
readyMessages
|
|
200
|
-
.forEach(function (message) {
|
|
201
|
-
console.debug(_this.peer.address().port, 'recved message of size', message.length);
|
|
202
|
-
// TODO debug
|
|
203
|
-
// @ts-expect-error
|
|
204
|
-
listener(message);
|
|
205
|
-
});
|
|
206
|
-
});
|
|
207
|
-
};
|
|
208
|
-
return Peer;
|
|
209
|
-
}());
|
|
210
|
-
exports.Peer = Peer;
|