@epfml/discojs 0.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/README.md +28 -8
  2. package/dist/{async_buffer.d.ts → core/async_buffer.d.ts} +3 -3
  3. package/dist/{async_buffer.js → core/async_buffer.js} +5 -6
  4. package/dist/{async_informant.d.ts → core/async_informant.d.ts} +0 -0
  5. package/dist/{async_informant.js → core/async_informant.js} +0 -0
  6. package/dist/{client → core/client}/base.d.ts +4 -7
  7. package/dist/{client → core/client}/base.js +3 -2
  8. package/dist/core/client/decentralized/base.d.ts +32 -0
  9. package/dist/core/client/decentralized/base.js +212 -0
  10. package/dist/core/client/decentralized/clear_text.d.ts +14 -0
  11. package/dist/core/client/decentralized/clear_text.js +96 -0
  12. package/dist/{client → core/client}/decentralized/index.d.ts +0 -0
  13. package/dist/{client → core/client}/decentralized/index.js +0 -0
  14. package/dist/core/client/decentralized/messages.d.ts +41 -0
  15. package/dist/core/client/decentralized/messages.js +54 -0
  16. package/dist/core/client/decentralized/peer.d.ts +26 -0
  17. package/dist/core/client/decentralized/peer.js +210 -0
  18. package/dist/core/client/decentralized/peer_pool.d.ts +14 -0
  19. package/dist/core/client/decentralized/peer_pool.js +92 -0
  20. package/dist/core/client/decentralized/sec_agg.d.ts +22 -0
  21. package/dist/core/client/decentralized/sec_agg.js +190 -0
  22. package/dist/core/client/decentralized/secret_shares.d.ts +3 -0
  23. package/dist/core/client/decentralized/secret_shares.js +39 -0
  24. package/dist/core/client/decentralized/types.d.ts +2 -0
  25. package/dist/core/client/decentralized/types.js +7 -0
  26. package/dist/core/client/event_connection.d.ts +37 -0
  27. package/dist/core/client/event_connection.js +158 -0
  28. package/dist/core/client/federated/client.d.ts +37 -0
  29. package/dist/core/client/federated/client.js +273 -0
  30. package/dist/core/client/federated/index.d.ts +2 -0
  31. package/dist/core/client/federated/index.js +7 -0
  32. package/dist/core/client/federated/messages.d.ts +38 -0
  33. package/dist/core/client/federated/messages.js +25 -0
  34. package/dist/{client → core/client}/index.d.ts +2 -1
  35. package/dist/{client → core/client}/index.js +3 -3
  36. package/dist/{client → core/client}/local.d.ts +2 -2
  37. package/dist/{client → core/client}/local.js +0 -0
  38. package/dist/core/client/messages.d.ts +28 -0
  39. package/dist/core/client/messages.js +33 -0
  40. package/dist/core/client/utils.d.ts +2 -0
  41. package/dist/core/client/utils.js +19 -0
  42. package/dist/core/dataset/data/data.d.ts +11 -0
  43. package/dist/core/dataset/data/data.js +20 -0
  44. package/dist/core/dataset/data/data_split.d.ts +5 -0
  45. package/dist/{client/decentralized/types.js → core/dataset/data/data_split.js} +0 -0
  46. package/dist/core/dataset/data/image_data.d.ts +8 -0
  47. package/dist/core/dataset/data/image_data.js +64 -0
  48. package/dist/core/dataset/data/index.d.ts +5 -0
  49. package/dist/core/dataset/data/index.js +11 -0
  50. package/dist/core/dataset/data/preprocessing.d.ts +13 -0
  51. package/dist/core/dataset/data/preprocessing.js +33 -0
  52. package/dist/core/dataset/data/tabular_data.d.ts +8 -0
  53. package/dist/core/dataset/data/tabular_data.js +40 -0
  54. package/dist/{dataset → core/dataset}/data_loader/data_loader.d.ts +4 -11
  55. package/dist/{dataset → core/dataset}/data_loader/data_loader.js +0 -0
  56. package/dist/core/dataset/data_loader/image_loader.d.ts +17 -0
  57. package/dist/core/dataset/data_loader/image_loader.js +141 -0
  58. package/dist/core/dataset/data_loader/index.d.ts +3 -0
  59. package/dist/core/dataset/data_loader/index.js +9 -0
  60. package/dist/core/dataset/data_loader/tabular_loader.d.ts +29 -0
  61. package/dist/core/dataset/data_loader/tabular_loader.js +101 -0
  62. package/dist/core/dataset/dataset.d.ts +2 -0
  63. package/dist/{task/training_information.js → core/dataset/dataset.js} +0 -0
  64. package/dist/{dataset → core/dataset}/dataset_builder.d.ts +5 -5
  65. package/dist/{dataset → core/dataset}/dataset_builder.js +14 -10
  66. package/dist/core/dataset/index.d.ts +4 -0
  67. package/dist/core/dataset/index.js +14 -0
  68. package/dist/core/index.d.ts +18 -0
  69. package/dist/core/index.js +41 -0
  70. package/dist/{informant → core/informant}/graph_informant.d.ts +0 -0
  71. package/dist/{informant → core/informant}/graph_informant.js +0 -0
  72. package/dist/{informant → core/informant}/index.d.ts +0 -0
  73. package/dist/{informant → core/informant}/index.js +0 -0
  74. package/dist/{informant → core/informant}/training_informant/base.d.ts +3 -3
  75. package/dist/{informant → core/informant}/training_informant/base.js +3 -2
  76. package/dist/{informant → core/informant}/training_informant/decentralized.d.ts +0 -0
  77. package/dist/{informant → core/informant}/training_informant/decentralized.js +0 -0
  78. package/dist/{informant → core/informant}/training_informant/federated.d.ts +0 -0
  79. package/dist/{informant → core/informant}/training_informant/federated.js +0 -0
  80. package/dist/{informant → core/informant}/training_informant/index.d.ts +0 -0
  81. package/dist/{informant → core/informant}/training_informant/index.js +0 -0
  82. package/dist/{informant → core/informant}/training_informant/local.d.ts +2 -2
  83. package/dist/{informant → core/informant}/training_informant/local.js +2 -2
  84. package/dist/{logging → core/logging}/console_logger.d.ts +0 -0
  85. package/dist/{logging → core/logging}/console_logger.js +0 -0
  86. package/dist/{logging → core/logging}/index.d.ts +0 -0
  87. package/dist/{logging → core/logging}/index.js +0 -0
  88. package/dist/{logging → core/logging}/logger.d.ts +0 -0
  89. package/dist/{logging → core/logging}/logger.js +0 -0
  90. package/dist/{logging → core/logging}/trainer_logger.d.ts +0 -0
  91. package/dist/{logging → core/logging}/trainer_logger.js +0 -0
  92. package/dist/{memory → core/memory}/base.d.ts +2 -2
  93. package/dist/{memory → core/memory}/base.js +0 -0
  94. package/dist/{memory → core/memory}/empty.d.ts +0 -0
  95. package/dist/{memory → core/memory}/empty.js +0 -0
  96. package/dist/core/memory/index.d.ts +3 -0
  97. package/dist/core/memory/index.js +9 -0
  98. package/dist/{memory → core/memory}/model_type.d.ts +0 -0
  99. package/dist/{memory → core/memory}/model_type.js +0 -0
  100. package/dist/{privacy.d.ts → core/privacy.d.ts} +2 -3
  101. package/dist/{privacy.js → core/privacy.js} +3 -16
  102. package/dist/{serialization → core/serialization}/index.d.ts +0 -0
  103. package/dist/{serialization → core/serialization}/index.js +0 -0
  104. package/dist/{serialization → core/serialization}/model.d.ts +0 -0
  105. package/dist/{serialization → core/serialization}/model.js +0 -0
  106. package/dist/core/serialization/weights.d.ts +5 -0
  107. package/dist/{serialization → core/serialization}/weights.js +11 -9
  108. package/dist/{task → core/task}/data_example.d.ts +0 -0
  109. package/dist/{task → core/task}/data_example.js +0 -0
  110. package/dist/{task → core/task}/display_information.d.ts +5 -5
  111. package/dist/{task → core/task}/display_information.js +5 -10
  112. package/dist/{task → core/task}/index.d.ts +0 -0
  113. package/dist/{task → core/task}/index.js +0 -0
  114. package/dist/core/task/model_compile_data.d.ts +6 -0
  115. package/dist/core/task/model_compile_data.js +22 -0
  116. package/dist/{task → core/task}/summary.d.ts +0 -0
  117. package/dist/{task → core/task}/summary.js +0 -4
  118. package/dist/{task → core/task}/task.d.ts +2 -2
  119. package/dist/{task → core/task}/task.js +6 -7
  120. package/dist/{task → core/task}/training_information.d.ts +10 -14
  121. package/dist/core/task/training_information.js +66 -0
  122. package/dist/{tasks → core/tasks}/cifar10.d.ts +1 -2
  123. package/dist/{tasks → core/tasks}/cifar10.js +12 -23
  124. package/dist/core/tasks/geotags.d.ts +3 -0
  125. package/dist/core/tasks/geotags.js +67 -0
  126. package/dist/{tasks → core/tasks}/index.d.ts +2 -1
  127. package/dist/{tasks → core/tasks}/index.js +3 -2
  128. package/dist/core/tasks/lus_covid.d.ts +3 -0
  129. package/dist/{tasks → core/tasks}/lus_covid.js +26 -24
  130. package/dist/{tasks → core/tasks}/mnist.d.ts +1 -2
  131. package/dist/{tasks → core/tasks}/mnist.js +18 -16
  132. package/dist/core/tasks/simple_face.d.ts +2 -0
  133. package/dist/core/tasks/simple_face.js +41 -0
  134. package/dist/{tasks → core/tasks}/titanic.d.ts +1 -2
  135. package/dist/{tasks → core/tasks}/titanic.js +11 -11
  136. package/dist/core/training/disco.d.ts +23 -0
  137. package/dist/core/training/disco.js +130 -0
  138. package/dist/{training → core/training}/index.d.ts +0 -0
  139. package/dist/{training → core/training}/index.js +0 -0
  140. package/dist/{training → core/training}/trainer/distributed_trainer.d.ts +1 -2
  141. package/dist/{training → core/training}/trainer/distributed_trainer.js +6 -5
  142. package/dist/{training → core/training}/trainer/local_trainer.d.ts +2 -2
  143. package/dist/{training → core/training}/trainer/local_trainer.js +0 -0
  144. package/dist/{training → core/training}/trainer/round_tracker.d.ts +0 -0
  145. package/dist/{training → core/training}/trainer/round_tracker.js +0 -0
  146. package/dist/{training → core/training}/trainer/trainer.d.ts +1 -2
  147. package/dist/{training → core/training}/trainer/trainer.js +2 -2
  148. package/dist/{training → core/training}/trainer/trainer_builder.d.ts +0 -0
  149. package/dist/{training → core/training}/trainer/trainer_builder.js +0 -0
  150. package/dist/core/training/training_schemes.d.ts +5 -0
  151. package/dist/{training → core/training}/training_schemes.js +2 -2
  152. package/dist/{types.d.ts → core/types.d.ts} +0 -0
  153. package/dist/{types.js → core/types.js} +0 -0
  154. package/dist/{validation → core/validation}/index.d.ts +0 -0
  155. package/dist/{validation → core/validation}/index.js +0 -0
  156. package/dist/{validation → core/validation}/validator.d.ts +5 -8
  157. package/dist/{validation → core/validation}/validator.js +9 -11
  158. package/dist/core/weights/aggregation.d.ts +8 -0
  159. package/dist/core/weights/aggregation.js +96 -0
  160. package/dist/core/weights/index.d.ts +2 -0
  161. package/dist/core/weights/index.js +7 -0
  162. package/dist/core/weights/weights_container.d.ts +19 -0
  163. package/dist/core/weights/weights_container.js +64 -0
  164. package/dist/dataset/data_loader/image_loader.d.ts +3 -15
  165. package/dist/dataset/data_loader/image_loader.js +12 -125
  166. package/dist/dataset/data_loader/index.d.ts +2 -3
  167. package/dist/dataset/data_loader/index.js +3 -5
  168. package/dist/dataset/data_loader/tabular_loader.d.ts +3 -28
  169. package/dist/dataset/data_loader/tabular_loader.js +11 -92
  170. package/dist/imports.d.ts +2 -0
  171. package/dist/imports.js +7 -0
  172. package/dist/index.d.ts +2 -19
  173. package/dist/index.js +3 -39
  174. package/dist/memory/index.d.ts +1 -3
  175. package/dist/memory/index.js +3 -7
  176. package/dist/memory/memory.d.ts +26 -0
  177. package/dist/memory/memory.js +160 -0
  178. package/package.json +13 -26
  179. package/dist/aggregation.d.ts +0 -5
  180. package/dist/aggregation.js +0 -33
  181. package/dist/client/decentralized/base.d.ts +0 -43
  182. package/dist/client/decentralized/base.js +0 -243
  183. package/dist/client/decentralized/clear_text.d.ts +0 -13
  184. package/dist/client/decentralized/clear_text.js +0 -78
  185. package/dist/client/decentralized/messages.d.ts +0 -37
  186. package/dist/client/decentralized/messages.js +0 -15
  187. package/dist/client/decentralized/sec_agg.d.ts +0 -18
  188. package/dist/client/decentralized/sec_agg.js +0 -169
  189. package/dist/client/decentralized/secret_shares.d.ts +0 -5
  190. package/dist/client/decentralized/secret_shares.js +0 -58
  191. package/dist/client/decentralized/types.d.ts +0 -1
  192. package/dist/client/federated.d.ts +0 -30
  193. package/dist/client/federated.js +0 -218
  194. package/dist/dataset/index.d.ts +0 -2
  195. package/dist/dataset/index.js +0 -7
  196. package/dist/model_actor.d.ts +0 -16
  197. package/dist/model_actor.js +0 -20
  198. package/dist/serialization/weights.d.ts +0 -5
  199. package/dist/task/model_compile_data.d.ts +0 -6
  200. package/dist/task/model_compile_data.js +0 -12
  201. package/dist/tasks/lus_covid.d.ts +0 -4
  202. package/dist/tasks/simple_face.d.ts +0 -4
  203. package/dist/tasks/simple_face.js +0 -84
  204. package/dist/tfjs.d.ts +0 -2
  205. package/dist/tfjs.js +0 -6
  206. package/dist/training/disco.d.ts +0 -14
  207. package/dist/training/disco.js +0 -70
  208. package/dist/training/training_schemes.d.ts +0 -5
package/README.md CHANGED
@@ -1,33 +1,53 @@
1
- # discojs
1
+ # Disco.js Web Module
2
2
 
3
- discojs contains the core code of disco.
3
+ `discojs-web` contains the browser-only code of Disco.js, based off and extending `discojs-core`.
4
4
 
5
5
  ## Installation
6
6
 
7
- The app is running under Node.js v16. NPM is a package manager for the JavaScript runtime environment Node.js.
7
+ The `discojs-web` project is available as the `@epfml/discojs` NPM package, which can be installed with
8
+ `npm i @epfml/discojs`.
9
+
10
+ ### Development Environment
11
+
12
+ The dev tools run on Node.js and require `npm`, a package manager for the Node.js runtime environment.
8
13
  We recommend using [nvm](https://github.com/nvm-sh/nvm) for installing both Node.js and NPM.
9
14
 
10
- To install the application's dependencies, run the following command:
15
+ To install the project's dependencies, run:
11
16
 
12
17
  ```
18
+ cd ..
13
19
  npm ci
14
20
  ```
15
21
 
22
+ Since the dependencies of `discojs-core`, `discojs-web` and `discojs-node` are the same, they are specified in a top-level `package.json` file, to ease installation and building.
23
+
16
24
  > **⚠ WARNING: Apple Silicon.**
17
- > `TensorFlow.js` since version `3.13.0` and newer do support M1 processors for macs. To do so, make sure you have an `arm` node executable installed (not `x86`). It can be checked using:
25
+ > `TensorFlow.js` version `3` do support M1 processors for macs. To do so, make sure you have an `arm` Node.js executable installed (not `x86_64`). It can be checked using:
18
26
 
19
27
  ```
20
28
  node -p "process.arch"
21
29
  ```
22
30
 
23
- which should return `arm64`.
31
+ which should return something similar to `arm64`.
24
32
 
25
33
  ## Build
26
34
 
27
- In order to enable the browser and server modules to use the `discojs` package, we must build discojs:
35
+ The server and CLI modules, as well as all unit tests (except Cypress) use the `discojs-web` interface, i.e. they all run on Node.js. This Disco.js Node module is build on top of and extends `discojs-core`, whose code is [symlinked](https://en.wikipedia.org/wiki/Symbolic_link) into `discojs-web/src/core`. To build this project:
28
36
 
29
37
  ```
30
38
  npm run build
31
39
  ```
32
40
 
33
- This invokes the TypeScript compiler (`tsc`). To recompile from stratch, simply `rm -rf dist/` before running `npm run build` again.
41
+ This invokes the TypeScript compiler (`tsc`). It will output the compilation files of `discojs-web` in a `dist/` directory. To recompile from stratch, simply `rm -rf dist/` before running `npm run build` again.
42
+
43
+ ## Development
44
+
45
+ ### Contributing
46
+
47
+ Contributions to `discojs-web` must only include browser-specific code. Code common to both the browser and Node.js must be added to `discojs-core` instead.
48
+
49
+ As a rule of thumb, the `src/core/` directory must never be modified when modifying `discojs-web`, since it is [symlinked](https://en.wikipedia.org/wiki/Symbolic_link) to `discojs-core`.
50
+
51
+ If you wish to add a new file or submodule to the project, please do so in a similar way as `src/core/` is structured. That is, [adding a new task](../../docs/TASK.md) to `discojs-web` would mean adding a new file to `src/tasks/` and modifying `src/tasks/index.ts` (NOT `src/core/tasks/...`).
52
+
53
+ Note that, if you end up making calls to the Tensorflow.js API, you must import it from the root index. This is to ensure the Node version of TF.js is loaded, and only once.
@@ -1,5 +1,5 @@
1
- import { AsyncInformant } from './async_informant';
2
- import { TaskID } from './task';
1
+ import { Map } from 'immutable';
2
+ import { TaskID, AsyncInformant } from '.';
3
3
  /**
4
4
  * The AsyncWeightsBuffer class holds and manipulates information about the
5
5
  * async weights buffer. It works as follows:
@@ -25,7 +25,7 @@ export declare class AsyncBuffer<T> {
25
25
  buffer: Map<string, T>;
26
26
  round: number;
27
27
  private observer;
28
- constructor(taskID: TaskID, bufferCapacity: number, aggregateAndStoreWeights: (weights: T[]) => Promise<void>, roundCutoff?: number);
28
+ constructor(taskID: TaskID, bufferCapacity: number, aggregateAndStoreWeights: (weights: Iterable<T>) => Promise<void>, roundCutoff?: number);
29
29
  registerObserver(observer: AsyncInformant<T>): void;
30
30
  bufferIsFull(): boolean;
31
31
  private updateWeightsIfBufferIsFull;
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.AsyncBuffer = void 0;
4
4
  var tslib_1 = require("tslib");
5
+ var immutable_1 = require("immutable");
5
6
  /**
6
7
  * The AsyncWeightsBuffer class holds and manipulates information about the
7
8
  * async weights buffer. It works as follows:
@@ -26,7 +27,7 @@ var AsyncBuffer = /** @class */ (function () {
26
27
  this.bufferCapacity = bufferCapacity;
27
28
  this.aggregateAndStoreWeights = aggregateAndStoreWeights;
28
29
  this.roundCutoff = roundCutoff;
29
- this.buffer = new Map();
30
+ this.buffer = (0, immutable_1.Map)();
30
31
  this.round = 0;
31
32
  }
32
33
  AsyncBuffer.prototype.registerObserver = function (observer) {
@@ -39,18 +40,16 @@ var AsyncBuffer = /** @class */ (function () {
39
40
  AsyncBuffer.prototype.updateWeightsIfBufferIsFull = function () {
40
41
  var _a;
41
42
  return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
42
- var allWeights;
43
43
  return (0, tslib_1.__generator)(this, function (_b) {
44
44
  switch (_b.label) {
45
45
  case 0:
46
46
  if (!this.bufferIsFull()) return [3 /*break*/, 2];
47
- allWeights = Array.from(this.buffer.values());
48
- return [4 /*yield*/, this.aggregateAndStoreWeights(allWeights)];
47
+ return [4 /*yield*/, this.aggregateAndStoreWeights(this.buffer.values())];
49
48
  case 1:
50
49
  _b.sent();
51
50
  this.round += 1;
52
51
  (_a = this.observer) === null || _a === void 0 ? void 0 : _a.update();
53
- this.buffer.clear();
52
+ this.buffer = (0, immutable_1.Map)();
54
53
  console.log('\n************************************************************');
55
54
  console.log("Buffer is full; Aggregating weights and starting round: " + this.round + "\n");
56
55
  _b.label = 2;
@@ -84,7 +83,7 @@ var AsyncBuffer = /** @class */ (function () {
84
83
  weightsUpdatedByUser = this.buffer.has(id);
85
84
  msg = weightsUpdatedByUser ? '\tUpdating' : '-> Adding new';
86
85
  console.log(msg + " weights of " + id + " to buffer.");
87
- this.buffer.set(id, weights);
86
+ this.buffer = this.buffer.set(id, weights);
88
87
  return [4 /*yield*/, this.updateWeightsIfBufferIsFull()];
89
88
  case 1:
90
89
  _a.sent();
@@ -1,11 +1,8 @@
1
- /// <reference types="node" />
2
- import * as tf from '@tensorflow/tfjs';
3
- import { Task } from '@/task';
4
- import { TrainingInformant } from '@/informant';
5
- import { Weights } from '@/types';
1
+ import { tf, WeightsContainer, Task, TrainingInformant } from '..';
6
2
  export declare abstract class Base {
7
3
  readonly url: URL;
8
4
  readonly task: Task;
5
+ protected connected: boolean;
9
6
  constructor(url: URL, task: Task);
10
7
  /**
11
8
  * Handles the connection process from the client to any sort of
@@ -23,7 +20,7 @@ export declare abstract class Base {
23
20
  * onTrainEnd callback when training a TFJS model object. See the
24
21
  * training manager for more details.
25
22
  */
26
- abstract onTrainEndCommunication(weights: Weights, trainingInformant: TrainingInformant): Promise<void>;
23
+ abstract onTrainEndCommunication(weights: WeightsContainer, trainingInformant: TrainingInformant): Promise<void>;
27
24
  /**
28
25
  * This function will be called whenever a local round has ended.
29
26
  *
@@ -32,5 +29,5 @@ export declare abstract class Base {
32
29
  * @param round
33
30
  * @param trainingInformant
34
31
  */
35
- abstract onRoundEndCommunication(updatedWeights: Weights, staleWeights: Weights, round: number, trainingInformant: TrainingInformant): Promise<Weights>;
32
+ abstract onRoundEndCommunication(updatedWeights: WeightsContainer, staleWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<WeightsContainer>;
36
33
  }
@@ -3,11 +3,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.Base = void 0;
4
4
  var tslib_1 = require("tslib");
5
5
  var axios_1 = (0, tslib_1.__importDefault)(require("axios"));
6
- var serialization = (0, tslib_1.__importStar)(require("../serialization"));
6
+ var __1 = require("..");
7
7
  var Base = /** @class */ (function () {
8
8
  function Base(url, task) {
9
9
  this.url = url;
10
10
  this.task = task;
11
+ this.connected = false;
11
12
  }
12
13
  Base.prototype.getLatestModel = function () {
13
14
  return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
@@ -23,7 +24,7 @@ var Base = /** @class */ (function () {
23
24
  return [4 /*yield*/, axios_1.default.get(url.href)];
24
25
  case 1:
25
26
  response = _a.sent();
26
- return [4 /*yield*/, serialization.model.decode(response.data)];
27
+ return [4 /*yield*/, __1.serialization.model.decode(response.data)];
27
28
  case 2: return [2 /*return*/, _a.sent()];
28
29
  }
29
30
  });
@@ -0,0 +1,32 @@
1
+ import { List, Map } from 'immutable';
2
+ import { TrainingInformant, WeightsContainer, Task } from '../..';
3
+ import { Base as ClientBase } from '../base';
4
+ import { PeerID } from './types';
5
+ import * as messages from './messages';
6
+ import { PeerConnection } from '../event_connection';
7
+ /**
8
+ * Abstract class for decentralized clients, executes onRoundEndCommunication as well as connecting
9
+ * to the signaling server
10
+ */
11
+ export declare abstract class Base extends ClientBase {
12
+ readonly url: URL;
13
+ readonly task: Task;
14
+ protected readonly minimumReadyPeers: number;
15
+ private server?;
16
+ private peers?;
17
+ private ID?;
18
+ private pool?;
19
+ constructor(url: URL, task: Task);
20
+ private waitForPeers;
21
+ protected sendMessagetoPeer(peer: PeerConnection, msg: messages.PeerMessage): void;
22
+ private connectServer;
23
+ /**
24
+ * Initialize the connection to the peers and to the other nodes.
25
+ */
26
+ connect(): Promise<void>;
27
+ disconnect(): Promise<void>;
28
+ onTrainEndCommunication(_: WeightsContainer, trainingInformant: TrainingInformant): Promise<void>;
29
+ onRoundEndCommunication(updatedWeights: WeightsContainer, staleWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<WeightsContainer>;
30
+ abstract sendAndReceiveWeights(peers: Map<PeerID, PeerConnection>, noisyWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<List<WeightsContainer>>;
31
+ abstract clientHandle(peers: Map<PeerID, PeerConnection>): void;
32
+ }
@@ -0,0 +1,212 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Base = void 0;
4
+ var tslib_1 = require("tslib");
5
+ var immutable_1 = require("immutable");
6
+ var nodeUrl = (0, tslib_1.__importStar)(require("url"));
7
+ var __1 = require("../..");
8
+ var base_1 = require("../base");
9
+ var peer_pool_1 = require("./peer_pool");
10
+ var messages = (0, tslib_1.__importStar)(require("./messages"));
11
+ var messages_1 = require("../messages");
12
+ var event_connection_1 = require("../event_connection");
13
+ var utils_1 = require("../utils");
14
+ /**
15
+ * Abstract class for decentralized clients, executes onRoundEndCommunication as well as connecting
16
+ * to the signaling server
17
+ */
18
+ var Base = /** @class */ (function (_super) {
19
+ (0, tslib_1.__extends)(Base, _super);
20
+ function Base(url, task) {
21
+ var _a, _b;
22
+ var _this = _super.call(this, url, task) || this;
23
+ _this.url = url;
24
+ _this.task = task;
25
+ _this.minimumReadyPeers = (_b = (_a = _this.task.trainingInformation) === null || _a === void 0 ? void 0 : _a.minimumReadyPeers) !== null && _b !== void 0 ? _b : 3;
26
+ return _this;
27
+ }
28
+ // send message to server that client is ready
29
+ Base.prototype.waitForPeers = function (round) {
30
+ var _a;
31
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
32
+ var msg, receivedMessage, peers, ret;
33
+ var _this = this;
34
+ return (0, tslib_1.__generator)(this, function (_b) {
35
+ switch (_b.label) {
36
+ case 0:
37
+ console.debug(this.ID, 'is ready for round', round);
38
+ // clean old round
39
+ this.peers = undefined;
40
+ msg = { type: messages_1.type.PeerIsReady };
41
+ if (this.server === undefined) {
42
+ throw new Error('server undefined, could not connect peers');
43
+ }
44
+ this.server.send(msg);
45
+ return [4 /*yield*/, (0, event_connection_1.waitMessageWithTimeout)(this.server, messages_1.type.PeersForRound, utils_1.MAX_WAIT_PER_ROUND)];
46
+ case 1:
47
+ receivedMessage = _b.sent();
48
+ peers = (0, immutable_1.Set)(receivedMessage.peers);
49
+ if (this.ID !== undefined && peers.has(this.ID)) {
50
+ throw new Error('received peer list contains our own id');
51
+ }
52
+ if (this.peers !== undefined) {
53
+ throw new Error('got new peer list from server but was already received for this round');
54
+ }
55
+ if (peers.size + 1 < this.minimumReadyPeers) {
56
+ throw new Error('new peer list do not contain enough ready peers');
57
+ }
58
+ this.peers = peers;
59
+ console.debug(this.ID, 'got peers for round:', peers.toJS());
60
+ if (this.pool === undefined) {
61
+ throw new Error('waiting for peers but peer pool is undefined');
62
+ }
63
+ return [4 /*yield*/, this.pool];
64
+ case 2: return [4 /*yield*/, (_b.sent()).getPeers((0, immutable_1.Set)((_a = this.peers) !== null && _a !== void 0 ? _a : []), this.server, function (p) { return _this.clientHandle(p); })];
65
+ case 3:
66
+ ret = _b.sent();
67
+ console.debug(this.ID, "got peers for round " + round + ":", ret.keySeq().toJS());
68
+ return [2 /*return*/, ret];
69
+ }
70
+ });
71
+ });
72
+ };
73
+ // TODO inline? have a serialization mod
74
+ Base.prototype.sendMessagetoPeer = function (peer, msg) {
75
+ console.debug(this.ID, 'sends message to peer', msg.peer, msg);
76
+ peer.send(msg);
77
+ };
78
+ /*
79
+ creation of the websocket for the server, connection of client to that webSocket,
80
+ deals with message reception from decentralized client perspective (messages received by client)
81
+ */
82
+ Base.prototype.connectServer = function (url) {
83
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
84
+ var server;
85
+ var _this = this;
86
+ return (0, tslib_1.__generator)(this, function (_a) {
87
+ switch (_a.label) {
88
+ case 0: return [4 /*yield*/, event_connection_1.WebSocketServer.connect(url, messages.isMessageFromServer, messages.isMessageToServer)];
89
+ case 1:
90
+ server = _a.sent();
91
+ server.on(messages_1.type.SignalForPeer, function (event) {
92
+ console.debug(_this.ID, 'got signal from', event.peer);
93
+ if (_this.pool === undefined) {
94
+ throw new Error('got signal but peer pool is undefined');
95
+ }
96
+ void _this.pool.then(function (pool) { return pool.signal(event.peer, event.signal); });
97
+ });
98
+ return [2 /*return*/, server];
99
+ }
100
+ });
101
+ });
102
+ };
103
+ /**
104
+ * Initialize the connection to the peers and to the other nodes.
105
+ */
106
+ Base.prototype.connect = function () {
107
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
108
+ var URL, serverURL, _a, msg, peerIdMsg;
109
+ return (0, tslib_1.__generator)(this, function (_b) {
110
+ switch (_b.label) {
111
+ case 0:
112
+ URL = typeof window !== 'undefined' ? window.URL : nodeUrl.URL;
113
+ serverURL = new URL('', this.url.href);
114
+ switch (this.url.protocol) {
115
+ case 'http:':
116
+ serverURL.protocol = 'ws:';
117
+ break;
118
+ case 'https:':
119
+ serverURL.protocol = 'wss:';
120
+ break;
121
+ default:
122
+ throw new Error("unknown protocol: " + this.url.protocol);
123
+ }
124
+ serverURL.pathname += "deai/" + this.task.taskID;
125
+ _a = this;
126
+ return [4 /*yield*/, this.connectServer(serverURL)];
127
+ case 1:
128
+ _a.server = _b.sent();
129
+ msg = {
130
+ type: messages_1.type.clientConnected
131
+ };
132
+ this.server.send(msg);
133
+ return [4 /*yield*/, (0, event_connection_1.waitMessage)(this.server, messages_1.type.PeerID)];
134
+ case 2:
135
+ peerIdMsg = _b.sent();
136
+ console.debug(peerIdMsg.id, 'got own id from server');
137
+ if (this.ID !== undefined) {
138
+ throw new Error('got ID from server but was already received');
139
+ }
140
+ this.ID = peerIdMsg.id;
141
+ this.pool = peer_pool_1.PeerPool.init(peerIdMsg.id);
142
+ this.connected = true; // Is this still needed?
143
+ console.debug(this.ID, 'client connected to server');
144
+ return [2 /*return*/];
145
+ }
146
+ });
147
+ });
148
+ };
149
+ // disconnect from server & peers
150
+ Base.prototype.disconnect = function () {
151
+ var _a, _b;
152
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
153
+ return (0, tslib_1.__generator)(this, function (_c) {
154
+ switch (_c.label) {
155
+ case 0:
156
+ console.debug(this.ID, 'disconnect');
157
+ return [4 /*yield*/, this.pool];
158
+ case 1:
159
+ (_a = (_c.sent())) === null || _a === void 0 ? void 0 : _a.shutdown();
160
+ this.pool = undefined;
161
+ (_b = this.server) === null || _b === void 0 ? void 0 : _b.disconnect();
162
+ this.server = undefined;
163
+ this.connected = false;
164
+ return [2 /*return*/];
165
+ }
166
+ });
167
+ });
168
+ };
169
+ Base.prototype.onTrainEndCommunication = function (_, trainingInformant) {
170
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
171
+ return (0, tslib_1.__generator)(this, function (_a) {
172
+ // TODO: enter seeding mode?
173
+ trainingInformant.addMessage('Training finished.');
174
+ return [2 /*return*/];
175
+ });
176
+ });
177
+ };
178
+ Base.prototype.onRoundEndCommunication = function (updatedWeights, staleWeights, round, trainingInformant) {
179
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
180
+ var peers, noisyWeights, finalWeights, e_1, msg;
181
+ return (0, tslib_1.__generator)(this, function (_a) {
182
+ switch (_a.label) {
183
+ case 0:
184
+ _a.trys.push([0, 3, , 4]);
185
+ return [4 /*yield*/, this.waitForPeers(round)
186
+ // centralized phase of communication --> client tells server that they have finished a local round and are ready to aggregate
187
+ // Apply clipping and DP to updates that will be sent
188
+ ];
189
+ case 1:
190
+ peers = _a.sent();
191
+ noisyWeights = __1.privacy.addDifferentialPrivacy(updatedWeights, staleWeights, this.task);
192
+ return [4 /*yield*/, this.sendAndReceiveWeights(peers, noisyWeights, round, trainingInformant)];
193
+ case 2:
194
+ finalWeights = _a.sent();
195
+ console.debug(this.ID, 'sent and received', finalWeights.size, 'weights at round', round);
196
+ return [2 /*return*/, __1.aggregation.avg(finalWeights)];
197
+ case 3:
198
+ e_1 = _a.sent();
199
+ msg = "errored on round " + round;
200
+ if (e_1 instanceof Error) {
201
+ msg += ": " + e_1.message;
202
+ }
203
+ console.warn(this.ID, msg);
204
+ return [2 /*return*/, updatedWeights];
205
+ case 4: return [2 /*return*/];
206
+ }
207
+ });
208
+ });
209
+ };
210
+ return Base;
211
+ }(base_1.Base));
212
+ exports.Base = Base;
@@ -0,0 +1,14 @@
1
+ import { List, Map } from 'immutable';
2
+ import { TrainingInformant, WeightsContainer } from '../..';
3
+ import { Base } from './base';
4
+ import { PeerID } from './types';
5
+ import { PeerConnection } from '../event_connection';
6
+ /**
7
+ * Decentralized client that does not utilize secure aggregation, but sends model updates in clear text
8
+ */
9
+ export declare class ClearText extends Base {
10
+ private receivedWeights?;
11
+ sendAndReceiveWeights(peers: Map<PeerID, PeerConnection>, noisyWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<List<WeightsContainer>>;
12
+ private receiveWeights;
13
+ clientHandle(peers: Map<PeerID, PeerConnection>): void;
14
+ }
@@ -0,0 +1,96 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ClearText = void 0;
4
+ var tslib_1 = require("tslib");
5
+ var immutable_1 = require("immutable");
6
+ var __1 = require("../..");
7
+ var base_1 = require("./base");
8
+ var messages_1 = require("../messages");
9
+ var event_connection_1 = require("../event_connection");
10
+ /**
11
+ * Decentralized client that does not utilize secure aggregation, but sends model updates in clear text
12
+ */
13
+ var ClearText = /** @class */ (function (_super) {
14
+ (0, tslib_1.__extends)(ClearText, _super);
15
+ function ClearText() {
16
+ return _super !== null && _super.apply(this, arguments) || this;
17
+ }
18
+ ClearText.prototype.sendAndReceiveWeights = function (peers, noisyWeights, round, trainingInformant) {
19
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
20
+ var weights, ret;
21
+ var _this = this;
22
+ return (0, tslib_1.__generator)(this, function (_a) {
23
+ switch (_a.label) {
24
+ case 0:
25
+ if (!this.receivedWeights) {
26
+ throw new Error('no promise setup for receiving weights');
27
+ }
28
+ // PHASE 1 COMMUNICATION --> create weights message and send to all peers (only one phase of communication for clear_text)
29
+ // send weights asynchronously
30
+ __1.serialization.weights.encode(noisyWeights).then(function (encodedWeights) {
31
+ // create weights message and send to all peers
32
+ peers.forEach(function (peer, id) {
33
+ return _this.sendMessagetoPeer(peer, {
34
+ type: messages_1.type.Weights,
35
+ peer: id,
36
+ weights: encodedWeights
37
+ });
38
+ });
39
+ }).catch(function () {
40
+ throw new Error('error while sending weights');
41
+ });
42
+ return [4 /*yield*/, this.receivedWeights];
43
+ case 1:
44
+ weights = _a.sent();
45
+ trainingInformant.update({
46
+ currentNumberOfParticipants: weights.size + 1
47
+ });
48
+ ret = weights.push(noisyWeights);
49
+ this.receivedWeights = undefined;
50
+ return [2 /*return*/, ret];
51
+ }
52
+ });
53
+ });
54
+ };
55
+ ClearText.prototype.receiveWeights = function (peers) {
56
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
57
+ var waitWeights, receivedWeights;
58
+ var _this = this;
59
+ return (0, tslib_1.__generator)(this, function (_a) {
60
+ switch (_a.label) {
61
+ case 0:
62
+ console.debug('beginning of receiveWeights');
63
+ waitWeights = Array.from(peers.values()).map(function (peer) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () {
64
+ return (0, tslib_1.__generator)(this, function (_a) {
65
+ switch (_a.label) {
66
+ case 0: return [4 /*yield*/, (0, event_connection_1.waitMessage)(peer, messages_1.type.Weights)];
67
+ case 1: return [2 /*return*/, _a.sent()];
68
+ }
69
+ });
70
+ }); });
71
+ receivedWeights = (0, immutable_1.List)();
72
+ return [4 /*yield*/, Promise.allSettled(waitWeights)];
73
+ case 1: return [4 /*yield*/, (_a.sent()).forEach(function (message) {
74
+ if (message.status === 'fulfilled') {
75
+ receivedWeights = receivedWeights.push(__1.serialization.weights.decode(message.value.weights));
76
+ }
77
+ })];
78
+ case 2:
79
+ _a.sent();
80
+ if (receivedWeights.size < peers.size) {
81
+ throw new Error('not enough peer weights received');
82
+ }
83
+ return [2 /*return*/, receivedWeights];
84
+ }
85
+ });
86
+ });
87
+ };
88
+ /*
89
+ handles received messages from signaling server
90
+ */
91
+ ClearText.prototype.clientHandle = function (peers) {
92
+ this.receivedWeights = this.receiveWeights(peers);
93
+ };
94
+ return ClearText;
95
+ }(base_1.Base));
96
+ exports.ClearText = ClearText;
@@ -0,0 +1,41 @@
1
+ import { SignalData } from 'simple-peer';
2
+ import { weights } from '../../serialization';
3
+ import { PeerID as PeerIDType } from './types';
4
+ import { type, clientConnected } from '../messages';
5
+ export interface PeerID {
6
+ type: type.PeerID;
7
+ id: PeerIDType;
8
+ }
9
+ export interface SignalForPeer {
10
+ type: type.SignalForPeer;
11
+ peer: PeerIDType;
12
+ signal: SignalData;
13
+ }
14
+ export interface PeerIsReady {
15
+ type: type.PeerIsReady;
16
+ }
17
+ export interface PeersForRound {
18
+ type: type.PeersForRound;
19
+ peers: PeerIDType[];
20
+ }
21
+ export interface Weights {
22
+ type: type.Weights;
23
+ peer: PeerIDType;
24
+ weights: weights.Encoded;
25
+ }
26
+ export interface Shares {
27
+ type: type.Shares;
28
+ peer: PeerIDType;
29
+ weights: weights.Encoded;
30
+ }
31
+ export interface PartialSums {
32
+ type: type.PartialSums;
33
+ peer: PeerIDType;
34
+ partials: weights.Encoded;
35
+ }
36
+ export declare type MessageFromServer = PeerID | SignalForPeer | PeersForRound;
37
+ export declare type MessageToServer = clientConnected | SignalForPeer | PeerIsReady;
38
+ export declare type PeerMessage = Weights | Shares | PartialSums;
39
+ export declare function isMessageFromServer(o: unknown): o is MessageFromServer;
40
+ export declare function isMessageToServer(o: unknown): o is MessageToServer;
41
+ export declare function isPeerMessage(o: unknown): o is PeerMessage;
@@ -0,0 +1,54 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.isPeerMessage = exports.isMessageToServer = exports.isMessageFromServer = void 0;
4
+ var serialization_1 = require("../../serialization");
5
+ var types_1 = require("./types");
6
+ var messages_1 = require("../messages");
7
+ function isMessageFromServer(o) {
8
+ if (!(0, messages_1.hasMessageType)(o)) {
9
+ return false;
10
+ }
11
+ switch (o.type) {
12
+ case messages_1.type.PeerID:
13
+ return 'id' in o && (0, types_1.isPeerID)(o.id);
14
+ case messages_1.type.SignalForPeer:
15
+ return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
16
+ 'signal' in o; // TODO check signal content?
17
+ case messages_1.type.PeersForRound:
18
+ return 'peers' in o && Array.isArray(o.peers) && o.peers.every(types_1.isPeerID);
19
+ }
20
+ return false;
21
+ }
22
+ exports.isMessageFromServer = isMessageFromServer;
23
+ function isMessageToServer(o) {
24
+ if (!(0, messages_1.hasMessageType)(o)) {
25
+ return false;
26
+ }
27
+ switch (o.type) {
28
+ case messages_1.type.clientConnected:
29
+ return true;
30
+ case messages_1.type.SignalForPeer:
31
+ return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
32
+ 'signal' in o; // TODO check signal content?
33
+ case messages_1.type.PeerIsReady:
34
+ return true;
35
+ }
36
+ return false;
37
+ }
38
+ exports.isMessageToServer = isMessageToServer;
39
+ function isPeerMessage(o) {
40
+ if (!(0, messages_1.hasMessageType)(o)) {
41
+ return false;
42
+ }
43
+ switch (o.type) {
44
+ case messages_1.type.Weights:
45
+ case messages_1.type.Shares:
46
+ return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
47
+ 'weights' in o && serialization_1.weights.isEncoded(o.weights);
48
+ case messages_1.type.PartialSums:
49
+ return 'peer' in o && (0, types_1.isPeerID)(o.peer) &&
50
+ 'partials' in o && serialization_1.weights.isEncoded(o.partials);
51
+ }
52
+ return false;
53
+ }
54
+ exports.isPeerMessage = isPeerMessage;