@epfml/discojs 2.0.0 → 2.1.2-p20240506085037.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (334) hide show
  1. package/dist/aggregator/base.d.ts +180 -0
  2. package/dist/aggregator/base.js +236 -0
  3. package/dist/aggregator/get.d.ts +16 -0
  4. package/dist/aggregator/get.js +31 -0
  5. package/dist/aggregator/index.d.ts +7 -0
  6. package/dist/aggregator/index.js +4 -0
  7. package/dist/aggregator/mean.d.ts +23 -0
  8. package/dist/aggregator/mean.js +69 -0
  9. package/dist/aggregator/secure.d.ts +27 -0
  10. package/dist/aggregator/secure.js +91 -0
  11. package/dist/async_informant.d.ts +15 -0
  12. package/dist/async_informant.js +42 -0
  13. package/dist/client/base.d.ts +76 -0
  14. package/dist/client/base.js +88 -0
  15. package/dist/client/decentralized/base.d.ts +32 -0
  16. package/dist/client/decentralized/base.js +192 -0
  17. package/dist/client/decentralized/index.d.ts +2 -0
  18. package/dist/client/decentralized/index.js +2 -0
  19. package/dist/client/decentralized/messages.d.ts +28 -0
  20. package/dist/client/decentralized/messages.js +44 -0
  21. package/dist/client/decentralized/peer.d.ts +40 -0
  22. package/dist/client/decentralized/peer.js +189 -0
  23. package/dist/client/decentralized/peer_pool.d.ts +12 -0
  24. package/dist/client/decentralized/peer_pool.js +44 -0
  25. package/dist/client/event_connection.d.ts +34 -0
  26. package/dist/client/event_connection.js +105 -0
  27. package/dist/client/federated/base.d.ts +54 -0
  28. package/dist/client/federated/base.js +151 -0
  29. package/dist/client/federated/index.d.ts +2 -0
  30. package/dist/client/federated/index.js +2 -0
  31. package/dist/client/federated/messages.d.ts +30 -0
  32. package/dist/client/federated/messages.js +24 -0
  33. package/dist/client/index.d.ts +8 -0
  34. package/dist/client/index.js +8 -0
  35. package/dist/client/local.d.ts +3 -0
  36. package/dist/client/local.js +3 -0
  37. package/dist/client/messages.d.ts +30 -0
  38. package/dist/client/messages.js +26 -0
  39. package/dist/client/types.d.ts +2 -0
  40. package/dist/client/types.js +4 -0
  41. package/dist/client/utils.d.ts +2 -0
  42. package/dist/client/utils.js +7 -0
  43. package/dist/dataset/data/data.d.ts +48 -0
  44. package/dist/dataset/data/data.js +72 -0
  45. package/dist/dataset/data/data_split.d.ts +8 -0
  46. package/dist/dataset/data/data_split.js +1 -0
  47. package/dist/dataset/data/image_data.d.ts +11 -0
  48. package/dist/dataset/data/image_data.js +38 -0
  49. package/dist/dataset/data/index.d.ts +6 -0
  50. package/dist/dataset/data/index.js +5 -0
  51. package/dist/dataset/data/preprocessing/base.d.ts +16 -0
  52. package/dist/dataset/data/preprocessing/base.js +1 -0
  53. package/dist/dataset/data/preprocessing/image_preprocessing.d.ts +13 -0
  54. package/dist/dataset/data/preprocessing/image_preprocessing.js +40 -0
  55. package/dist/dataset/data/preprocessing/index.d.ts +4 -0
  56. package/dist/dataset/data/preprocessing/index.js +3 -0
  57. package/dist/dataset/data/preprocessing/tabular_preprocessing.d.ts +13 -0
  58. package/dist/dataset/data/preprocessing/tabular_preprocessing.js +45 -0
  59. package/dist/dataset/data/preprocessing/text_preprocessing.d.ts +13 -0
  60. package/dist/dataset/data/preprocessing/text_preprocessing.js +85 -0
  61. package/dist/dataset/data/tabular_data.d.ts +11 -0
  62. package/dist/dataset/data/tabular_data.js +25 -0
  63. package/dist/dataset/data/text_data.d.ts +11 -0
  64. package/dist/dataset/data/text_data.js +14 -0
  65. package/dist/{core/dataset → dataset}/data_loader/data_loader.d.ts +3 -5
  66. package/dist/dataset/data_loader/data_loader.js +2 -0
  67. package/dist/dataset/data_loader/image_loader.d.ts +20 -3
  68. package/dist/dataset/data_loader/image_loader.js +98 -23
  69. package/dist/dataset/data_loader/index.d.ts +5 -2
  70. package/dist/dataset/data_loader/index.js +4 -7
  71. package/dist/dataset/data_loader/tabular_loader.d.ts +34 -3
  72. package/dist/dataset/data_loader/tabular_loader.js +75 -15
  73. package/dist/dataset/data_loader/text_loader.d.ts +14 -0
  74. package/dist/dataset/data_loader/text_loader.js +25 -0
  75. package/dist/dataset/dataset.d.ts +5 -0
  76. package/dist/dataset/dataset.js +1 -0
  77. package/dist/dataset/dataset_builder.d.ts +60 -0
  78. package/dist/dataset/dataset_builder.js +142 -0
  79. package/dist/dataset/index.d.ts +5 -0
  80. package/dist/dataset/index.js +3 -0
  81. package/dist/default_tasks/cifar10/index.d.ts +2 -0
  82. package/dist/default_tasks/cifar10/index.js +60 -0
  83. package/dist/default_tasks/cifar10/model.d.ts +434 -0
  84. package/dist/default_tasks/cifar10/model.js +2385 -0
  85. package/dist/default_tasks/geotags/index.d.ts +2 -0
  86. package/dist/default_tasks/geotags/index.js +65 -0
  87. package/dist/default_tasks/geotags/model.d.ts +593 -0
  88. package/dist/default_tasks/geotags/model.js +4715 -0
  89. package/dist/default_tasks/index.d.ts +8 -0
  90. package/dist/default_tasks/index.js +8 -0
  91. package/dist/default_tasks/lus_covid.d.ts +2 -0
  92. package/dist/default_tasks/lus_covid.js +89 -0
  93. package/dist/default_tasks/mnist.d.ts +2 -0
  94. package/dist/default_tasks/mnist.js +61 -0
  95. package/dist/default_tasks/simple_face/index.d.ts +2 -0
  96. package/dist/default_tasks/simple_face/index.js +48 -0
  97. package/dist/default_tasks/simple_face/model.d.ts +513 -0
  98. package/dist/default_tasks/simple_face/model.js +4301 -0
  99. package/dist/default_tasks/skin_mnist.d.ts +2 -0
  100. package/dist/default_tasks/skin_mnist.js +80 -0
  101. package/dist/default_tasks/titanic.d.ts +2 -0
  102. package/dist/default_tasks/titanic.js +88 -0
  103. package/dist/default_tasks/wikitext.d.ts +2 -0
  104. package/dist/default_tasks/wikitext.js +38 -0
  105. package/dist/index.d.ts +18 -2
  106. package/dist/index.js +18 -6
  107. package/dist/{core/informant → informant}/graph_informant.d.ts +1 -1
  108. package/dist/informant/graph_informant.js +20 -0
  109. package/dist/informant/index.d.ts +1 -0
  110. package/dist/informant/index.js +1 -0
  111. package/dist/{core/logging → logging}/console_logger.d.ts +2 -2
  112. package/dist/logging/console_logger.js +22 -0
  113. package/dist/logging/index.d.ts +2 -0
  114. package/dist/logging/index.js +1 -0
  115. package/dist/{core/logging → logging}/logger.d.ts +3 -3
  116. package/dist/logging/logger.js +1 -0
  117. package/dist/memory/base.d.ts +119 -0
  118. package/dist/memory/base.js +9 -0
  119. package/dist/memory/empty.d.ts +20 -0
  120. package/dist/memory/empty.js +43 -0
  121. package/dist/memory/index.d.ts +3 -1
  122. package/dist/memory/index.js +3 -5
  123. package/dist/memory/model_type.d.ts +9 -0
  124. package/dist/memory/model_type.js +10 -0
  125. package/dist/{core/privacy.d.ts → privacy.d.ts} +1 -1
  126. package/dist/{core/privacy.js → privacy.js} +11 -16
  127. package/dist/serialization/index.d.ts +2 -0
  128. package/dist/serialization/index.js +2 -0
  129. package/dist/serialization/model.d.ts +5 -0
  130. package/dist/serialization/model.js +67 -0
  131. package/dist/{core/serialization → serialization}/weights.d.ts +2 -2
  132. package/dist/serialization/weights.js +37 -0
  133. package/dist/task/data_example.js +14 -0
  134. package/dist/task/digest.d.ts +5 -0
  135. package/dist/task/digest.js +14 -0
  136. package/dist/{core/task → task}/display_information.d.ts +5 -3
  137. package/dist/task/display_information.js +46 -0
  138. package/dist/task/index.d.ts +7 -0
  139. package/dist/task/index.js +5 -0
  140. package/dist/task/label_type.d.ts +9 -0
  141. package/dist/task/label_type.js +28 -0
  142. package/dist/task/summary.js +13 -0
  143. package/dist/task/task.d.ts +12 -0
  144. package/dist/task/task.js +22 -0
  145. package/dist/task/task_handler.d.ts +5 -0
  146. package/dist/task/task_handler.js +20 -0
  147. package/dist/task/task_provider.d.ts +5 -0
  148. package/dist/task/task_provider.js +1 -0
  149. package/dist/{core/task → task}/training_information.d.ts +9 -10
  150. package/dist/task/training_information.js +88 -0
  151. package/dist/training/disco.d.ts +40 -0
  152. package/dist/training/disco.js +107 -0
  153. package/dist/training/index.d.ts +2 -0
  154. package/dist/training/index.js +1 -0
  155. package/dist/training/trainer/distributed_trainer.d.ts +20 -0
  156. package/dist/training/trainer/distributed_trainer.js +36 -0
  157. package/dist/training/trainer/local_trainer.d.ts +12 -0
  158. package/dist/training/trainer/local_trainer.js +19 -0
  159. package/dist/training/trainer/trainer.d.ts +33 -0
  160. package/dist/training/trainer/trainer.js +52 -0
  161. package/dist/{core/training → training}/trainer/trainer_builder.d.ts +5 -7
  162. package/dist/training/trainer/trainer_builder.js +43 -0
  163. package/dist/types.d.ts +8 -0
  164. package/dist/types.js +1 -0
  165. package/dist/utils/event_emitter.d.ts +40 -0
  166. package/dist/utils/event_emitter.js +57 -0
  167. package/dist/validation/index.d.ts +1 -0
  168. package/dist/validation/index.js +1 -0
  169. package/dist/validation/validator.d.ts +28 -0
  170. package/dist/validation/validator.js +132 -0
  171. package/dist/weights/aggregation.d.ts +21 -0
  172. package/dist/weights/aggregation.js +44 -0
  173. package/dist/weights/index.d.ts +2 -0
  174. package/dist/weights/index.js +2 -0
  175. package/dist/weights/weights_container.d.ts +68 -0
  176. package/dist/weights/weights_container.js +96 -0
  177. package/package.json +25 -16
  178. package/README.md +0 -53
  179. package/dist/core/async_buffer.d.ts +0 -41
  180. package/dist/core/async_buffer.js +0 -97
  181. package/dist/core/async_informant.d.ts +0 -20
  182. package/dist/core/async_informant.js +0 -69
  183. package/dist/core/client/base.d.ts +0 -33
  184. package/dist/core/client/base.js +0 -35
  185. package/dist/core/client/decentralized/base.d.ts +0 -32
  186. package/dist/core/client/decentralized/base.js +0 -212
  187. package/dist/core/client/decentralized/clear_text.d.ts +0 -14
  188. package/dist/core/client/decentralized/clear_text.js +0 -96
  189. package/dist/core/client/decentralized/index.d.ts +0 -4
  190. package/dist/core/client/decentralized/index.js +0 -9
  191. package/dist/core/client/decentralized/messages.d.ts +0 -41
  192. package/dist/core/client/decentralized/messages.js +0 -54
  193. package/dist/core/client/decentralized/peer.d.ts +0 -26
  194. package/dist/core/client/decentralized/peer.js +0 -210
  195. package/dist/core/client/decentralized/peer_pool.d.ts +0 -14
  196. package/dist/core/client/decentralized/peer_pool.js +0 -92
  197. package/dist/core/client/decentralized/sec_agg.d.ts +0 -22
  198. package/dist/core/client/decentralized/sec_agg.js +0 -190
  199. package/dist/core/client/decentralized/secret_shares.d.ts +0 -3
  200. package/dist/core/client/decentralized/secret_shares.js +0 -39
  201. package/dist/core/client/decentralized/types.d.ts +0 -2
  202. package/dist/core/client/decentralized/types.js +0 -7
  203. package/dist/core/client/event_connection.d.ts +0 -37
  204. package/dist/core/client/event_connection.js +0 -158
  205. package/dist/core/client/federated/client.d.ts +0 -37
  206. package/dist/core/client/federated/client.js +0 -273
  207. package/dist/core/client/federated/index.d.ts +0 -2
  208. package/dist/core/client/federated/index.js +0 -7
  209. package/dist/core/client/federated/messages.d.ts +0 -38
  210. package/dist/core/client/federated/messages.js +0 -25
  211. package/dist/core/client/index.d.ts +0 -5
  212. package/dist/core/client/index.js +0 -11
  213. package/dist/core/client/local.d.ts +0 -8
  214. package/dist/core/client/local.js +0 -36
  215. package/dist/core/client/messages.d.ts +0 -28
  216. package/dist/core/client/messages.js +0 -33
  217. package/dist/core/client/utils.d.ts +0 -2
  218. package/dist/core/client/utils.js +0 -19
  219. package/dist/core/dataset/data/data.d.ts +0 -11
  220. package/dist/core/dataset/data/data.js +0 -20
  221. package/dist/core/dataset/data/data_split.d.ts +0 -5
  222. package/dist/core/dataset/data/data_split.js +0 -2
  223. package/dist/core/dataset/data/image_data.d.ts +0 -8
  224. package/dist/core/dataset/data/image_data.js +0 -64
  225. package/dist/core/dataset/data/index.d.ts +0 -5
  226. package/dist/core/dataset/data/index.js +0 -11
  227. package/dist/core/dataset/data/preprocessing.d.ts +0 -13
  228. package/dist/core/dataset/data/preprocessing.js +0 -33
  229. package/dist/core/dataset/data/tabular_data.d.ts +0 -8
  230. package/dist/core/dataset/data/tabular_data.js +0 -40
  231. package/dist/core/dataset/data_loader/data_loader.js +0 -10
  232. package/dist/core/dataset/data_loader/image_loader.d.ts +0 -17
  233. package/dist/core/dataset/data_loader/image_loader.js +0 -141
  234. package/dist/core/dataset/data_loader/index.d.ts +0 -3
  235. package/dist/core/dataset/data_loader/index.js +0 -9
  236. package/dist/core/dataset/data_loader/tabular_loader.d.ts +0 -29
  237. package/dist/core/dataset/data_loader/tabular_loader.js +0 -101
  238. package/dist/core/dataset/dataset.d.ts +0 -2
  239. package/dist/core/dataset/dataset.js +0 -2
  240. package/dist/core/dataset/dataset_builder.d.ts +0 -18
  241. package/dist/core/dataset/dataset_builder.js +0 -96
  242. package/dist/core/dataset/index.d.ts +0 -4
  243. package/dist/core/dataset/index.js +0 -14
  244. package/dist/core/index.d.ts +0 -18
  245. package/dist/core/index.js +0 -41
  246. package/dist/core/informant/graph_informant.js +0 -23
  247. package/dist/core/informant/index.d.ts +0 -3
  248. package/dist/core/informant/index.js +0 -9
  249. package/dist/core/informant/training_informant/base.d.ts +0 -31
  250. package/dist/core/informant/training_informant/base.js +0 -83
  251. package/dist/core/informant/training_informant/decentralized.d.ts +0 -5
  252. package/dist/core/informant/training_informant/decentralized.js +0 -22
  253. package/dist/core/informant/training_informant/federated.d.ts +0 -14
  254. package/dist/core/informant/training_informant/federated.js +0 -32
  255. package/dist/core/informant/training_informant/index.d.ts +0 -4
  256. package/dist/core/informant/training_informant/index.js +0 -11
  257. package/dist/core/informant/training_informant/local.d.ts +0 -6
  258. package/dist/core/informant/training_informant/local.js +0 -20
  259. package/dist/core/logging/console_logger.js +0 -33
  260. package/dist/core/logging/index.d.ts +0 -3
  261. package/dist/core/logging/index.js +0 -9
  262. package/dist/core/logging/logger.js +0 -9
  263. package/dist/core/logging/trainer_logger.d.ts +0 -24
  264. package/dist/core/logging/trainer_logger.js +0 -59
  265. package/dist/core/memory/base.d.ts +0 -22
  266. package/dist/core/memory/base.js +0 -9
  267. package/dist/core/memory/empty.d.ts +0 -14
  268. package/dist/core/memory/empty.js +0 -75
  269. package/dist/core/memory/index.d.ts +0 -3
  270. package/dist/core/memory/index.js +0 -9
  271. package/dist/core/memory/model_type.d.ts +0 -4
  272. package/dist/core/memory/model_type.js +0 -9
  273. package/dist/core/serialization/index.d.ts +0 -2
  274. package/dist/core/serialization/index.js +0 -6
  275. package/dist/core/serialization/model.d.ts +0 -5
  276. package/dist/core/serialization/model.js +0 -55
  277. package/dist/core/serialization/weights.js +0 -64
  278. package/dist/core/task/data_example.js +0 -24
  279. package/dist/core/task/display_information.js +0 -49
  280. package/dist/core/task/index.d.ts +0 -3
  281. package/dist/core/task/index.js +0 -8
  282. package/dist/core/task/model_compile_data.d.ts +0 -6
  283. package/dist/core/task/model_compile_data.js +0 -22
  284. package/dist/core/task/summary.js +0 -19
  285. package/dist/core/task/task.d.ts +0 -10
  286. package/dist/core/task/task.js +0 -31
  287. package/dist/core/task/training_information.js +0 -66
  288. package/dist/core/tasks/cifar10.d.ts +0 -3
  289. package/dist/core/tasks/cifar10.js +0 -65
  290. package/dist/core/tasks/geotags.d.ts +0 -3
  291. package/dist/core/tasks/geotags.js +0 -67
  292. package/dist/core/tasks/index.d.ts +0 -6
  293. package/dist/core/tasks/index.js +0 -10
  294. package/dist/core/tasks/lus_covid.d.ts +0 -3
  295. package/dist/core/tasks/lus_covid.js +0 -87
  296. package/dist/core/tasks/mnist.d.ts +0 -3
  297. package/dist/core/tasks/mnist.js +0 -60
  298. package/dist/core/tasks/simple_face.d.ts +0 -2
  299. package/dist/core/tasks/simple_face.js +0 -41
  300. package/dist/core/tasks/titanic.d.ts +0 -3
  301. package/dist/core/tasks/titanic.js +0 -88
  302. package/dist/core/training/disco.d.ts +0 -23
  303. package/dist/core/training/disco.js +0 -130
  304. package/dist/core/training/index.d.ts +0 -2
  305. package/dist/core/training/index.js +0 -7
  306. package/dist/core/training/trainer/distributed_trainer.d.ts +0 -20
  307. package/dist/core/training/trainer/distributed_trainer.js +0 -65
  308. package/dist/core/training/trainer/local_trainer.d.ts +0 -11
  309. package/dist/core/training/trainer/local_trainer.js +0 -34
  310. package/dist/core/training/trainer/round_tracker.d.ts +0 -30
  311. package/dist/core/training/trainer/round_tracker.js +0 -47
  312. package/dist/core/training/trainer/trainer.d.ts +0 -65
  313. package/dist/core/training/trainer/trainer.js +0 -160
  314. package/dist/core/training/trainer/trainer_builder.js +0 -95
  315. package/dist/core/training/training_schemes.d.ts +0 -5
  316. package/dist/core/training/training_schemes.js +0 -10
  317. package/dist/core/types.d.ts +0 -4
  318. package/dist/core/types.js +0 -2
  319. package/dist/core/validation/index.d.ts +0 -1
  320. package/dist/core/validation/index.js +0 -5
  321. package/dist/core/validation/validator.d.ts +0 -17
  322. package/dist/core/validation/validator.js +0 -104
  323. package/dist/core/weights/aggregation.d.ts +0 -8
  324. package/dist/core/weights/aggregation.js +0 -96
  325. package/dist/core/weights/index.d.ts +0 -2
  326. package/dist/core/weights/index.js +0 -7
  327. package/dist/core/weights/weights_container.d.ts +0 -19
  328. package/dist/core/weights/weights_container.js +0 -64
  329. package/dist/imports.d.ts +0 -2
  330. package/dist/imports.js +0 -7
  331. package/dist/memory/memory.d.ts +0 -26
  332. package/dist/memory/memory.js +0 -160
  333. package/dist/{core/task → task}/data_example.d.ts +1 -1
  334. package/dist/{core/task → task}/summary.d.ts +1 -1
@@ -1,14 +0,0 @@
1
- import { Map, Set } from 'immutable';
2
- import { SignalData } from 'simple-peer';
3
- import { PeerID } from './types';
4
- import { PeerConnection, EventConnection } from '../event_connection';
5
- export declare class PeerPool {
6
- private readonly id;
7
- private readonly wrtc?;
8
- private peers;
9
- private constructor();
10
- static init(id: PeerID): Promise<PeerPool>;
11
- shutdown(): void;
12
- signal(peerID: PeerID, signal: SignalData): void;
13
- getPeers(peersToConnect: Set<PeerID>, signallingServer: EventConnection, clientHandle: (connections: Map<PeerID, PeerConnection>) => void): Promise<Map<PeerID, PeerConnection>>;
14
- }
@@ -1,92 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.PeerPool = void 0;
4
- var tslib_1 = require("tslib");
5
- var immutable_1 = require("immutable");
6
- var peer_1 = require("./peer");
7
- var event_connection_1 = require("../event_connection");
8
- // TODO cleanup old peers
9
- var PeerPool = /** @class */ (function () {
10
- function PeerPool(id, wrtc) {
11
- this.id = id;
12
- this.wrtc = wrtc;
13
- this.peers = (0, immutable_1.Map)();
14
- }
15
- PeerPool.init = function (id) {
16
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
17
- var wrtc, path, e_1;
18
- return (0, tslib_1.__generator)(this, function (_a) {
19
- switch (_a.label) {
20
- case 0:
21
- _a.trys.push([0, 2, , 3]);
22
- path = require.resolve('@koush/wrtc', { paths: ['.'] });
23
- return [4 /*yield*/, Promise.resolve().then(function () { return (0, tslib_1.__importStar)(require(path)); })];
24
- case 1:
25
- wrtc = _a.sent();
26
- return [3 /*break*/, 3];
27
- case 2:
28
- e_1 = _a.sent();
29
- return [3 /*break*/, 3];
30
- case 3: return [2 /*return*/, new PeerPool(id, wrtc)];
31
- }
32
- });
33
- });
34
- };
35
- PeerPool.prototype.shutdown = function () {
36
- console.debug(this.id, 'shutdown their peers');
37
- this.peers.forEach(function (peer) { return peer.disconnect(); });
38
- this.peers = (0, immutable_1.Map)();
39
- };
40
- PeerPool.prototype.signal = function (peerID, signal) {
41
- console.debug(this.id, 'signals for', peerID);
42
- var peer = this.peers.get(peerID);
43
- if (peer === undefined) {
44
- throw new Error("received signal for unknown peer: " + peerID);
45
- }
46
- peer.signal(signal);
47
- };
48
- PeerPool.prototype.getPeers = function (peersToConnect, signallingServer,
49
- // TODO as event?
50
- clientHandle) {
51
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
52
- var newPeers, newPeersConnections;
53
- var _this = this;
54
- return (0, tslib_1.__generator)(this, function (_a) {
55
- switch (_a.label) {
56
- case 0:
57
- if (peersToConnect.contains(this.id)) {
58
- throw new Error('peers to connect contains our id');
59
- }
60
- console.debug(this.id, 'is connecting peers:', peersToConnect.toJS());
61
- newPeers = (0, immutable_1.Map)(peersToConnect
62
- .filter(function (id) { return !_this.peers.has(id); })
63
- .map(function (id) { return [id, id < _this.id]; })
64
- .map(function (_a) {
65
- var _b = (0, tslib_1.__read)(_a, 2), id = _b[0], initiator = _b[1];
66
- var p = new peer_1.Peer(id, { initiator: initiator, wrtc: _this.wrtc });
67
- // onNewPeer(id, p)
68
- return [id, p];
69
- }));
70
- console.debug(this.id, 'asked to connect new peers:', newPeers.keySeq().toJS());
71
- newPeersConnections = newPeers.map(function (peer, id) { return new event_connection_1.PeerConnection(_this.id, peer, signallingServer); });
72
- // adding peers to pool before connecting them because they must be set to call signal on them
73
- this.peers = this.peers.merge(newPeersConnections);
74
- clientHandle(this.peers);
75
- return [4 /*yield*/, Promise.all(Array.from(newPeersConnections.values()).map(function (connection) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () { return (0, tslib_1.__generator)(this, function (_a) {
76
- switch (_a.label) {
77
- case 0: return [4 /*yield*/, connection.connect()];
78
- case 1: return [2 /*return*/, _a.sent()];
79
- }
80
- }); }); }))];
81
- case 1:
82
- _a.sent();
83
- console.debug(this.id, 'knowns connected peers:', this.peers.keySeq().toJS());
84
- return [2 /*return*/, this.peers
85
- .filter(function (_, id) { return peersToConnect.has(id); })];
86
- }
87
- });
88
- });
89
- };
90
- return PeerPool;
91
- }());
92
- exports.PeerPool = PeerPool;
@@ -1,22 +0,0 @@
1
- import { List, Map } from 'immutable';
2
- import { Task, TrainingInformant, WeightsContainer } from '../..';
3
- import { Base } from './base';
4
- import { PeerID } from './types';
5
- import { PeerConnection } from '../event_connection';
6
- /**
7
- * Decentralized client that utilizes secure aggregation so client updates remain private
8
- */
9
- export declare class SecAgg extends Base {
10
- readonly url: URL;
11
- readonly task: Task;
12
- private readonly maxShareValue;
13
- private receivedShares?;
14
- private receivedPartialSums?;
15
- constructor(url: URL, task: Task);
16
- private sendShares;
17
- private sendPartialSums;
18
- sendAndReceiveWeights(peers: Map<PeerID, PeerConnection>, noisyWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<List<WeightsContainer>>;
19
- private receiveShares;
20
- private receivePartials;
21
- clientHandle(peers: Map<PeerID, PeerConnection>): void;
22
- }
@@ -1,190 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.SecAgg = void 0;
4
- var tslib_1 = require("tslib");
5
- var immutable_1 = require("immutable");
6
- var __1 = require("../..");
7
- var base_1 = require("./base");
8
- var messages_1 = require("../messages");
9
- var secret_shares = (0, tslib_1.__importStar)(require("./secret_shares"));
10
- var event_connection_1 = require("../event_connection");
11
- /**
12
- * Decentralized client that utilizes secure aggregation so client updates remain private
13
- */
14
- var SecAgg = /** @class */ (function (_super) {
15
- (0, tslib_1.__extends)(SecAgg, _super);
16
- function SecAgg(url, task) {
17
- var _a, _b;
18
- var _this = _super.call(this, url, task) || this;
19
- _this.url = url;
20
- _this.task = task;
21
- _this.maxShareValue = (_b = (_a = _this.task.trainingInformation) === null || _a === void 0 ? void 0 : _a.maxShareValue) !== null && _b !== void 0 ? _b : 100;
22
- return _this;
23
- }
24
- /*
25
- generates shares and sends to all ready peers adds differential privacy
26
- */
27
- SecAgg.prototype.sendShares = function (peers, weightShares, round, trainingInformant) {
28
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
29
- var encodedWeightShares, _a;
30
- var _this = this;
31
- return (0, tslib_1.__generator)(this, function (_b) {
32
- switch (_b.label) {
33
- case 0:
34
- _a = immutable_1.List;
35
- return [4 /*yield*/, Promise.all(weightShares.rest().map(function (weights) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () { return (0, tslib_1.__generator)(this, function (_a) {
36
- switch (_a.label) {
37
- case 0: return [4 /*yield*/, __1.serialization.weights.encode(weights)];
38
- case 1: return [2 /*return*/, _a.sent()];
39
- }
40
- }); }); }))];
41
- case 1:
42
- encodedWeightShares = _a.apply(void 0, [_b.sent()]);
43
- // Broadcast our weights to ith peer in the SERVER LIST OF PEERS (seen in signaling_server.ts)
44
- peers
45
- .entrySeq()
46
- .toSeq()
47
- .zip(encodedWeightShares)
48
- .forEach(function (_a) {
49
- var _b = (0, tslib_1.__read)(_a, 2), _c = (0, tslib_1.__read)(_b[0], 2), id = _c[0], peer = _c[1], weights = _b[1];
50
- return _this.sendMessagetoPeer(peer, {
51
- type: messages_1.type.Shares,
52
- peer: id,
53
- weights: weights
54
- });
55
- });
56
- return [2 /*return*/];
57
- }
58
- });
59
- });
60
- };
61
- /*
62
- sends partial sums to connected peers so final update can be calculated
63
- */
64
- SecAgg.prototype.sendPartialSums = function (partial, peers) {
65
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
66
- var myEncodedSum;
67
- var _this = this;
68
- return (0, tslib_1.__generator)(this, function (_a) {
69
- switch (_a.label) {
70
- case 0: return [4 /*yield*/, __1.serialization.weights.encode(partial)
71
- // calculate, encode, and send sum
72
- ];
73
- case 1:
74
- myEncodedSum = _a.sent();
75
- // calculate, encode, and send sum
76
- peers.forEach(function (peer, id) {
77
- return _this.sendMessagetoPeer(peer, {
78
- type: messages_1.type.PartialSums,
79
- peer: id,
80
- partials: myEncodedSum
81
- });
82
- });
83
- return [2 /*return*/];
84
- }
85
- });
86
- });
87
- };
88
- SecAgg.prototype.sendAndReceiveWeights = function (peers, noisyWeights, round, trainingInformant) {
89
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
90
- var weightShares, shares, mySum, partials;
91
- return (0, tslib_1.__generator)(this, function (_a) {
92
- switch (_a.label) {
93
- case 0:
94
- if (!this.receivedShares || !this.receivedPartialSums) {
95
- throw new Error('no promise setup for receiving weights');
96
- }
97
- weightShares = secret_shares.generateAllShares(noisyWeights, peers.size + 1, this.maxShareValue);
98
- return [4 /*yield*/, this.sendShares(peers, weightShares, round, trainingInformant)];
99
- case 1:
100
- _a.sent();
101
- return [4 /*yield*/, this.receivedShares
102
- // add own share to list
103
- ];
104
- case 2:
105
- shares = _a.sent();
106
- // add own share to list
107
- shares = shares.insert(0, weightShares.first());
108
- mySum = __1.aggregation.sum(shares);
109
- void this.sendPartialSums(mySum, peers);
110
- return [4 /*yield*/, this.receivedPartialSums];
111
- case 3:
112
- partials = _a.sent();
113
- partials = partials.insert(0, mySum);
114
- trainingInformant.update({
115
- currentNumberOfParticipants: partials.size
116
- });
117
- // resets state
118
- this.receivedPartialSums = undefined;
119
- this.receivedShares = undefined;
120
- return [2 /*return*/, partials];
121
- }
122
- });
123
- });
124
- };
125
- SecAgg.prototype.receiveShares = function (peers) {
126
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
127
- var sharesPromises, receivedShares, sharesMessages;
128
- var _this = this;
129
- return (0, tslib_1.__generator)(this, function (_a) {
130
- switch (_a.label) {
131
- case 0:
132
- sharesPromises = Array.from(peers.values()).map(function (peer) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () { return (0, tslib_1.__generator)(this, function (_a) {
133
- switch (_a.label) {
134
- case 0: return [4 /*yield*/, (0, event_connection_1.waitMessage)(peer, messages_1.type.Shares)];
135
- case 1: return [2 /*return*/, _a.sent()];
136
- }
137
- }); }); });
138
- receivedShares = (0, immutable_1.List)();
139
- return [4 /*yield*/, Promise.all(sharesPromises)];
140
- case 1:
141
- sharesMessages = _a.sent();
142
- sharesMessages.forEach(function (message) {
143
- receivedShares = receivedShares.push(__1.serialization.weights.decode(message.weights));
144
- });
145
- if (receivedShares.size < peers.size) {
146
- throw new Error('Not enough shares received');
147
- }
148
- return [2 /*return*/, receivedShares];
149
- }
150
- });
151
- });
152
- };
153
- SecAgg.prototype.receivePartials = function (peers) {
154
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
155
- var partialsPromises, receivedPartials, partialMessages;
156
- var _this = this;
157
- return (0, tslib_1.__generator)(this, function (_a) {
158
- switch (_a.label) {
159
- case 0:
160
- partialsPromises = Array.from(peers.values()).map(function (peer) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () { return (0, tslib_1.__generator)(this, function (_a) {
161
- switch (_a.label) {
162
- case 0: return [4 /*yield*/, (0, event_connection_1.waitMessage)(peer, messages_1.type.PartialSums)];
163
- case 1: return [2 /*return*/, _a.sent()];
164
- }
165
- }); }); });
166
- receivedPartials = (0, immutable_1.List)();
167
- return [4 /*yield*/, Promise.all(partialsPromises)];
168
- case 1:
169
- partialMessages = _a.sent();
170
- partialMessages.forEach(function (message) {
171
- receivedPartials = receivedPartials.push(__1.serialization.weights.decode(message.partials));
172
- });
173
- if (receivedPartials.size < peers.size) {
174
- throw new Error('Not enough partials received');
175
- }
176
- return [2 /*return*/, receivedPartials];
177
- }
178
- });
179
- });
180
- };
181
- /*
182
- handles received messages from signaling server
183
- */
184
- SecAgg.prototype.clientHandle = function (peers) {
185
- this.receivedShares = this.receiveShares(peers);
186
- this.receivedPartialSums = this.receivePartials(peers);
187
- };
188
- return SecAgg;
189
- }(base_1.Base));
190
- exports.SecAgg = SecAgg;
@@ -1,3 +0,0 @@
1
- import { List } from 'immutable';
2
- import { WeightsContainer } from '../..';
3
- export declare function generateAllShares(secret: WeightsContainer, nParticipants: number, maxShareValue: number): List<WeightsContainer>;
@@ -1,39 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.generateAllShares = void 0;
4
- var tslib_1 = require("tslib");
5
- var immutable_1 = require("immutable");
6
- var crypto = (0, tslib_1.__importStar)(require("crypto"));
7
- var __1 = require("../..");
8
- var maxSeed = Math.pow(2, 47);
9
- /*
10
- Return Weights in the remaining share once N-1 shares have been constructed (where N is number of ready clients)
11
- */
12
- function lastShare(currentShares, secret) {
13
- if (currentShares.size === 0) {
14
- throw new Error('Need at least one current share to be able to subtract secret from');
15
- }
16
- return secret.sub(__1.aggregation.sum(currentShares));
17
- }
18
- /*
19
- Generate N additive shares that aggregate to the secret weights array (where N is number of ready clients)
20
- */
21
- function generateAllShares(secret, nParticipants, maxShareValue) {
22
- if (nParticipants < 1) {
23
- throw new Error('too few participants to genreate shares');
24
- }
25
- var randomShares = (0, immutable_1.Range)(0, nParticipants - 1)
26
- .map(function () { return generateRandomShare(secret, maxShareValue); })
27
- .toList();
28
- return randomShares
29
- .push(lastShare(randomShares, secret));
30
- }
31
- exports.generateAllShares = generateAllShares;
32
- /*
33
- generates one share in the same shape as the secret that is populated with values randomly chosend from
34
- a uniform distribution between (-maxShareValue, maxShareValue).
35
- */
36
- function generateRandomShare(secret, maxShareValue) {
37
- var seed = crypto.randomInt(maxSeed);
38
- return secret.map(function (t) { return __1.tf.randomUniform(t.shape, -maxShareValue, maxShareValue, 'float32', seed); });
39
- }
@@ -1,2 +0,0 @@
1
- export declare type PeerID = number;
2
- export declare function isPeerID(raw: unknown): raw is PeerID;
@@ -1,7 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.isPeerID = void 0;
4
- function isPeerID(raw) {
5
- return typeof raw === 'number';
6
- }
7
- exports.isPeerID = isPeerID;
@@ -1,37 +0,0 @@
1
- import { Peer } from './decentralized/peer';
2
- import { PeerID } from './decentralized/types';
3
- import { type, NarrowMessage, Message } from './messages';
4
- import { SignalData } from 'simple-peer';
5
- export interface EventConnection {
6
- on: <K extends type>(type: K, handler: (event: NarrowMessage<K>) => void) => void;
7
- once: <K extends type>(type: K, handler: (event: NarrowMessage<K>) => void) => void;
8
- send: <T extends Message>(msg: T) => void;
9
- disconnect: () => void;
10
- }
11
- export declare function waitMessage<T extends type>(connection: EventConnection, type: T): Promise<NarrowMessage<T>>;
12
- export declare function waitMessageWithTimeout<T extends type>(connection: EventConnection, type: T, timeoutMs: number): Promise<NarrowMessage<T>>;
13
- export declare class PeerConnection implements EventConnection {
14
- private readonly selfId;
15
- private readonly peer;
16
- private readonly signallingServer;
17
- private readonly eventEmitter;
18
- constructor(selfId: PeerID, peer: Peer, signallingServer: EventConnection);
19
- connect(): Promise<void>;
20
- signal(signal: SignalData): void;
21
- on<K extends type>(type: K, handler: (event: NarrowMessage<K>) => void): void;
22
- once<K extends type>(type: K, handler: (event: NarrowMessage<K>) => void): void;
23
- send<T extends Message>(msg: T): void;
24
- disconnect(): void;
25
- }
26
- export declare class WebSocketServer implements EventConnection {
27
- private readonly socket;
28
- private readonly eventEmitter;
29
- private readonly validateReceived?;
30
- private readonly validateSent?;
31
- private constructor();
32
- static connect(url: URL, validateReceived?: (msg: any) => boolean, validateSent?: (msg: any) => boolean): Promise<WebSocketServer>;
33
- disconnect(): void;
34
- on<K extends type>(type: K, handler: (event: NarrowMessage<K>) => void): void;
35
- once<K extends type>(type: K, handler: (event: NarrowMessage<K>) => void): void;
36
- send(msg: Message): void;
37
- }
@@ -1,158 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.WebSocketServer = exports.PeerConnection = exports.waitMessageWithTimeout = exports.waitMessage = void 0;
4
- var tslib_1 = require("tslib");
5
- var isomorphic_ws_1 = (0, tslib_1.__importDefault)(require("isomorphic-ws"));
6
- var events_1 = require("events");
7
- var msgpack_lite_1 = (0, tslib_1.__importDefault)(require("msgpack-lite"));
8
- var decentralizedMessages = (0, tslib_1.__importStar)(require("./decentralized/messages"));
9
- var messages_1 = require("./messages");
10
- var utils_1 = require("./utils");
11
- function waitMessage(connection, type) {
12
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
13
- return (0, tslib_1.__generator)(this, function (_a) {
14
- switch (_a.label) {
15
- case 0: return [4 /*yield*/, new Promise(function (resolve) {
16
- // "once" is important because we can't resolve the same promise multiple time
17
- connection.once(type, function (event) {
18
- resolve(event);
19
- });
20
- })];
21
- case 1: return [2 /*return*/, _a.sent()];
22
- }
23
- });
24
- });
25
- }
26
- exports.waitMessage = waitMessage;
27
- function waitMessageWithTimeout(connection, type, timeoutMs) {
28
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
29
- return (0, tslib_1.__generator)(this, function (_a) {
30
- switch (_a.label) {
31
- case 0: return [4 /*yield*/, Promise.race([waitMessage(connection, type), (0, utils_1.timeout)(timeoutMs)])];
32
- case 1: return [2 /*return*/, _a.sent()];
33
- }
34
- });
35
- });
36
- }
37
- exports.waitMessageWithTimeout = waitMessageWithTimeout;
38
- var PeerConnection = /** @class */ (function () {
39
- function PeerConnection(selfId, peer, signallingServer) {
40
- this.eventEmitter = new events_1.EventEmitter();
41
- this.selfId = selfId;
42
- this.peer = peer;
43
- this.signallingServer = signallingServer;
44
- }
45
- PeerConnection.prototype.connect = function () {
46
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
47
- var _this = this;
48
- return (0, tslib_1.__generator)(this, function (_a) {
49
- switch (_a.label) {
50
- case 0:
51
- this.peer.on('signal', function (signal) {
52
- console.debug(_this.selfId, 'generates signal for', _this.peer.id);
53
- var msg = {
54
- type: messages_1.type.SignalForPeer,
55
- peer: _this.peer.id,
56
- signal: signal
57
- };
58
- _this.signallingServer.send(msg);
59
- });
60
- this.peer.on('data', function (data) {
61
- var msg = msgpack_lite_1.default.decode(data);
62
- if (!decentralizedMessages.isPeerMessage(msg)) {
63
- throw new Error("invalid message received: " + JSON.stringify(msg));
64
- }
65
- _this.eventEmitter.emit(msg.type.toString(), msg);
66
- });
67
- this.peer.on('close', function () { return console.warn('peer', _this.peer.id, 'closed connection'); });
68
- return [4 /*yield*/, new Promise(function (resolve) {
69
- _this.peer.on('connect', function () {
70
- console.debug('connected new peer', _this.peer.id);
71
- resolve();
72
- });
73
- })];
74
- case 1: return [2 /*return*/, _a.sent()];
75
- }
76
- });
77
- });
78
- };
79
- PeerConnection.prototype.signal = function (signal) {
80
- this.peer.signal(signal);
81
- };
82
- PeerConnection.prototype.on = function (type, handler) {
83
- this.eventEmitter.on(type.toString(), handler);
84
- };
85
- PeerConnection.prototype.once = function (type, handler) {
86
- this.eventEmitter.once(type.toString(), handler);
87
- };
88
- PeerConnection.prototype.send = function (msg) {
89
- if (!decentralizedMessages.isPeerMessage(msg)) {
90
- throw new Error("can't send this type of message: " + JSON.stringify(msg));
91
- }
92
- this.peer.send(msgpack_lite_1.default.encode(msg));
93
- };
94
- PeerConnection.prototype.disconnect = function () {
95
- this.peer.destroy();
96
- };
97
- return PeerConnection;
98
- }());
99
- exports.PeerConnection = PeerConnection;
100
- var WebSocketServer = /** @class */ (function () {
101
- function WebSocketServer(socket, eventEmitter, validateReceived, validateSent) {
102
- this.socket = socket;
103
- this.eventEmitter = eventEmitter;
104
- this.validateReceived = validateReceived;
105
- this.validateSent = validateSent;
106
- }
107
- WebSocketServer.connect = function (url, validateReceived, validateSent) {
108
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
109
- var WS, ws, emitter, server;
110
- return (0, tslib_1.__generator)(this, function (_a) {
111
- switch (_a.label) {
112
- case 0:
113
- WS = typeof window !== 'undefined' ? window.WebSocket : isomorphic_ws_1.default.WebSocket;
114
- ws = new WS(url);
115
- ws.binaryType = 'arraybuffer';
116
- emitter = new events_1.EventEmitter();
117
- server = new WebSocketServer(ws, emitter, validateReceived, validateSent);
118
- ws.onmessage = function (event) {
119
- if (!(event.data instanceof ArrayBuffer)) {
120
- throw new Error('server did not send an ArrayBuffer');
121
- }
122
- var msg = msgpack_lite_1.default.decode(new Uint8Array(event.data));
123
- // Validate message format
124
- if (validateReceived && !validateReceived(msg)) {
125
- throw new Error("invalid message received: " + JSON.stringify(msg));
126
- }
127
- emitter.emit(msg.type.toString(), msg);
128
- };
129
- return [4 /*yield*/, new Promise(function (resolve, reject) {
130
- ws.onerror = function (err) {
131
- return reject(new Error("connecting server: " + err.message));
132
- }; // eslint-disable-line @typescript-eslint/restrict-template-expressions
133
- ws.onopen = function () { return resolve(server); };
134
- })];
135
- case 1: return [2 /*return*/, _a.sent()];
136
- }
137
- });
138
- });
139
- };
140
- WebSocketServer.prototype.disconnect = function () {
141
- this.socket.close();
142
- };
143
- // Not straigtforward way of making sure the handler take the correct message type as a parameter, for typesafety
144
- WebSocketServer.prototype.on = function (type, handler) {
145
- this.eventEmitter.on(type.toString(), handler);
146
- };
147
- WebSocketServer.prototype.once = function (type, handler) {
148
- this.eventEmitter.once(type.toString(), handler);
149
- };
150
- WebSocketServer.prototype.send = function (msg) {
151
- if (this.validateSent && !this.validateSent(msg)) {
152
- throw new Error("can't send this type of message: " + JSON.stringify(msg));
153
- }
154
- this.socket.send(msgpack_lite_1.default.encode(msg));
155
- };
156
- return WebSocketServer;
157
- }());
158
- exports.WebSocketServer = WebSocketServer;
@@ -1,37 +0,0 @@
1
- import { informant, MetadataID, WeightsContainer } from '../..';
2
- import { Base } from '../base';
3
- import { EventConnection } from '../event_connection';
4
- /**
5
- * Class that deals with communication with the centralized server when training
6
- * a specific task in the federated setting.
7
- */
8
- export declare class Client extends Base {
9
- private readonly clientID;
10
- private readonly peer;
11
- private round;
12
- protected _server?: EventConnection;
13
- private serverRound?;
14
- private serverWeights?;
15
- private receivedStatistics?;
16
- private metadataMap?;
17
- get server(): EventConnection;
18
- private connectServer;
19
- /**
20
- * Initialize the connection to the server. TODO: In the case of FeAI,
21
- * should return the current server-side round for the task.
22
- */
23
- connect(): Promise<void>;
24
- /**
25
- * Disconnection process when user quits the task.
26
- */
27
- disconnect(): Promise<void>;
28
- private sendMessage;
29
- postWeightsToServer(weights: WeightsContainer): Promise<void>;
30
- getLatestServerRound(): Promise<number | undefined>;
31
- pullRoundAndFetchWeights(): Promise<WeightsContainer | undefined>;
32
- pullServerStatistics(trainingInformant: informant.FederatedInformant): Promise<void>;
33
- postMetadata(metadataID: MetadataID, metadata: string): Promise<void>;
34
- getMetadataMap(metadataId: MetadataID): Promise<Map<string, unknown> | undefined>;
35
- onRoundEndCommunication(updatedWeights: WeightsContainer, staleWeights: WeightsContainer, _: number, trainingInformant: informant.FederatedInformant): Promise<WeightsContainer>;
36
- onTrainEndCommunication(): Promise<void>;
37
- }