@epfml/discojs 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/README.md +28 -8
  2. package/dist/{async_buffer.d.ts → core/async_buffer.d.ts} +3 -3
  3. package/dist/{async_buffer.js → core/async_buffer.js} +5 -6
  4. package/dist/{async_informant.d.ts → core/async_informant.d.ts} +0 -0
  5. package/dist/{async_informant.js → core/async_informant.js} +0 -0
  6. package/dist/{client → core/client}/base.d.ts +4 -7
  7. package/dist/{client → core/client}/base.js +3 -2
  8. package/dist/core/client/decentralized/base.d.ts +32 -0
  9. package/dist/core/client/decentralized/base.js +212 -0
  10. package/dist/core/client/decentralized/clear_text.d.ts +14 -0
  11. package/dist/core/client/decentralized/clear_text.js +96 -0
  12. package/dist/{client → core/client}/decentralized/index.d.ts +0 -0
  13. package/dist/{client → core/client}/decentralized/index.js +0 -0
  14. package/dist/core/client/decentralized/messages.d.ts +41 -0
  15. package/dist/core/client/decentralized/messages.js +54 -0
  16. package/dist/core/client/decentralized/peer.d.ts +26 -0
  17. package/dist/core/client/decentralized/peer.js +210 -0
  18. package/dist/core/client/decentralized/peer_pool.d.ts +14 -0
  19. package/dist/core/client/decentralized/peer_pool.js +92 -0
  20. package/dist/core/client/decentralized/sec_agg.d.ts +22 -0
  21. package/dist/core/client/decentralized/sec_agg.js +190 -0
  22. package/dist/core/client/decentralized/secret_shares.d.ts +3 -0
  23. package/dist/core/client/decentralized/secret_shares.js +39 -0
  24. package/dist/core/client/decentralized/types.d.ts +2 -0
  25. package/dist/core/client/decentralized/types.js +7 -0
  26. package/dist/core/client/event_connection.d.ts +37 -0
  27. package/dist/core/client/event_connection.js +158 -0
  28. package/dist/core/client/federated/client.d.ts +37 -0
  29. package/dist/core/client/federated/client.js +273 -0
  30. package/dist/core/client/federated/index.d.ts +2 -0
  31. package/dist/core/client/federated/index.js +7 -0
  32. package/dist/core/client/federated/messages.d.ts +38 -0
  33. package/dist/core/client/federated/messages.js +25 -0
  34. package/dist/{client → core/client}/index.d.ts +2 -1
  35. package/dist/{client → core/client}/index.js +3 -3
  36. package/dist/{client → core/client}/local.d.ts +2 -2
  37. package/dist/{client → core/client}/local.js +0 -0
  38. package/dist/core/client/messages.d.ts +28 -0
  39. package/dist/core/client/messages.js +33 -0
  40. package/dist/core/client/utils.d.ts +2 -0
  41. package/dist/core/client/utils.js +19 -0
  42. package/dist/core/dataset/data/data.d.ts +11 -0
  43. package/dist/core/dataset/data/data.js +20 -0
  44. package/dist/core/dataset/data/data_split.d.ts +5 -0
  45. package/dist/{client/decentralized/types.js → core/dataset/data/data_split.js} +0 -0
  46. package/dist/core/dataset/data/image_data.d.ts +8 -0
  47. package/dist/core/dataset/data/image_data.js +64 -0
  48. package/dist/core/dataset/data/index.d.ts +5 -0
  49. package/dist/core/dataset/data/index.js +11 -0
  50. package/dist/core/dataset/data/preprocessing.d.ts +13 -0
  51. package/dist/core/dataset/data/preprocessing.js +33 -0
  52. package/dist/core/dataset/data/tabular_data.d.ts +8 -0
  53. package/dist/core/dataset/data/tabular_data.js +40 -0
  54. package/dist/{dataset → core/dataset}/data_loader/data_loader.d.ts +4 -11
  55. package/dist/{dataset → core/dataset}/data_loader/data_loader.js +0 -0
  56. package/dist/core/dataset/data_loader/image_loader.d.ts +17 -0
  57. package/dist/core/dataset/data_loader/image_loader.js +141 -0
  58. package/dist/core/dataset/data_loader/index.d.ts +3 -0
  59. package/dist/core/dataset/data_loader/index.js +9 -0
  60. package/dist/core/dataset/data_loader/tabular_loader.d.ts +29 -0
  61. package/dist/core/dataset/data_loader/tabular_loader.js +101 -0
  62. package/dist/core/dataset/dataset.d.ts +2 -0
  63. package/dist/{task/training_information.js → core/dataset/dataset.js} +0 -0
  64. package/dist/{dataset → core/dataset}/dataset_builder.d.ts +5 -5
  65. package/dist/{dataset → core/dataset}/dataset_builder.js +14 -10
  66. package/dist/core/dataset/index.d.ts +4 -0
  67. package/dist/core/dataset/index.js +14 -0
  68. package/dist/core/index.d.ts +18 -0
  69. package/dist/core/index.js +41 -0
  70. package/dist/{informant → core/informant}/graph_informant.d.ts +0 -0
  71. package/dist/{informant → core/informant}/graph_informant.js +0 -0
  72. package/dist/{informant → core/informant}/index.d.ts +0 -0
  73. package/dist/{informant → core/informant}/index.js +0 -0
  74. package/dist/{informant → core/informant}/training_informant/base.d.ts +3 -3
  75. package/dist/{informant → core/informant}/training_informant/base.js +3 -2
  76. package/dist/{informant → core/informant}/training_informant/decentralized.d.ts +0 -0
  77. package/dist/{informant → core/informant}/training_informant/decentralized.js +0 -0
  78. package/dist/{informant → core/informant}/training_informant/federated.d.ts +0 -0
  79. package/dist/{informant → core/informant}/training_informant/federated.js +0 -0
  80. package/dist/{informant → core/informant}/training_informant/index.d.ts +0 -0
  81. package/dist/{informant → core/informant}/training_informant/index.js +0 -0
  82. package/dist/{informant → core/informant}/training_informant/local.d.ts +2 -2
  83. package/dist/{informant → core/informant}/training_informant/local.js +2 -2
  84. package/dist/{logging → core/logging}/console_logger.d.ts +0 -0
  85. package/dist/{logging → core/logging}/console_logger.js +0 -0
  86. package/dist/{logging → core/logging}/index.d.ts +0 -0
  87. package/dist/{logging → core/logging}/index.js +0 -0
  88. package/dist/{logging → core/logging}/logger.d.ts +0 -0
  89. package/dist/{logging → core/logging}/logger.js +0 -0
  90. package/dist/{logging → core/logging}/trainer_logger.d.ts +0 -0
  91. package/dist/{logging → core/logging}/trainer_logger.js +0 -0
  92. package/dist/{memory → core/memory}/base.d.ts +2 -2
  93. package/dist/{memory → core/memory}/base.js +0 -0
  94. package/dist/{memory → core/memory}/empty.d.ts +0 -0
  95. package/dist/{memory → core/memory}/empty.js +0 -0
  96. package/dist/core/memory/index.d.ts +3 -0
  97. package/dist/core/memory/index.js +9 -0
  98. package/dist/{memory → core/memory}/model_type.d.ts +0 -0
  99. package/dist/{memory → core/memory}/model_type.js +0 -0
  100. package/dist/{privacy.d.ts → core/privacy.d.ts} +2 -3
  101. package/dist/{privacy.js → core/privacy.js} +3 -16
  102. package/dist/{serialization → core/serialization}/index.d.ts +0 -0
  103. package/dist/{serialization → core/serialization}/index.js +0 -0
  104. package/dist/{serialization → core/serialization}/model.d.ts +0 -0
  105. package/dist/{serialization → core/serialization}/model.js +0 -0
  106. package/dist/core/serialization/weights.d.ts +5 -0
  107. package/dist/{serialization → core/serialization}/weights.js +11 -9
  108. package/dist/{task → core/task}/data_example.d.ts +0 -0
  109. package/dist/{task → core/task}/data_example.js +0 -0
  110. package/dist/{task → core/task}/display_information.d.ts +5 -5
  111. package/dist/{task → core/task}/display_information.js +5 -10
  112. package/dist/{task → core/task}/index.d.ts +0 -0
  113. package/dist/{task → core/task}/index.js +0 -0
  114. package/dist/core/task/model_compile_data.d.ts +6 -0
  115. package/dist/core/task/model_compile_data.js +22 -0
  116. package/dist/{task → core/task}/summary.d.ts +0 -0
  117. package/dist/{task → core/task}/summary.js +0 -4
  118. package/dist/{task → core/task}/task.d.ts +2 -2
  119. package/dist/{task → core/task}/task.js +6 -7
  120. package/dist/{task → core/task}/training_information.d.ts +10 -14
  121. package/dist/core/task/training_information.js +66 -0
  122. package/dist/{tasks → core/tasks}/cifar10.d.ts +1 -2
  123. package/dist/{tasks → core/tasks}/cifar10.js +12 -23
  124. package/dist/core/tasks/geotags.d.ts +3 -0
  125. package/dist/core/tasks/geotags.js +67 -0
  126. package/dist/{tasks → core/tasks}/index.d.ts +2 -1
  127. package/dist/{tasks → core/tasks}/index.js +3 -2
  128. package/dist/core/tasks/lus_covid.d.ts +3 -0
  129. package/dist/{tasks → core/tasks}/lus_covid.js +26 -24
  130. package/dist/{tasks → core/tasks}/mnist.d.ts +1 -2
  131. package/dist/{tasks → core/tasks}/mnist.js +18 -16
  132. package/dist/core/tasks/simple_face.d.ts +2 -0
  133. package/dist/core/tasks/simple_face.js +41 -0
  134. package/dist/{tasks → core/tasks}/titanic.d.ts +1 -2
  135. package/dist/{tasks → core/tasks}/titanic.js +11 -11
  136. package/dist/core/training/disco.d.ts +23 -0
  137. package/dist/core/training/disco.js +130 -0
  138. package/dist/{training → core/training}/index.d.ts +0 -0
  139. package/dist/{training → core/training}/index.js +0 -0
  140. package/dist/{training → core/training}/trainer/distributed_trainer.d.ts +1 -2
  141. package/dist/{training → core/training}/trainer/distributed_trainer.js +6 -5
  142. package/dist/{training → core/training}/trainer/local_trainer.d.ts +2 -2
  143. package/dist/{training → core/training}/trainer/local_trainer.js +0 -0
  144. package/dist/{training → core/training}/trainer/round_tracker.d.ts +0 -0
  145. package/dist/{training → core/training}/trainer/round_tracker.js +0 -0
  146. package/dist/{training → core/training}/trainer/trainer.d.ts +1 -2
  147. package/dist/{training → core/training}/trainer/trainer.js +2 -2
  148. package/dist/{training → core/training}/trainer/trainer_builder.d.ts +0 -0
  149. package/dist/{training → core/training}/trainer/trainer_builder.js +0 -0
  150. package/dist/core/training/training_schemes.d.ts +5 -0
  151. package/dist/{training → core/training}/training_schemes.js +2 -2
  152. package/dist/{types.d.ts → core/types.d.ts} +0 -0
  153. package/dist/{types.js → core/types.js} +0 -0
  154. package/dist/{validation → core/validation}/index.d.ts +0 -0
  155. package/dist/{validation → core/validation}/index.js +0 -0
  156. package/dist/{validation → core/validation}/validator.d.ts +5 -8
  157. package/dist/{validation → core/validation}/validator.js +9 -11
  158. package/dist/core/weights/aggregation.d.ts +8 -0
  159. package/dist/core/weights/aggregation.js +96 -0
  160. package/dist/core/weights/index.d.ts +2 -0
  161. package/dist/core/weights/index.js +7 -0
  162. package/dist/core/weights/weights_container.d.ts +19 -0
  163. package/dist/core/weights/weights_container.js +64 -0
  164. package/dist/dataset/data_loader/image_loader.d.ts +3 -15
  165. package/dist/dataset/data_loader/image_loader.js +12 -125
  166. package/dist/dataset/data_loader/index.d.ts +2 -3
  167. package/dist/dataset/data_loader/index.js +3 -5
  168. package/dist/dataset/data_loader/tabular_loader.d.ts +3 -28
  169. package/dist/dataset/data_loader/tabular_loader.js +11 -92
  170. package/dist/imports.d.ts +2 -0
  171. package/dist/imports.js +7 -0
  172. package/dist/index.d.ts +2 -19
  173. package/dist/index.js +3 -39
  174. package/dist/memory/index.d.ts +1 -3
  175. package/dist/memory/index.js +3 -7
  176. package/dist/memory/memory.d.ts +26 -0
  177. package/dist/memory/memory.js +160 -0
  178. package/package.json +14 -27
  179. package/dist/aggregation.d.ts +0 -5
  180. package/dist/aggregation.js +0 -33
  181. package/dist/client/decentralized/base.d.ts +0 -43
  182. package/dist/client/decentralized/base.js +0 -243
  183. package/dist/client/decentralized/clear_text.d.ts +0 -13
  184. package/dist/client/decentralized/clear_text.js +0 -78
  185. package/dist/client/decentralized/messages.d.ts +0 -37
  186. package/dist/client/decentralized/messages.js +0 -15
  187. package/dist/client/decentralized/sec_agg.d.ts +0 -18
  188. package/dist/client/decentralized/sec_agg.js +0 -169
  189. package/dist/client/decentralized/secret_shares.d.ts +0 -5
  190. package/dist/client/decentralized/secret_shares.js +0 -58
  191. package/dist/client/decentralized/types.d.ts +0 -1
  192. package/dist/client/federated.d.ts +0 -30
  193. package/dist/client/federated.js +0 -218
  194. package/dist/dataset/index.d.ts +0 -2
  195. package/dist/dataset/index.js +0 -7
  196. package/dist/model_actor.d.ts +0 -16
  197. package/dist/model_actor.js +0 -20
  198. package/dist/serialization/weights.d.ts +0 -5
  199. package/dist/task/model_compile_data.d.ts +0 -6
  200. package/dist/task/model_compile_data.js +0 -12
  201. package/dist/tasks/lus_covid.d.ts +0 -4
  202. package/dist/tasks/simple_face.d.ts +0 -4
  203. package/dist/tasks/simple_face.js +0 -84
  204. package/dist/tfjs.d.ts +0 -2
  205. package/dist/tfjs.js +0 -6
  206. package/dist/training/disco.d.ts +0 -14
  207. package/dist/training/disco.js +0 -70
  208. package/dist/training/training_schemes.d.ts +0 -5
@@ -1,139 +1,26 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ImageLoader = void 0;
3
+ exports.WebImageLoader = void 0;
4
4
  var tslib_1 = require("tslib");
5
- var immutable_1 = require("immutable");
6
5
  var __1 = require("../..");
7
- var data_loader_1 = require("./data_loader");
8
- /**
9
- * TODO @s314cy:
10
- * Load labels and correctly match them with their respective images, with the following constraints:
11
- * 1. Images are given as 1 image/1 file
12
- * 2. Labels are given as multiple labels/1 file, each label file can contain a different amount of labels
13
- */
14
- var ImageLoader = /** @class */ (function (_super) {
15
- (0, tslib_1.__extends)(ImageLoader, _super);
16
- function ImageLoader() {
6
+ var WebImageLoader = /** @class */ (function (_super) {
7
+ (0, tslib_1.__extends)(WebImageLoader, _super);
8
+ function WebImageLoader() {
17
9
  return _super !== null && _super.apply(this, arguments) || this;
18
10
  }
19
- ImageLoader.prototype.load = function (image, config) {
11
+ WebImageLoader.prototype.readImageFrom = function (source) {
20
12
  return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
21
- var tensorContainer;
22
- var _a;
23
- return (0, tslib_1.__generator)(this, function (_b) {
24
- switch (_b.label) {
25
- case 0:
26
- if (!(config === undefined || config.labels === undefined)) return [3 /*break*/, 2];
27
- return [4 /*yield*/, this.readImageFrom(image)];
28
- case 1:
29
- tensorContainer = _b.sent();
30
- return [3 /*break*/, 4];
31
- case 2:
32
- _a = {};
33
- return [4 /*yield*/, this.readImageFrom(image)];
34
- case 3:
35
- tensorContainer = (_a.xs = _b.sent(),
36
- _a.ys = config.labels[0],
37
- _a);
38
- _b.label = 4;
39
- case 4: return [2 /*return*/, __1.tf.data.array([tensorContainer])];
40
- }
41
- });
42
- });
43
- };
44
- ImageLoader.prototype.buildDataset = function (images, labels, indices, config) {
45
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
46
- var dataset;
47
- var _this = this;
48
- return (0, tslib_1.__generator)(this, function (_a) {
49
- dataset = __1.tf.data.generator(function () {
50
- var withLabels = (config === null || config === void 0 ? void 0 : config.labels) !== undefined;
51
- var index = 0;
52
- var iterator = {
53
- next: function () { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () {
54
- var sample, label, value;
55
- return (0, tslib_1.__generator)(this, function (_a) {
56
- switch (_a.label) {
57
- case 0:
58
- if (index === indices.length) {
59
- return [2 /*return*/, { done: true }];
60
- }
61
- return [4 /*yield*/, this.readImageFrom(images[indices[index]])];
62
- case 1:
63
- sample = _a.sent();
64
- label = withLabels ? labels[indices[index]] : undefined;
65
- value = withLabels ? { xs: sample, ys: label } : sample;
66
- index++;
67
- return [2 /*return*/, {
68
- value: value,
69
- done: false
70
- }];
71
- }
72
- });
73
- }); }
74
- };
75
- return iterator; // Lazy
76
- });
77
- return [2 /*return*/, {
78
- dataset: dataset,
79
- size: indices.length
80
- }];
81
- });
82
- });
83
- };
84
- ImageLoader.prototype.loadAll = function (images, config) {
85
- var _a, _b;
86
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
87
- var labels, indices, numberOfClasses, dataset, trainSize, trainIndices, valIndices, trainDataset, valDataset;
13
+ var _a, _b;
88
14
  return (0, tslib_1.__generator)(this, function (_c) {
89
15
  switch (_c.label) {
90
16
  case 0:
91
- labels = [];
92
- indices = (0, immutable_1.Range)(0, images.length).toArray();
93
- if ((config === null || config === void 0 ? void 0 : config.labels) !== undefined) {
94
- numberOfClasses = (_b = (_a = this.task.trainingInformation) === null || _a === void 0 ? void 0 : _a.LABEL_LIST) === null || _b === void 0 ? void 0 : _b.length;
95
- if (numberOfClasses === undefined) {
96
- throw new Error('wanted labels but none found in task');
97
- }
98
- labels = __1.tf.oneHot(__1.tf.tensor1d(config.labels, 'int32'), numberOfClasses).arraySync();
99
- }
100
- if ((config === null || config === void 0 ? void 0 : config.shuffle) === undefined || (config === null || config === void 0 ? void 0 : config.shuffle)) {
101
- this.shuffle(indices);
102
- }
103
- if (!((config === null || config === void 0 ? void 0 : config.validationSplit) === undefined)) return [3 /*break*/, 2];
104
- return [4 /*yield*/, this.buildDataset(images, labels, indices, config)];
105
- case 1:
106
- dataset = _c.sent();
107
- return [2 /*return*/, {
108
- train: dataset,
109
- validation: dataset
110
- }];
111
- case 2:
112
- trainSize = Math.floor(images.length * (1 - config.validationSplit));
113
- trainIndices = indices.slice(0, trainSize);
114
- valIndices = indices.slice(trainSize);
115
- return [4 /*yield*/, this.buildDataset(images, labels, trainIndices, config)];
116
- case 3:
117
- trainDataset = _c.sent();
118
- return [4 /*yield*/, this.buildDataset(images, labels, valIndices, config)];
119
- case 4:
120
- valDataset = _c.sent();
121
- return [2 /*return*/, {
122
- train: trainDataset,
123
- validation: valDataset
124
- }];
17
+ _b = (_a = __1.tf.browser).fromPixels;
18
+ return [4 /*yield*/, createImageBitmap(source)];
19
+ case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])];
125
20
  }
126
21
  });
127
22
  });
128
23
  };
129
- ImageLoader.prototype.shuffle = function (array) {
130
- for (var i = 0; i < array.length; i++) {
131
- var j = Math.floor(Math.random() * i);
132
- var swap = array[i];
133
- array[i] = array[j];
134
- array[j] = swap;
135
- }
136
- };
137
- return ImageLoader;
138
- }(data_loader_1.DataLoader));
139
- exports.ImageLoader = ImageLoader;
24
+ return WebImageLoader;
25
+ }(__1.data.ImageLoader));
26
+ exports.WebImageLoader = WebImageLoader;
@@ -1,3 +1,2 @@
1
- export { Data, DataTuple, DataConfig, DataLoader } from './data_loader';
2
- export { ImageLoader } from './image_loader';
3
- export { TabularLoader } from './tabular_loader';
1
+ export { WebImageLoader } from './image_loader';
2
+ export { WebTabularLoader } from './tabular_loader';
@@ -1,9 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.TabularLoader = exports.ImageLoader = exports.DataLoader = void 0;
4
- var data_loader_1 = require("./data_loader");
5
- Object.defineProperty(exports, "DataLoader", { enumerable: true, get: function () { return data_loader_1.DataLoader; } });
3
+ exports.WebTabularLoader = exports.WebImageLoader = void 0;
6
4
  var image_loader_1 = require("./image_loader");
7
- Object.defineProperty(exports, "ImageLoader", { enumerable: true, get: function () { return image_loader_1.ImageLoader; } });
5
+ Object.defineProperty(exports, "WebImageLoader", { enumerable: true, get: function () { return image_loader_1.WebImageLoader; } });
8
6
  var tabular_loader_1 = require("./tabular_loader");
9
- Object.defineProperty(exports, "TabularLoader", { enumerable: true, get: function () { return tabular_loader_1.TabularLoader; } });
7
+ Object.defineProperty(exports, "WebTabularLoader", { enumerable: true, get: function () { return tabular_loader_1.WebTabularLoader; } });
@@ -1,29 +1,4 @@
1
- import { DataLoader, DataConfig, DataTuple } from './data_loader';
2
- import { Dataset } from '../dataset_builder';
3
- import { Task } from '../../task';
4
- import { tf } from '../..';
5
- export declare abstract class TabularLoader<Source> extends DataLoader<Source> {
6
- private readonly delimiter;
7
- constructor(task: Task, delimiter: string);
8
- /**
9
- * Creates a CSV dataset object based off the given source.
10
- * @param source File object, URL string or local file system path.
11
- * @param csvConfig Object expected by TF.js to create a CSVDataset.
12
- * @returns The CSVDataset object built upon the given source.
13
- */
14
- abstract loadTabularDatasetFrom(source: Source, csvConfig: Record<string, unknown>): tf.data.CSVDataset;
15
- /**
16
- * Expects delimiter-separated tabular data made of N columns. The data may be
17
- * potentially split among several sources. Every source should contain N-1
18
- * feature columns and 1 single label column.
19
- * @param source List of File objects, URLs or file system paths.
20
- * @param config
21
- * @returns A TF.js dataset built upon read tabular data stored in the given sources.
22
- */
23
- load(source: Source, config?: DataConfig): Promise<Dataset>;
24
- /**
25
- * Creates the CSV datasets based off the given sources, then fuses them into a single CSV
26
- * dataset.
27
- */
28
- loadAll(sources: Source[], config: DataConfig): Promise<DataTuple>;
1
+ import { tf, data } from '../..';
2
+ export declare class WebTabularLoader extends data.TabularLoader<File> {
3
+ loadTabularDatasetFrom(source: File, csvConfig: Record<string, unknown>): tf.data.CSVDataset;
29
4
  }
@@ -1,97 +1,16 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.TabularLoader = void 0;
3
+ exports.WebTabularLoader = void 0;
4
4
  var tslib_1 = require("tslib");
5
- var data_loader_1 = require("./data_loader");
6
- var immutable_1 = require("immutable");
7
- // window size from which the dataset shuffling will sample
8
- var BUFFER_SIZE = 1000;
9
- var TabularLoader = /** @class */ (function (_super) {
10
- (0, tslib_1.__extends)(TabularLoader, _super);
11
- function TabularLoader(task, delimiter) {
12
- var _this = _super.call(this, task) || this;
13
- _this.delimiter = delimiter;
14
- return _this;
5
+ var __1 = require("../..");
6
+ var WebTabularLoader = /** @class */ (function (_super) {
7
+ (0, tslib_1.__extends)(WebTabularLoader, _super);
8
+ function WebTabularLoader() {
9
+ return _super !== null && _super.apply(this, arguments) || this;
15
10
  }
16
- /**
17
- * Expects delimiter-separated tabular data made of N columns. The data may be
18
- * potentially split among several sources. Every source should contain N-1
19
- * feature columns and 1 single label column.
20
- * @param source List of File objects, URLs or file system paths.
21
- * @param config
22
- * @returns A TF.js dataset built upon read tabular data stored in the given sources.
23
- */
24
- TabularLoader.prototype.load = function (source, config) {
25
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
26
- var columnConfigs, csvConfig, dataset;
27
- return (0, tslib_1.__generator)(this, function (_a) {
28
- /**
29
- * Prepare the CSV config object based off the given features and labels.
30
- * If labels is empty, then the returned dataset is comprised of samples only.
31
- * Otherwise, each entry is of the form `{ xs, ys }` with `xs` as features and `ys`
32
- * as labels.
33
- */
34
- if ((config === null || config === void 0 ? void 0 : config.features) === undefined) {
35
- // TODO @s314cy
36
- throw new Error('not implemented');
37
- }
38
- columnConfigs = (0, immutable_1.Map)((0, immutable_1.Set)(config.features).map(function (feature) { return [feature, { required: false, isLabel: false }]; })).merge((0, immutable_1.Set)(config.labels).map(function (label) { return [label, { required: true, isLabel: true }]; }));
39
- csvConfig = {
40
- hasHeader: true,
41
- columnConfigs: columnConfigs.toObject(),
42
- configuredColumnsOnly: true,
43
- delimiter: this.delimiter
44
- };
45
- dataset = this.loadTabularDatasetFrom(source, csvConfig).map(function (t) {
46
- if (typeof t === 'object' && ('xs' in t) && ('ys' in t)) {
47
- return t;
48
- }
49
- throw new Error('expected TensorContainerObject');
50
- }).map(function (t) {
51
- // TODO order may not be stable between tensor
52
- var _a = t, xs = _a.xs, ys = _a.ys;
53
- return {
54
- xs: Object.values(xs),
55
- ys: Object.values(ys)
56
- };
57
- });
58
- return [2 /*return*/, ((config === null || config === void 0 ? void 0 : config.shuffle) === undefined || (config === null || config === void 0 ? void 0 : config.shuffle)) ? dataset.shuffle(BUFFER_SIZE) : dataset];
59
- });
60
- });
11
+ WebTabularLoader.prototype.loadTabularDatasetFrom = function (source, csvConfig) {
12
+ return new __1.tf.data.CSVDataset(new __1.tf.data.FileDataSource(source), csvConfig);
61
13
  };
62
- /**
63
- * Creates the CSV datasets based off the given sources, then fuses them into a single CSV
64
- * dataset.
65
- */
66
- TabularLoader.prototype.loadAll = function (sources, config) {
67
- return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
68
- var datasets, dataset, data;
69
- var _this = this;
70
- return (0, tslib_1.__generator)(this, function (_a) {
71
- switch (_a.label) {
72
- case 0: return [4 /*yield*/, Promise.all(sources.map(function (source) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () { return (0, tslib_1.__generator)(this, function (_a) {
73
- switch (_a.label) {
74
- case 0: return [4 /*yield*/, this.load(source, (0, tslib_1.__assign)((0, tslib_1.__assign)({}, config), { shuffle: false }))];
75
- case 1: return [2 /*return*/, _a.sent()];
76
- }
77
- }); }); }))];
78
- case 1:
79
- datasets = _a.sent();
80
- dataset = (0, immutable_1.List)(datasets).reduce(function (acc, dataset) { return acc.concatenate(dataset); });
81
- data = {
82
- dataset: (config === null || config === void 0 ? void 0 : config.shuffle) ? dataset.shuffle(BUFFER_SIZE) : dataset,
83
- // dataset.size does not work for csv datasets
84
- // https://github.com/tensorflow/tfjs/issues/5845
85
- size: 0
86
- };
87
- // TODO: Implement validation split for tabular data (tricky due to streaming)
88
- return [2 /*return*/, {
89
- train: data
90
- }];
91
- }
92
- });
93
- });
94
- };
95
- return TabularLoader;
96
- }(data_loader_1.DataLoader));
97
- exports.TabularLoader = TabularLoader;
14
+ return WebTabularLoader;
15
+ }(__1.data.TabularLoader));
16
+ exports.WebTabularLoader = WebTabularLoader;
@@ -0,0 +1,2 @@
1
+ export * as data from './dataset/data_loader';
2
+ export { IndexedDB } from './memory';
@@ -0,0 +1,7 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.IndexedDB = exports.data = void 0;
4
+ var tslib_1 = require("tslib");
5
+ exports.data = (0, tslib_1.__importStar)(require("./dataset/data_loader"));
6
+ var memory_1 = require("./memory");
7
+ Object.defineProperty(exports, "IndexedDB", { enumerable: true, get: function () { return memory_1.IndexedDB; } });
package/dist/index.d.ts CHANGED
@@ -1,19 +1,2 @@
1
- export * as aggregation from './aggregation';
2
- export * as dataset from './dataset';
3
- export * as serialization from './serialization';
4
- export * as tasks from './tasks';
5
- export * as training from './training';
6
- export * as privacy from './privacy';
7
- export { GraphInformant, TrainingInformant, informant } from './informant';
8
- export { Base as Client } from './client';
9
- export * as client from './client';
10
- export { AsyncBuffer } from './async_buffer';
11
- export { AsyncInformant } from './async_informant';
12
- export { Logger, ConsoleLogger, TrainerLog } from './logging';
13
- export { Memory, ModelType, ModelInfo, Path, ModelSource, Empty as EmptyMemory } from './memory';
14
- export { ModelActor } from './model_actor';
15
- export { Disco, TrainingSchemes } from './training';
16
- export { Validator } from './validation';
17
- export { TrainingInformation, DisplayInformation, isTask, Task, isTaskID, TaskID } from './task';
18
- export * from './types';
19
- export { tf } from './tfjs';
1
+ export * from './core';
2
+ export * as browser from './imports';
package/dist/index.js CHANGED
@@ -1,42 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.tf = exports.isTaskID = exports.isTask = exports.Validator = exports.TrainingSchemes = exports.Disco = exports.ModelActor = exports.EmptyMemory = exports.ModelType = exports.Memory = exports.TrainerLog = exports.ConsoleLogger = exports.Logger = exports.AsyncInformant = exports.AsyncBuffer = exports.client = exports.Client = exports.informant = exports.TrainingInformant = exports.GraphInformant = exports.privacy = exports.training = exports.tasks = exports.serialization = exports.dataset = exports.aggregation = void 0;
3
+ exports.browser = void 0;
4
4
  var tslib_1 = require("tslib");
5
- exports.aggregation = (0, tslib_1.__importStar)(require("./aggregation"));
6
- exports.dataset = (0, tslib_1.__importStar)(require("./dataset"));
7
- exports.serialization = (0, tslib_1.__importStar)(require("./serialization"));
8
- exports.tasks = (0, tslib_1.__importStar)(require("./tasks"));
9
- exports.training = (0, tslib_1.__importStar)(require("./training"));
10
- exports.privacy = (0, tslib_1.__importStar)(require("./privacy"));
11
- var informant_1 = require("./informant");
12
- Object.defineProperty(exports, "GraphInformant", { enumerable: true, get: function () { return informant_1.GraphInformant; } });
13
- Object.defineProperty(exports, "TrainingInformant", { enumerable: true, get: function () { return informant_1.TrainingInformant; } });
14
- Object.defineProperty(exports, "informant", { enumerable: true, get: function () { return informant_1.informant; } });
15
- var client_1 = require("./client");
16
- Object.defineProperty(exports, "Client", { enumerable: true, get: function () { return client_1.Base; } });
17
- exports.client = (0, tslib_1.__importStar)(require("./client"));
18
- var async_buffer_1 = require("./async_buffer");
19
- Object.defineProperty(exports, "AsyncBuffer", { enumerable: true, get: function () { return async_buffer_1.AsyncBuffer; } });
20
- var async_informant_1 = require("./async_informant");
21
- Object.defineProperty(exports, "AsyncInformant", { enumerable: true, get: function () { return async_informant_1.AsyncInformant; } });
22
- var logging_1 = require("./logging");
23
- Object.defineProperty(exports, "Logger", { enumerable: true, get: function () { return logging_1.Logger; } });
24
- Object.defineProperty(exports, "ConsoleLogger", { enumerable: true, get: function () { return logging_1.ConsoleLogger; } });
25
- Object.defineProperty(exports, "TrainerLog", { enumerable: true, get: function () { return logging_1.TrainerLog; } });
26
- var memory_1 = require("./memory");
27
- Object.defineProperty(exports, "Memory", { enumerable: true, get: function () { return memory_1.Memory; } });
28
- Object.defineProperty(exports, "ModelType", { enumerable: true, get: function () { return memory_1.ModelType; } });
29
- Object.defineProperty(exports, "EmptyMemory", { enumerable: true, get: function () { return memory_1.Empty; } });
30
- var model_actor_1 = require("./model_actor");
31
- Object.defineProperty(exports, "ModelActor", { enumerable: true, get: function () { return model_actor_1.ModelActor; } });
32
- var training_1 = require("./training");
33
- Object.defineProperty(exports, "Disco", { enumerable: true, get: function () { return training_1.Disco; } });
34
- Object.defineProperty(exports, "TrainingSchemes", { enumerable: true, get: function () { return training_1.TrainingSchemes; } });
35
- var validation_1 = require("./validation");
36
- Object.defineProperty(exports, "Validator", { enumerable: true, get: function () { return validation_1.Validator; } });
37
- var task_1 = require("./task");
38
- Object.defineProperty(exports, "isTask", { enumerable: true, get: function () { return task_1.isTask; } });
39
- Object.defineProperty(exports, "isTaskID", { enumerable: true, get: function () { return task_1.isTaskID; } });
40
- (0, tslib_1.__exportStar)(require("./types"), exports);
41
- var tfjs_1 = require("./tfjs");
42
- Object.defineProperty(exports, "tf", { enumerable: true, get: function () { return tfjs_1.tf; } });
5
+ (0, tslib_1.__exportStar)(require("./core"), exports);
6
+ exports.browser = (0, tslib_1.__importStar)(require("./imports"));
@@ -1,3 +1 @@
1
- export { Empty } from './empty';
2
- export { Memory, ModelInfo, Path, ModelSource } from './base';
3
- export { ModelType } from './model_type';
1
+ export { IndexedDB } from './memory';
@@ -1,9 +1,5 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ModelType = exports.Memory = exports.Empty = void 0;
4
- var empty_1 = require("./empty");
5
- Object.defineProperty(exports, "Empty", { enumerable: true, get: function () { return empty_1.Empty; } });
6
- var base_1 = require("./base");
7
- Object.defineProperty(exports, "Memory", { enumerable: true, get: function () { return base_1.Memory; } });
8
- var model_type_1 = require("./model_type");
9
- Object.defineProperty(exports, "ModelType", { enumerable: true, get: function () { return model_type_1.ModelType; } });
3
+ exports.IndexedDB = void 0;
4
+ var memory_1 = require("./memory");
5
+ Object.defineProperty(exports, "IndexedDB", { enumerable: true, get: function () { return memory_1.IndexedDB; } });
@@ -0,0 +1,26 @@
1
+ import { tf, Memory, Path, ModelInfo, ModelSource } from '..';
2
+ export declare class IndexedDB extends Memory {
3
+ pathFor(source: ModelSource): Path;
4
+ infoFor(source: ModelSource): ModelInfo;
5
+ getModelMetadata(source: ModelSource): Promise<tf.io.ModelArtifactsInfo | undefined>;
6
+ contains(source: ModelSource): Promise<boolean>;
7
+ getModel(source: ModelSource): Promise<tf.LayersModel>;
8
+ deleteModel(source: ModelSource): Promise<void>;
9
+ loadModel(source: ModelSource): Promise<void>;
10
+ /**
11
+ * Saves the working model to the source.
12
+ * @param source the destination
13
+ * @param model the model
14
+ */
15
+ updateWorkingModel(source: ModelSource, model: tf.LayersModel): Promise<void>;
16
+ /**
17
+ * Creates a saved copy of the working model corresponding to the source.
18
+ * @param source the source
19
+ */
20
+ saveWorkingModel(source: ModelSource): Promise<void>;
21
+ /**
22
+ * Downloads the model corresponding to the source.
23
+ * @param source the source
24
+ */
25
+ downloadModel(source: ModelSource): Promise<void>;
26
+ }
@@ -0,0 +1,160 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.IndexedDB = void 0;
4
+ var tslib_1 = require("tslib");
5
+ /**
6
+ * Helper functions used to load and save TFJS models from IndexedDB. The
7
+ * working model is the model currently being trained for a task. Saved models
8
+ * are models that were explicitly saved to IndexedDB. The two working/ and saved/
9
+ * folders are invisible to the user. The user only interacts with the saved/
10
+ * folder via the model library. The working/ folder is only used by the backend.
11
+ * The working model is loaded from IndexedDB for training (model.fit) only.
12
+ */
13
+ var path_1 = (0, tslib_1.__importDefault)(require("path"));
14
+ var __1 = require("..");
15
+ var IndexedDB = /** @class */ (function (_super) {
16
+ (0, tslib_1.__extends)(IndexedDB, _super);
17
+ function IndexedDB() {
18
+ return _super !== null && _super.apply(this, arguments) || this;
19
+ }
20
+ IndexedDB.prototype.pathFor = function (source) {
21
+ if (typeof source === 'string') {
22
+ return source;
23
+ }
24
+ if (source.type === undefined || source.taskID === undefined || source.name === undefined) {
25
+ throw new TypeError('source incomplete');
26
+ }
27
+ return 'indexeddb://' + path_1.default.join(source.type, source.taskID, source.name);
28
+ };
29
+ IndexedDB.prototype.infoFor = function (source) {
30
+ if (typeof source !== 'string') {
31
+ return source;
32
+ }
33
+ var _a = (0, tslib_1.__read)(source.split('/').splice(2), 3), stringType = _a[0], taskID = _a[1], name = _a[2];
34
+ var type = stringType === 'working' ? __1.ModelType.WORKING : __1.ModelType.SAVED;
35
+ return { type: type, taskID: taskID, name: name };
36
+ };
37
+ IndexedDB.prototype.getModelMetadata = function (source) {
38
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
39
+ var models;
40
+ return (0, tslib_1.__generator)(this, function (_a) {
41
+ switch (_a.label) {
42
+ case 0: return [4 /*yield*/, __1.tf.io.listModels()];
43
+ case 1:
44
+ models = _a.sent();
45
+ return [2 /*return*/, models[this.pathFor(source)]];
46
+ }
47
+ });
48
+ });
49
+ };
50
+ IndexedDB.prototype.contains = function (source) {
51
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
52
+ return (0, tslib_1.__generator)(this, function (_a) {
53
+ switch (_a.label) {
54
+ case 0: return [4 /*yield*/, this.getModelMetadata(source)];
55
+ case 1: return [2 /*return*/, (_a.sent()) !== undefined];
56
+ }
57
+ });
58
+ });
59
+ };
60
+ IndexedDB.prototype.getModel = function (source) {
61
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
62
+ return (0, tslib_1.__generator)(this, function (_a) {
63
+ switch (_a.label) {
64
+ case 0: return [4 /*yield*/, __1.tf.loadLayersModel(this.pathFor(source))];
65
+ case 1: return [2 /*return*/, _a.sent()];
66
+ }
67
+ });
68
+ });
69
+ };
70
+ IndexedDB.prototype.deleteModel = function (source) {
71
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
72
+ return (0, tslib_1.__generator)(this, function (_a) {
73
+ switch (_a.label) {
74
+ case 0: return [4 /*yield*/, __1.tf.io.removeModel(this.pathFor(source))];
75
+ case 1:
76
+ _a.sent();
77
+ return [2 /*return*/];
78
+ }
79
+ });
80
+ });
81
+ };
82
+ IndexedDB.prototype.loadModel = function (source) {
83
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
84
+ return (0, tslib_1.__generator)(this, function (_a) {
85
+ switch (_a.label) {
86
+ case 0: return [4 /*yield*/, __1.tf.io.copyModel(this.pathFor(source), this.pathFor((0, tslib_1.__assign)((0, tslib_1.__assign)({}, this.infoFor(source)), { type: __1.ModelType.WORKING })))];
87
+ case 1:
88
+ _a.sent();
89
+ return [2 /*return*/];
90
+ }
91
+ });
92
+ });
93
+ };
94
+ /**
95
+ * Saves the working model to the source.
96
+ * @param source the destination
97
+ * @param model the model
98
+ */
99
+ IndexedDB.prototype.updateWorkingModel = function (source, model) {
100
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
101
+ var src;
102
+ return (0, tslib_1.__generator)(this, function (_a) {
103
+ switch (_a.label) {
104
+ case 0:
105
+ src = this.infoFor(source);
106
+ if (src.type !== undefined && src.type !== __1.ModelType.WORKING) {
107
+ throw new TypeError('expected working model');
108
+ }
109
+ return [4 /*yield*/, model.save(this.pathFor((0, tslib_1.__assign)((0, tslib_1.__assign)({}, src), { type: __1.ModelType.WORKING })))];
110
+ case 1:
111
+ _a.sent();
112
+ return [2 /*return*/];
113
+ }
114
+ });
115
+ });
116
+ };
117
+ /**
118
+ * Creates a saved copy of the working model corresponding to the source.
119
+ * @param source the source
120
+ */
121
+ IndexedDB.prototype.saveWorkingModel = function (source) {
122
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
123
+ var src;
124
+ return (0, tslib_1.__generator)(this, function (_a) {
125
+ switch (_a.label) {
126
+ case 0:
127
+ src = this.infoFor(source);
128
+ if (src.type !== undefined && src.type !== __1.ModelType.WORKING) {
129
+ throw new TypeError('expected working model');
130
+ }
131
+ return [4 /*yield*/, __1.tf.io.copyModel(this.pathFor((0, tslib_1.__assign)((0, tslib_1.__assign)({}, src), { type: __1.ModelType.WORKING })), this.pathFor((0, tslib_1.__assign)((0, tslib_1.__assign)({}, src), { type: __1.ModelType.SAVED })))];
132
+ case 1:
133
+ _a.sent();
134
+ return [2 /*return*/];
135
+ }
136
+ });
137
+ });
138
+ };
139
+ /**
140
+ * Downloads the model corresponding to the source.
141
+ * @param source the source
142
+ */
143
+ IndexedDB.prototype.downloadModel = function (source) {
144
+ return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
145
+ var src;
146
+ return (0, tslib_1.__generator)(this, function (_a) {
147
+ switch (_a.label) {
148
+ case 0:
149
+ src = this.infoFor(source);
150
+ return [4 /*yield*/, __1.tf.io.copyModel(this.pathFor(source), "downloads://" + src.taskID + "_" + src.name)];
151
+ case 1:
152
+ _a.sent();
153
+ return [2 /*return*/];
154
+ }
155
+ });
156
+ });
157
+ };
158
+ return IndexedDB;
159
+ }(__1.Memory));
160
+ exports.IndexedDB = IndexedDB;