@epfml/discojs 0.0.1 → 0.1.0

package/dist/training/trainer/trainer.d.ts

@@ -3,12 +3,12 @@ import { Memory, Task, TrainingInformant, TrainingInformation } from '@/.';
 import { RoundTracker } from './round_tracker';
 import { TrainerLog } from '../../logging/trainer_logger';
 /** Abstract class whose role is to train a model with a given dataset. This can be either done
- * locally or in a distributed way. The Trainer works as follows:
+ * locally (alone) or in a distributed way with collaborators. The Trainer works as follows:
  *
  * 1. Call trainModel(dataset) to start training
- * 2. Once a batch ends, onBatchEnd is triggered, witch will then call onRoundEnd once the round has ended.
+ * 2. Once a batch ends, onBatchEnd is triggered, which will then call onRoundEnd once the round has ended.
  *
- * The onRoundEnd needs to be implemented to specify what actions to do when the round has ended. To know when
+ * The onRoundEnd needs to be implemented to specify what actions to do when the round has ended, such as a communication step with collaborators. To know when
  * a round has ended we use the roundTracker object.
  */
 export declare abstract class Trainer {
@@ -30,10 +30,17 @@ export declare abstract class Trainer {
      * Every time a round ends this function will be called
      */
     protected abstract onRoundEnd(accuracy: number): Promise<void>;
+    /** onBatchEnd callback, when a round ends, we call onRoundEnd (to be implemented for local and distributed instances)
+     */
+    protected onBatchEnd(_: number, logs?: tf.Logs): Promise<void>;
+    /**
+     * We update the training graph, this needs to be done on epoch end as there is no validation accuracy onBatchEnd.
+     */
+    protected onEpochEnd(epoch: number, logs?: tf.Logs): void;
     /**
      * When the training ends this function will be call
      */
-    protected abstract onTrainEnd(): Promise<void>;
+    protected onTrainEnd(logs?: tf.Logs): Promise<void>;
     /**
      * Request stop training to be used from the Disco instance or any class that is taking care of the trainer.
      */
@@ -42,25 +49,18 @@ export declare abstract class Trainer {
      * Start training the model with the given dataset
      * @param dataset
      */
-    trainModel(dataset: tf.data.Dataset<tf.TensorContainer>): Promise<void>;
+    trainModel(dataset: tf.data.Dataset<tf.TensorContainer>, valDataset: tf.data.Dataset<tf.TensorContainer>): Promise<void>;
     /**
      * Format accuracy
      */
-    private roundDecimals;
-    /**
-     * We update the training graph, this needs to be done on epoch end as there is no validation accuracy onBatchEnd.
-     */
-    private onEpochEnd;
-    /** onBatchEnd callback, when a round ends, we call onRoundEnd (to be implemented for local and distributed instances)
-     */
-    private onBatchEnd;
+    protected roundDecimals(accuracy: number, decimalsToRound?: number): number;
     /**
      * reset stop training state
      */
-    private resetStopTrainerState;
+    protected resetStopTrainerState(): void;
     /**
      * If stop training is requested, do so
      */
-    private stopTrainModelIfRequested;
+    protected stopTrainModelIfRequested(): void;
     getTrainerLog(): TrainerLog;
 }

package/dist/training/trainer/trainer.js

@@ -5,12 +5,12 @@ var tslib_1 = require("tslib");
 var round_tracker_1 = require("./round_tracker");
 var trainer_logger_1 = require("../../logging/trainer_logger");
 /** Abstract class whose role is to train a model with a given dataset. This can be either done
- * locally or in a distributed way. The Trainer works as follows:
+ * locally (alone) or in a distributed way with collaborators. The Trainer works as follows:
  *
  * 1. Call trainModel(dataset) to start training
- * 2. Once a batch ends, onBatchEnd is triggered, witch will then call onRoundEnd once the round has ended.
+ * 2. Once a batch ends, onBatchEnd is triggered, which will then call onRoundEnd once the round has ended.
  *
- * The onRoundEnd needs to be implemented to specify what actions to do when the round has ended. To know when
+ * The onRoundEnd needs to be implemented to specify what actions to do when the round has ended, such as a communication step with collaborators. To know when
  * a round has ended we use the roundTracker object.
  */
 var Trainer = /** @class */ (function () {
@@ -33,9 +33,52 @@ var Trainer = /** @class */ (function () {
         this.trainingInformation = trainingInformation;
         this.roundTracker = new round_tracker_1.RoundTracker(trainingInformation.roundDuration);
     }
-    //* ***************************************************************
-    //* Functions to be used by the training manager
-    //* ***************************************************************
+    /** onBatchEnd callback, when a round ends, we call onRoundEnd (to be implemented for local and distributed instances)
+     */
+    Trainer.prototype.onBatchEnd = function (_, logs) {
+        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
+            return (0, tslib_1.__generator)(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (logs === undefined) {
+                            return [2 /*return*/];
+                        }
+                        this.roundTracker.updateBatch();
+                        this.stopTrainModelIfRequested();
+                        if (!this.roundTracker.roundHasEnded()) return [3 /*break*/, 2];
+                        return [4 /*yield*/, this.onRoundEnd(logs.acc)];
+                    case 1:
+                        _a.sent();
+                        _a.label = 2;
+                    case 2: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * We update the training graph, this needs to be done on epoch end as there is no validation accuracy onBatchEnd.
+     */
+    Trainer.prototype.onEpochEnd = function (epoch, logs) {
+        this.trainerLogger.onEpochEnd(epoch, logs);
+        if (logs !== undefined && !isNaN(logs.acc) && !isNaN(logs.val_acc)) {
+            this.trainingInformant.updateTrainingGraph(this.roundDecimals(logs.acc));
+            this.trainingInformant.updateValidationGraph(this.roundDecimals(logs.val_acc));
+        }
+        else {
+            this.trainerLogger.error('onEpochEnd: NaN value');
+        }
+    };
+    /**
+     * When the training ends this function will be call
+     */
+    Trainer.prototype.onTrainEnd = function (logs) {
+        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
+            return (0, tslib_1.__generator)(this, function (_a) {
+                this.trainingInformant.addMessage('Training finished.');
+                return [2 /*return*/];
+            });
+        });
+    };
     /**
      * Request stop training to be used from the Disco instance or any class that is taking care of the trainer.
      */
@@ -51,7 +94,7 @@ var Trainer = /** @class */ (function () {
      * Start training the model with the given dataset
      * @param dataset
      */
-    Trainer.prototype.trainModel = function (dataset) {
+    Trainer.prototype.trainModel = function (dataset, valDataset) {
         return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
             var _this = this;
             return (0, tslib_1.__generator)(this, function (_a) {
@@ -61,7 +104,7 @@ var Trainer = /** @class */ (function () {
                         // Assign callbacks and start training
                         return [4 /*yield*/, this.model.fitDataset(dataset.batch(this.trainingInformation.batchSize), {
                                 epochs: this.trainingInformation.epochs,
-                                validationData: dataset.batch(this.trainingInformation.batchSize),
+                                validationData: valDataset.batch(this.trainingInformation.batchSize),
                                 callbacks: {
                                     onEpochEnd: function (epoch, logs) { return _this.onEpochEnd(epoch, logs); },
                                     onBatchEnd: function (epoch, logs) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () { return (0, tslib_1.__generator)(this, function (_a) {
@@ -69,6 +112,12 @@ var Trainer = /** @class */ (function () {
                                             case 0: return [4 /*yield*/, this.onBatchEnd(epoch, logs)];
                                             case 1: return [2 /*return*/, _a.sent()];
                                         }
+                                    }); }); },
+                                    onTrainEnd: function (logs) { return (0, tslib_1.__awaiter)(_this, void 0, void 0, function () { return (0, tslib_1.__generator)(this, function (_a) {
+                                        switch (_a.label) {
+                                            case 0: return [4 /*yield*/, this.onTrainEnd(logs)];
+                                            case 1: return [2 /*return*/, _a.sent()];
+                                        }
                                     }); }); }
                                 }
                             })];
@@ -87,41 +136,6 @@ var Trainer = /** @class */ (function () {
         if (decimalsToRound === void 0) { decimalsToRound = 2; }
         return +(accuracy * 100).toFixed(decimalsToRound);
     };
-    /**
-     * We update the training graph, this needs to be done on epoch end as there is no validation accuracy onBatchEnd.
-     */
-    Trainer.prototype.onEpochEnd = function (epoch, logs) {
-        this.trainerLogger.onEpochEnd(epoch, logs);
-        if (logs !== undefined && !isNaN(logs.acc) && !isNaN(logs.val_acc)) {
-            this.trainingInformant.updateTrainingAccuracyGraph(this.roundDecimals(logs.acc));
-            this.trainingInformant.updateValidationAccuracyGraph(this.roundDecimals(logs.val_acc));
-        }
-        else {
-            this.trainerLogger.error('onEpochEnd: NaN value');
-        }
-    };
-    /** onBatchEnd callback, when a round ends, we call onRoundEnd (to be implemented for local and distributed instances)
-     */
-    Trainer.prototype.onBatchEnd = function (_, logs) {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            return (0, tslib_1.__generator)(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (logs === undefined) {
-                            return [2 /*return*/];
-                        }
-                        this.roundTracker.updateBatch();
-                        this.stopTrainModelIfRequested();
-                        if (!this.roundTracker.roundHasEnded()) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.onRoundEnd(logs.acc)];
-                    case 1:
-                        _a.sent();
-                        _a.label = 2;
-                    case 2: return [2 /*return*/];
-                }
-            });
-        });
-    };
     /**
      * reset stop training state
      */

package/dist/training/trainer/trainer_builder.js

@@ -48,28 +48,21 @@ var TrainerBuilder = /** @class */ (function () {
     TrainerBuilder.prototype.getModel = function (client) {
         var _a;
         return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var modelID, modelExistsInMemory, model;
+            var modelID, info, model;
             return (0, tslib_1.__generator)(this, function (_b) {
                 switch (_b.label) {
                     case 0:
                         modelID = (_a = this.task.trainingInformation) === null || _a === void 0 ? void 0 : _a.modelID;
                         if (modelID === undefined) {
-                            throw new Error('undefined model ID');
+                            throw new TypeError('model ID is undefined');
                         }
-                        return [4 /*yield*/, this.memory.getModelMetadata(__1.ModelType.WORKING, this.task.taskID, modelID)];
-                    case 1:
-                        modelExistsInMemory = _b.sent();
-                        if (!(modelExistsInMemory !== undefined)) return [3 /*break*/, 3];
-                        return [4 /*yield*/, this.memory.getModel(__1.ModelType.WORKING, this.task.taskID, modelID)];
+                        info = { type: __1.ModelType.WORKING, taskID: this.task.taskID, name: modelID };
+                        return [4 /*yield*/, this.memory.contains(info)];
+                    case 1: return [4 /*yield*/, ((_b.sent()) ? this.memory.getModel(info) : client.getLatestModel())];
                     case 2:
                         model = _b.sent();
-                        return [3 /*break*/, 5];
-                    case 3: return [4 /*yield*/, client.getLatestModel()];
-                    case 4:
-                        model = _b.sent();
-                        _b.label = 5;
-                    case 5: return [4 /*yield*/, this.updateModelInformation(model)];
-                    case 6: return [2 /*return*/, _b.sent()];
+                        return [4 /*yield*/, this.updateModelInformation(model)];
+                    case 3: return [2 /*return*/, _b.sent()];
                 }
             });
         });
@@ -84,7 +77,7 @@ var TrainerBuilder = /** @class */ (function () {
                 }
                 info = this.task.trainingInformation;
                 if (info === undefined) {
-                    throw new Error('undefined training information');
+                    throw new TypeError('training information is undefined');
                 }
                 model.compile(info.modelCompileData);
                 if (info.learningRate !== undefined) {

package/dist/types.d.ts

@@ -1,4 +1,4 @@
-import * as tf from '@tensorflow/tfjs';
+import { tf } from '.';
 export declare type Path = string;
 export declare type Weights = tf.Tensor[];
 export declare type MetadataID = string;

package/dist/validation/index.d.ts

@@ -0,0 +1 @@
+export { Validator } from './validator';

package/dist/validation/index.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Validator = void 0;
+var validator_1 = require("./validator");
+Object.defineProperty(exports, "Validator", { enumerable: true, get: function () { return validator_1.Validator; } });

package/dist/validation/validator.d.ts

@@ -0,0 +1,20 @@
+import * as tf from '@tensorflow/tfjs';
+import { ModelActor } from '../model_actor';
+import { Task } from '@/task';
+import { Data } from '@/dataset';
+import { Logger } from '@/logging';
+import { List } from 'immutable';
+import { Client, Memory, ModelSource } from '..';
+export declare class Validator extends ModelActor {
+    private readonly memory;
+    private readonly source?;
+    private readonly client?;
+    private readonly graphInformant;
+    private size;
+    constructor(task: Task, logger: Logger, memory: Memory, source?: ModelSource | undefined, client?: Client | undefined);
+    assess(data: Data): Promise<void>;
+    getModel(): Promise<tf.LayersModel>;
+    accuracyData(): List<number>;
+    accuracy(): number;
+    visitedSamples(): number;
+}

package/dist/validation/validator.js

@@ -0,0 +1,106 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Validator = void 0;
+var tslib_1 = require("tslib");
+var model_actor_1 = require("../model_actor");
+var immutable_1 = require("immutable");
+var __1 = require("..");
+var Validator = /** @class */ (function (_super) {
+    (0, tslib_1.__extends)(Validator, _super);
+    function Validator(task, logger, memory, source, client) {
+        var _this = _super.call(this, task, logger) || this;
+        _this.memory = memory;
+        _this.source = source;
+        _this.client = client;
+        _this.graphInformant = new __1.GraphInformant();
+        _this.size = 0;
+        if (source === undefined && client === undefined) {
+            throw new Error('cannot identify model');
+        }
+        return _this;
+    }
+    Validator.prototype.assess = function (data) {
+        var _a, _b, _c;
+        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
+            var batchSize, labels, classes, model, hits;
+            var _this = this;
+            return (0, tslib_1.__generator)(this, function (_d) {
+                switch (_d.label) {
+                    case 0:
+                        batchSize = (_a = this.task.trainingInformation) === null || _a === void 0 ? void 0 : _a.batchSize;
+                        if (batchSize === undefined) {
+                            throw new TypeError('batch size is undefined');
+                        }
+                        labels = (_b = this.task.trainingInformation) === null || _b === void 0 ? void 0 : _b.LABEL_LIST;
+                        classes = (_c = labels === null || labels === void 0 ? void 0 : labels.length) !== null && _c !== void 0 ? _c : 1;
+                        return [4 /*yield*/, this.getModel()];
+                    case 1:
+                        model = _d.sent();
+                        hits = 0;
+                        return [4 /*yield*/, data.dataset.batch(batchSize).forEachAsync(function (e) {
+                                if (typeof e === 'object' && 'xs' in e && 'ys' in e) {
+                                    var xs = e.xs;
+                                    var ys = e.ys.dataSync();
+                                    var pred = model.predict(xs, { batchSize: batchSize })
+                                        .dataSync()
+                                        .map(Math.round);
+                                    _this.size += xs.shape[0];
+                                    hits += (0, immutable_1.List)(pred).zip((0, immutable_1.List)(ys))
+                                        .map(function (_a) {
+                                        var _b = (0, tslib_1.__read)(_a, 2), p = _b[0], y = _b[1];
+                                        return 1 - Math.abs(p - y);
+                                    })
+                                        .reduce(function (acc, e) { return acc + e; }) / classes;
+                                    var currentAccuracy = hits / _this.size;
+                                    _this.graphInformant.updateAccuracy(currentAccuracy);
+                                }
+                                else {
+                                    throw new TypeError('missing feature/label in dataset');
+                                }
+                            })];
+                    case 2:
+                        _d.sent();
+                        this.logger.success("Obtained validation accuracy of " + this.accuracy());
+                        this.logger.success("Visited " + this.visitedSamples() + " samples");
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    Validator.prototype.getModel = function () {
+        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
+            var _a;
+            return (0, tslib_1.__generator)(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _a = this.source !== undefined;
+                        if (!_a) return [3 /*break*/, 2];
+                        return [4 /*yield*/, this.memory.contains(this.source)];
+                    case 1:
+                        _a = (_b.sent());
+                        _b.label = 2;
+                    case 2:
+                        if (!_a) return [3 /*break*/, 4];
+                        return [4 /*yield*/, this.memory.getModel(this.source)];
+                    case 3: return [2 /*return*/, _b.sent()];
+                    case 4:
+                        if (!(this.client !== undefined)) return [3 /*break*/, 6];
+                        return [4 /*yield*/, this.client.getLatestModel()];
+                    case 5: return [2 /*return*/, _b.sent()];
+                    case 6: throw new Error('cannot identify model');
+                }
+            });
+        });
+    };
+    Validator.prototype.accuracyData = function () {
+        return this.graphInformant.data();
+    };
+    Validator.prototype.accuracy = function () {
+        return this.graphInformant.accuracy();
+    };
+    Validator.prototype.visitedSamples = function () {
+        return this.size;
+    };
+    return Validator;
+}(model_actor_1.ModelActor));
+exports.Validator = Validator;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epfml/discojs",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
@@ -16,7 +16,6 @@
   },
   "homepage": "https://github.com/epfml/disco#readme",
   "dependencies": {
-    "@tensorflow/tfjs": "3",
     "axios": "0.27",
     "immutable": "4",
     "isomorphic-ws": "4",
@@ -45,4 +44,4 @@
     "ts-node": "10",
     "typescript": "<4.5.0"
   }
-}
+}

package/dist/client/decentralized.d.ts

@@ -1,23 +0,0 @@
-import { TrainingInformant, Weights } from '..';
-import { Base } from './base';
-/**
- * Class that deals with communication with the PeerJS server.
- * Collects the list of receivers currently connected to the PeerJS server.
- */
-export declare class Decentralized extends Base {
-    private server?;
-    private peers;
-    private readonly weights;
-    private connectServer;
-    private connectNewPeer;
-    /**
-     * Initialize the connection to the peers and to the other nodes.
-     */
-    connect(): Promise<void>;
-    /**
-     * Disconnection process when user quits the task.
-     */
-    disconnect(): Promise<void>;
-    onRoundEndCommunication(updatedWeights: Weights, staleWeights: Weights, epoch: number, trainingInformant: TrainingInformant): Promise<Weights>;
-    onTrainEndCommunication(_: Weights, trainingInformant: TrainingInformant): Promise<void>;
-}