@epfml/discojs-node 2.1.1 → 2.1.2-p20240506085559.0

package/dist/data/image_loader.d.ts

@@ -0,0 +1,5 @@
+import type tf from '@tensorflow/tfjs';
+import { data } from '@epfml/discojs';
+export declare class ImageLoader extends data.ImageLoader<string> {
+    readImageFrom(source: string, channels?: number): Promise<tf.Tensor3D>;
+}

package/dist/data/image_loader.js

@@ -0,0 +1,11 @@
+import fs from 'node:fs/promises';
+import { node as tfNode } from '@tensorflow/tfjs-node';
+import { data } from '@epfml/discojs';
+export class ImageLoader extends data.ImageLoader {
+    async readImageFrom(source, channels) {
+        // We allow specifying the number of channels because the default number of channels
+        // differs between web and node for the same image 
+        // E.g. lus covid images have 1 channel with fs.readFile but 3 when loaded with discojs-web
+        return tfNode.decodeImage(await fs.readFile(source), channels);
+    }
+}

package/dist/data/index.d.ts

@@ -0,0 +1,3 @@
+export { ImageLoader as NodeImageLoader } from './image_loader.js';
+export { TabularLoader as NodeTabularLoader } from './tabular_loader.js';
+export { TextLoader as NodeTextLoader } from './text_loader.js';

package/dist/data/index.js

@@ -0,0 +1,3 @@
+export { ImageLoader as NodeImageLoader } from './image_loader.js';
+export { TabularLoader as NodeTabularLoader } from './tabular_loader.js';
+export { TextLoader as NodeTextLoader } from './text_loader.js';

package/dist/data/tabular_loader.d.ts

@@ -0,0 +1,4 @@
+import { data } from '@epfml/discojs';
+export declare class TabularLoader extends data.TabularLoader<string> {
+    loadDatasetFrom(source: string, csvConfig: Record<string, unknown>): Promise<data.Dataset>;
+}

package/dist/data/tabular_loader.js

@@ -0,0 +1,11 @@
+import { data as tfData } from '@tensorflow/tfjs-node';
+import { data } from '@epfml/discojs';
+export class TabularLoader extends data.TabularLoader {
+    loadDatasetFrom(source, csvConfig) {
+        const prefix = 'file://';
+        if (source.slice(0, 7) !== prefix) {
+            source = prefix + source;
+        }
+        return Promise.resolve(tfData.csv(source, csvConfig));
+    }
+}

package/dist/data/text_loader.d.ts

@@ -0,0 +1,4 @@
+import { data } from '@epfml/discojs';
+export declare class TextLoader extends data.TextLoader<string> {
+    loadDatasetFrom(source: string): Promise<data.Dataset>;
+}

package/dist/data/text_loader.js

@@ -0,0 +1,14 @@
+import { data as tfData } from '@tensorflow/tfjs-node';
+import fs from 'node:fs/promises';
+import { data } from '@epfml/discojs';
+export class TextLoader extends data.TextLoader {
+    async loadDatasetFrom(source) {
+        // TODO: reads all the file at once, 
+        // inputting the file path to FileDataSource isn't supported anymore
+        const inputFile = await fs.readFile(source);
+        const file = new tfData.FileDataSource(inputFile, { chunkSize: 1024 });
+        // TODO: reading files line by line is an issue for LLM tokenization
+        const dataset = new tfData.TextLineDataset(file).filter(s => s != ' '); // newline creates empty strings
+        return Promise.resolve(dataset);
+    }
+}

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-export * from './core';
-export * as node from './imports';
+export * from './data/index.js';
+export { saveModelToDisk, loadModelFromDisk } from './models/model_loader.js';

package/dist/index.js

@@ -1,6 +1,2 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.node = void 0;
-var tslib_1 = require("tslib");
-(0, tslib_1.__exportStar)(require("./core"), exports);
-exports.node = (0, tslib_1.__importStar)(require("./imports"));
+export * from './data/index.js';
+export { saveModelToDisk, loadModelFromDisk } from './models/model_loader.js';

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "@epfml/discojs-node",
-  "version": "2.1.1",
+  "version": "2.1.2-p20240506085559.0",
+  "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
-    "build": "cp ./src/core/index.node.ts ./src/core/index.ts && tsc",
-    "build-win": "copy ..\\discojs-core\\src\\index.node.ts ..\\discojs-core\\src\\index.ts && tsc",
-    "test": "cp ./src/core/index.node.ts ./src/core/index.ts && mocha",
-    "lint": "cp ./src/core/index.node.ts ./src/core/index.ts && npx eslint --max-warnings 0 --ignore-pattern '*.spec.ts' --ignore-pattern 'src/core' ."
+    "watch": "nodemon --ext ts --ignore dist --watch ../discojs/dist --watch . --exec npm run",
+    "build": "tsc",
+    "lint": "npx eslint .",
+    "test": "mocha"
   },
   "repository": {
     "type": "git",
@@ -18,17 +19,13 @@
   },
   "homepage": "https://github.com/epfml/disco#readme",
   "dependencies": {
-    "msgpack-lite": "0.1",
-    "immutable": "4",
-    "tslib": "2",
-    "@tensorflow/tfjs-node": "4",
-    "isomorphic-ws": "4",
-    "url": "0.11",
+    "@epfml/discojs": "*",
     "@koush/wrtc": "0.5",
-    "axios": "0.27",
-    "@types/msgpack-lite": "0.1",
-    "uuid": "8",
-    "ws": "8",
-    "simple-peer": "9"
+    "@tensorflow/tfjs-node": "4"
+  },
+  "devDependencies": {
+    "@types/node": "20",
+    "nodemon": "3",
+    "ts-node": "10"
   }
 }

package/README.md

@@ -1,53 +0,0 @@
-# Disco.js Node Module
-
-`discojs-node` contains the Node.js code of Disco.js, based off and extending `discojs-core`.
-
-## Installation
-
-The `discojs-node` project is available as the `@epfml/discojs-node` NPM package, which can be installed with
-`npm i @epfml/discojs-node`.
-
-### Development Environment
-
-The dev tools run on Node.js and require `npm`, a package manager for the Node.js runtime environment.
-We recommend using [nvm](https://github.com/nvm-sh/nvm) for installing both Node.js and NPM.
-
-To install the project's dependencies, run:
-
-```
-cd ..
-npm ci
-```
-
-Since the dependencies of `discojs-core`, `discojs-web` and `discojs-node` are the same, they are specified in a top-level `package.json` file, to ease installation and building.
-
-> **⚠ WARNING: Apple Silicon.**
-> `TensorFlow.js` version `3` do support M1 processors for macs. To do so, make sure you have an `arm` Node.js executable installed (not `x86_64`). It can be checked using:
-
-```
-node -p "process.arch"
-```
-
-which should return something similar to `arm64`.
-
-## Build
-
-The server and CLI modules, as well as all unit tests (except Cypress) use the `discojs-node` interface, i.e. they all run on Node.js. This Disco.js Node module is build on top of and extends `discojs-core`, whose code is [symlinked](https://en.wikipedia.org/wiki/Symbolic_link) into `discojs-node/src/core`. To build this project:
-
-```
-npm run build
-```
-
-This invokes the TypeScript compiler (`tsc`). It will output the compilation files of `discojs-node` in a `dist/` directory. To recompile from stratch, simply `rm -rf dist/` before running `npm run build` again.
-
-## Development
-
-### Contributing
-
-Contributions to `discojs-node` must only include Node-specific code. Code common to both Node and the browser must be added to `discojs-core` instead.
-
-As a rule of thumb, the `src/core/` directory must never be modified when modifying `discojs-node`, since it is [symlinked](https://en.wikipedia.org/wiki/Symbolic_link) to `discojs-core`.
-
-If you wish to add a new file or submodule to the project, please do so in a similar way as `src/core/` is structured. That is, [adding a new task](../../docs/TASK.md) to `discojs-node` would mean adding a new file to `src/tasks/` and modifying `src/tasks/index.ts` (NOT `src/core/tasks/...`).
-
-Note that, if you end up making calls to the Tensorflow.js API, you must import it from the root index. This is to ensure the Node version of TF.js is loaded, and only once.

package/dist/core/async_buffer.d.ts

@@ -1,41 +0,0 @@
-import { Map } from 'immutable';
-import { TaskID, AsyncInformant } from '.';
-/**
- * The AsyncWeightsBuffer class holds and manipulates information about the
- * async weights buffer. It works as follows:
- *
- * Setup: Init round to zero and create empty buffer (a map from user id to weights)
- *
- * - When a user adds weights only do so when they are recent weights: i.e. this.round - round <= roundCutoff.
- * - If a user already added weights, update them. (-> there can be at most one entry of weights per id in a buffer).
- * - When the buffer is full, call aggregateAndStoreWeights with the weights in the buffer and then increment round by one  and reset the buffer.
- *
- * @remarks
- * taskID: corresponds to the task that weights correspond to.
- * bufferCapacity: size of the buffer.
- * buffer: holds a map of users to their added weights.
- * round: the latest round of the weight buffer.
- * roundCutoff: cutoff for accepted rounds.
- */
-export declare class AsyncBuffer<T> {
-    readonly taskID: TaskID;
-    private readonly bufferCapacity;
-    private readonly aggregateAndStoreWeights;
-    private readonly roundCutoff;
-    buffer: Map<string, T>;
-    round: number;
-    private observer;
-    constructor(taskID: TaskID, bufferCapacity: number, aggregateAndStoreWeights: (weights: Iterable<T>) => Promise<void>, roundCutoff?: number);
-    registerObserver(observer: AsyncInformant<T>): void;
-    bufferIsFull(): boolean;
-    private updateWeightsIfBufferIsFull;
-    isNotWithinRoundCutoff(round: number): boolean;
-    /**
-       * Add weights originating from weights of a given round.
-       * Only add to buffer if the given round is not old.
-       * @param weights
-       * @param round
-       * @returns true if weights were added, and false otherwise
-       */
-    add(id: string, weights: T, round: number): Promise<boolean>;
-}

package/dist/core/async_buffer.js

@@ -1,97 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.AsyncBuffer = void 0;
-var tslib_1 = require("tslib");
-var immutable_1 = require("immutable");
-/**
- * The AsyncWeightsBuffer class holds and manipulates information about the
- * async weights buffer. It works as follows:
- *
- * Setup: Init round to zero and create empty buffer (a map from user id to weights)
- *
- * - When a user adds weights only do so when they are recent weights: i.e. this.round - round <= roundCutoff.
- * - If a user already added weights, update them. (-> there can be at most one entry of weights per id in a buffer).
- * - When the buffer is full, call aggregateAndStoreWeights with the weights in the buffer and then increment round by one  and reset the buffer.
- *
- * @remarks
- * taskID: corresponds to the task that weights correspond to.
- * bufferCapacity: size of the buffer.
- * buffer: holds a map of users to their added weights.
- * round: the latest round of the weight buffer.
- * roundCutoff: cutoff for accepted rounds.
- */
-var AsyncBuffer = /** @class */ (function () {
-    function AsyncBuffer(taskID, bufferCapacity, aggregateAndStoreWeights, roundCutoff) {
-        if (roundCutoff === void 0) { roundCutoff = 0; }
-        this.taskID = taskID;
-        this.bufferCapacity = bufferCapacity;
-        this.aggregateAndStoreWeights = aggregateAndStoreWeights;
-        this.roundCutoff = roundCutoff;
-        this.buffer = (0, immutable_1.Map)();
-        this.round = 0;
-    }
-    AsyncBuffer.prototype.registerObserver = function (observer) {
-        this.observer = observer;
-    };
-    // TODO do not test private
-    AsyncBuffer.prototype.bufferIsFull = function () {
-        return this.buffer.size >= this.bufferCapacity;
-    };
-    AsyncBuffer.prototype.updateWeightsIfBufferIsFull = function () {
-        var _a;
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            return (0, tslib_1.__generator)(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        if (!this.bufferIsFull()) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.aggregateAndStoreWeights(this.buffer.values())];
-                    case 1:
-                        _b.sent();
-                        this.round += 1;
-                        (_a = this.observer) === null || _a === void 0 ? void 0 : _a.update();
-                        this.buffer = (0, immutable_1.Map)();
-                        console.log('\n************************************************************');
-                        console.log("Buffer is full; Aggregating weights and starting round: " + this.round + "\n");
-                        _b.label = 2;
-                    case 2: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    // TODO do not test private
-    AsyncBuffer.prototype.isNotWithinRoundCutoff = function (round) {
-        // Note that always this.round >= round
-        return this.round - round > this.roundCutoff;
-    };
-    /**
-       * Add weights originating from weights of a given round.
-       * Only add to buffer if the given round is not old.
-       * @param weights
-       * @param round
-       * @returns true if weights were added, and false otherwise
-       */
-    AsyncBuffer.prototype.add = function (id, weights, round) {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var weightsUpdatedByUser, msg;
-            return (0, tslib_1.__generator)(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (this.isNotWithinRoundCutoff(round)) {
-                            console.log("Did not add weights of " + id + " to buffer. Due to old round update: " + round + ", current round is " + this.round);
-                            return [2 /*return*/, false];
-                        }
-                        weightsUpdatedByUser = this.buffer.has(id);
-                        msg = weightsUpdatedByUser ? '\tUpdating' : '-> Adding new';
-                        console.log(msg + " weights of " + id + " to buffer.");
-                        this.buffer = this.buffer.set(id, weights);
-                        return [4 /*yield*/, this.updateWeightsIfBufferIsFull()];
-                    case 1:
-                        _a.sent();
-                        return [2 /*return*/, true];
-                }
-            });
-        });
-    };
-    return AsyncBuffer;
-}());
-exports.AsyncBuffer = AsyncBuffer;

package/dist/core/async_informant.d.ts

@@ -1,20 +0,0 @@
-import { AsyncBuffer } from './async_buffer';
-export declare class AsyncInformant<T> {
-    private readonly asyncBuffer;
-    private round;
-    private currentNumberOfParticipants;
-    private totalNumberOfParticipants;
-    private averageNumberOfParticipants;
-    constructor(asyncBuffer: AsyncBuffer<T>);
-    update(): void;
-    private updateRound;
-    private updateNumberOfParticipants;
-    private updateAverageNumberOfParticipants;
-    private updateTotalNumberOfParticipants;
-    getCurrentRound(): number;
-    getNumberOfParticipants(): number;
-    getTotalNumberOfParticipants(): number;
-    getAverageNumberOfParticipants(): number;
-    getAllStatistics(): Record<'round' | 'currentNumberOfParticipants' | 'totalNumberOfParticipants' | 'averageNumberOfParticipants', number>;
-    printAllInfos(): void;
-}

package/dist/core/async_informant.js

@@ -1,69 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.AsyncInformant = void 0;
-var AsyncInformant = /** @class */ (function () {
-    function AsyncInformant(asyncBuffer) {
-        this.asyncBuffer = asyncBuffer;
-        this.round = 0;
-        this.currentNumberOfParticipants = 0;
-        this.totalNumberOfParticipants = 0;
-        this.averageNumberOfParticipants = 0;
-        this.asyncBuffer.registerObserver(this);
-    }
-    // Update functions
-    AsyncInformant.prototype.update = function () {
-        // DEBUG
-        console.log('Before update');
-        this.printAllInfos();
-        this.updateRound();
-        this.updateNumberOfParticipants();
-        // DEBUG
-        console.log('After update');
-        this.printAllInfos();
-    };
-    AsyncInformant.prototype.updateRound = function () {
-        this.round = this.asyncBuffer.round;
-    };
-    AsyncInformant.prototype.updateNumberOfParticipants = function () {
-        this.currentNumberOfParticipants = this.asyncBuffer.buffer.size;
-        this.updateTotalNumberOfParticipants(this.currentNumberOfParticipants);
-        this.updateAverageNumberOfParticipants();
-    };
-    AsyncInformant.prototype.updateAverageNumberOfParticipants = function () {
-        this.averageNumberOfParticipants = this.totalNumberOfParticipants / this.round;
-    };
-    AsyncInformant.prototype.updateTotalNumberOfParticipants = function (currentNumberOfParticipants) {
-        this.totalNumberOfParticipants += currentNumberOfParticipants;
-    };
-    // Getter functions
-    AsyncInformant.prototype.getCurrentRound = function () {
-        return this.round;
-    };
-    AsyncInformant.prototype.getNumberOfParticipants = function () {
-        return this.currentNumberOfParticipants;
-    };
-    AsyncInformant.prototype.getTotalNumberOfParticipants = function () {
-        return this.totalNumberOfParticipants;
-    };
-    AsyncInformant.prototype.getAverageNumberOfParticipants = function () {
-        return this.averageNumberOfParticipants;
-    };
-    AsyncInformant.prototype.getAllStatistics = function () {
-        return {
-            round: this.getCurrentRound(),
-            currentNumberOfParticipants: this.getNumberOfParticipants(),
-            totalNumberOfParticipants: this.getTotalNumberOfParticipants(),
-            averageNumberOfParticipants: this.getAverageNumberOfParticipants()
-        };
-    };
-    // Debug
-    AsyncInformant.prototype.printAllInfos = function () {
-        console.log('task : ', this.asyncBuffer.taskID);
-        console.log('round : ', this.getCurrentRound());
-        console.log('participants : ', this.getNumberOfParticipants());
-        console.log('total : ', this.getTotalNumberOfParticipants());
-        console.log('average : ', this.getAverageNumberOfParticipants());
-    };
-    return AsyncInformant;
-}());
-exports.AsyncInformant = AsyncInformant;

package/dist/core/client/base.d.ts

@@ -1,33 +0,0 @@
-import { tf, WeightsContainer, Task, TrainingInformant } from '..';
-export declare abstract class Base {
-    readonly url: URL;
-    readonly task: Task;
-    protected connected: boolean;
-    constructor(url: URL, task: Task);
-    /**
-     * Handles the connection process from the client to any sort of
-     * centralized server.
-     */
-    abstract connect(): Promise<void>;
-    /**
-     * Handles the disconnection process of the client from any sort
-     * of centralized server.
-     */
-    abstract disconnect(): Promise<void>;
-    getLatestModel(): Promise<tf.LayersModel>;
-    /**
-     * The training manager matches this function with the training loop's
-     * onTrainEnd callback when training a TFJS model object. See the
-     * training manager for more details.
-     */
-    abstract onTrainEndCommunication(weights: WeightsContainer, trainingInformant: TrainingInformant): Promise<void>;
-    /**
-     * This function will be called whenever a local round has ended.
-     *
-     * @param updatedWeights
-     * @param staleWeights
-     * @param round
-     * @param trainingInformant
-     */
-    abstract onRoundEndCommunication(updatedWeights: WeightsContainer, staleWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<WeightsContainer>;
-}

package/dist/core/client/base.js

@@ -1,35 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Base = void 0;
-var tslib_1 = require("tslib");
-var axios_1 = (0, tslib_1.__importDefault)(require("axios"));
-var __1 = require("..");
-var Base = /** @class */ (function () {
-    function Base(url, task) {
-        this.url = url;
-        this.task = task;
-        this.connected = false;
-    }
-    Base.prototype.getLatestModel = function () {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var url, response;
-            return (0, tslib_1.__generator)(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        url = new URL('', this.url.href);
-                        if (!url.pathname.endsWith('/')) {
-                            url.pathname += '/';
-                        }
-                        url.pathname += "tasks/" + this.task.taskID + "/model.json";
-                        return [4 /*yield*/, axios_1.default.get(url.href)];
-                    case 1:
-                        response = _a.sent();
-                        return [4 /*yield*/, __1.serialization.model.decode(response.data)];
-                    case 2: return [2 /*return*/, _a.sent()];
-                }
-            });
-        });
-    };
-    return Base;
-}());
-exports.Base = Base;

package/dist/core/client/decentralized/base.d.ts

@@ -1,32 +0,0 @@
-import { List, Map } from 'immutable';
-import { TrainingInformant, WeightsContainer, Task } from '../..';
-import { Base as ClientBase } from '../base';
-import { PeerID } from './types';
-import * as messages from './messages';
-import { PeerConnection } from '../event_connection';
-/**
- * Abstract class for decentralized clients, executes onRoundEndCommunication as well as connecting
- * to the signaling server
- */
-export declare abstract class Base extends ClientBase {
-    readonly url: URL;
-    readonly task: Task;
-    protected readonly minimumReadyPeers: number;
-    private server?;
-    private peers?;
-    private ID?;
-    private pool?;
-    constructor(url: URL, task: Task);
-    private waitForPeers;
-    protected sendMessagetoPeer(peer: PeerConnection, msg: messages.PeerMessage): void;
-    private connectServer;
-    /**
-     * Initialize the connection to the peers and to the other nodes.
-     */
-    connect(): Promise<void>;
-    disconnect(): Promise<void>;
-    onTrainEndCommunication(_: WeightsContainer, trainingInformant: TrainingInformant): Promise<void>;
-    onRoundEndCommunication(updatedWeights: WeightsContainer, staleWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<WeightsContainer>;
-    abstract sendAndReceiveWeights(peers: Map<PeerID, PeerConnection>, noisyWeights: WeightsContainer, round: number, trainingInformant: TrainingInformant): Promise<List<WeightsContainer>>;
-    abstract clientHandle(peers: Map<PeerID, PeerConnection>): void;
-}