@epfml/discojs 3.0.1-p20240902162912.0 → 3.0.1-p20240904094219.0

package/dist/default_tasks/cifar10.js

@@ -19,7 +19,6 @@ export const cifar10 = {
                 sampleDatasetInstructions: 'Opening the link should start downloading a zip file which you can unzip. To connect the data, use the CSV option below and select the file named "cifar10-labels.csv". You can now connect the images located in the "CIFAR10" folder. Note that there are only 24 images in this sample dataset which is far too few to successfully train a machine learning model.'
             },
             trainingInformation: {
-                modelID: 'cifar10-model',
                 epochs: 10,
                 roundDuration: 10,
                 validationSplit: 0.2,

package/dist/default_tasks/lus_covid.js

@@ -18,7 +18,6 @@ export const lusCovid = {
                 sampleDatasetInstructions: 'Opening the link will take you to a Switch Drive folder. You can click on the Download button in the top right corner. Unzip the file and you will get two subfolders: "COVID-" and "COVID+". You can connect the data by using the Group option and selecting each image group in its respective field.'
             },
             trainingInformation: {
-                modelID: 'lus-covid-model',
                 epochs: 50,
                 roundDuration: 2,
                 validationSplit: 0.2,

package/dist/default_tasks/mnist.js

@@ -18,7 +18,6 @@ export const mnist = {
                 sampleDatasetInstructions: 'Opening the link should start downloading a zip file which you can unzip. You can connect the data with the CSV option below using the CSV file named "mnist_labels.csv". After selecting in the CSV file, you will be able to connect the data under in the "images" folder.'
             },
             trainingInformation: {
-                modelID: 'mnist-model',
                 epochs: 20,
                 roundDuration: 2,
                 validationSplit: 0.2,

package/dist/default_tasks/simple_face.js

@@ -18,7 +18,6 @@ export const simpleFace = {
                 sampleDatasetInstructions: 'Opening the link should start downloading a zip file which you can unzip. Inside the "example_training_data" directory you should find the "simple_face" folder which contains the "adult" and "child" folders. To connect the data, select the Group option below and connect adults and children image groups.'
             },
             trainingInformation: {
-                modelID: 'simple_face-model',
                 epochs: 50,
                 roundDuration: 1,
                 validationSplit: 0.2,

package/dist/default_tasks/titanic.js

@@ -45,7 +45,6 @@ export const titanic = {
                 sampleDatasetInstructions: 'Opening the link should start downloading a CSV file which you can drag and drop in the field below.'
             },
             trainingInformation: {
-                modelID: 'titanic-model',
                 epochs: 10,
                 roundDuration: 2,
                 validationSplit: 0.2,

package/dist/default_tasks/wikitext.js

@@ -23,7 +23,6 @@ export const wikitext = {
             },
             trainingInformation: {
                 dataType: 'text',
-                modelID: 'llm-raw-model',
                 preprocessingFunctions: [data.TextPreprocessing.Tokenize, data.TextPreprocessing.LeftPadding],
                 scheme: 'federated',
                 minNbOfParticipants: 2,

package/dist/index.d.ts

@@ -8,7 +8,6 @@ export * as client from './client/index.js';
 export * as aggregator from './aggregator/index.js';
 export { WeightsContainer, aggregation } from './weights/index.js';
 export { Logger, ConsoleLogger } from './logging/index.js';
-export { Memory, type ModelInfo, type ModelSource, Empty as EmptyMemory } from './memory/index.js';
 export { Disco, RoundLogs, RoundStatus } from './training/index.js';
 export { Validator } from './validation/index.js';
 export { Model, BatchLogs, EpochLogs, ValidationMetrics } from './models/index.js';

package/dist/index.js

@@ -6,7 +6,6 @@ export * as client from './client/index.js';
 export * as aggregator from './aggregator/index.js';
 export { WeightsContainer, aggregation } from './weights/index.js';
 export { ConsoleLogger } from './logging/index.js';
-export { Memory, Empty as EmptyMemory } from './memory/index.js';
 export { Disco } from './training/index.js';
 export { Validator } from './validation/index.js';
 export { Model, EpochLogs } from './models/index.js';

package/dist/serialization/model.js

@@ -8,16 +8,25 @@ export function isEncoded(raw) {
     return raw instanceof Uint8Array;
 }
 export async function encode(model) {
-    if (model instanceof models.TFJS) {
-        const serialized = await model.serialize();
-        return msgpack.encode([Type.TFJS, serialized]);
-    }
-    if (model instanceof models.GPT) {
-        const { weights, config } = model.serialize();
-        const serializedWeights = await serialization.weights.encode(weights);
-        return msgpack.encode([Type.GPT, serializedWeights, config]);
+    let encoded;
+    switch (true) {
+        case model instanceof models.TFJS: {
+            const serialized = await model.serialize();
+            encoded = msgpack.encode([Type.TFJS, serialized]);
+            break;
+        }
+        case model instanceof models.GPT: {
+            const { weights, config } = model.serialize();
+            const serializedWeights = await serialization.weights.encode(weights);
+            encoded = msgpack.encode([Type.GPT, serializedWeights, config]);
+            break;
+        }
+        default:
+            throw new Error("unknown model type");
     }
-    throw new Error('unknown model type');
+    // Node's Buffer extends Node's Uint8Array, which might not be the same
+    // as the browser's Uint8Array. we ensure here that it is.
+    return new Uint8Array(encoded);
 }
 export async function decode(encoded) {
     if (!isEncoded(encoded)) {

package/dist/task/training_information.d.ts

@@ -5,7 +5,6 @@ interface Privacy {
     noiseScale?: number;
 }
 export interface TrainingInformation {
-    modelID: string;
     epochs: number;
     roundDuration: number;
     validationSplit: number;

package/dist/task/training_information.js

@@ -24,9 +24,8 @@ export function isTrainingInformation(raw) {
     if (typeof raw !== 'object' || raw === null) {
         return false;
     }
-    const { IMAGE_H, IMAGE_W, LABEL_LIST, aggregator, batchSize, dataType, decentralizedSecure, privacy, epochs, inputColumns, maxShareValue, minNbOfParticipants, modelID, outputColumns, preprocessingFunctions, roundDuration, scheme, validationSplit, tokenizer, maxSequenceLength, tensorBackend } = raw;
+    const { IMAGE_H, IMAGE_W, LABEL_LIST, aggregator, batchSize, dataType, decentralizedSecure, privacy, epochs, inputColumns, maxShareValue, minNbOfParticipants, outputColumns, preprocessingFunctions, roundDuration, scheme, validationSplit, tokenizer, maxSequenceLength, tensorBackend } = raw;
     if (typeof dataType !== 'string' ||
-        typeof modelID !== 'string' ||
         typeof epochs !== 'number' ||
         typeof batchSize !== 'number' ||
         typeof roundDuration !== 'number' ||
@@ -97,7 +96,6 @@ export function isTrainingInformation(raw) {
         inputColumns,
         maxShareValue,
         minNbOfParticipants,
-        modelID,
         outputColumns,
         preprocessingFunctions,
         roundDuration,

package/dist/training/disco.d.ts

@@ -1,4 +1,4 @@
-import { client as clients, BatchLogs, EpochLogs, Logger, Memory, Task, TrainingInformation } from "../index.js";
+import { client as clients, BatchLogs, EpochLogs, Logger, Task, TrainingInformation } from "../index.js";
 import type { TypedLabeledDataset } from "../index.js";
 import type { Aggregator } from "../aggregator/index.js";
 import { EventEmitter } from "../utils/event_emitter.js";
@@ -6,13 +6,12 @@ import { RoundLogs, Trainer } from "./trainer.js";
 interface DiscoConfig {
     scheme: TrainingInformation["scheme"];
     logger: Logger;
-    memory: Memory;
 }
 export type RoundStatus = "Waiting for more participants" | "Retrieving peers' information" | "Updating the model with other participants' models" | "Training the model on the data you connected";
 /**
  * Top-level class handling distributed training from a client's perspective. It is meant to be
- * a convenient object providing a reduced yet complete API that wraps model training,
- * communication with nodes, logs and model memory.
+ * a convenient object providing a reduced yet complete API that wraps model training and
+ * communication with nodes.
  */
 export declare class Disco extends EventEmitter<{
     'status': RoundStatus;

package/dist/training/disco.js

@@ -1,4 +1,4 @@
-import { async_iterator, client as clients, ConsoleLogger, EmptyMemory, } from "../index.js";
+import { async_iterator, client as clients, ConsoleLogger, } from "../index.js";
 import { getAggregator } from "../aggregator/index.js";
 import { enumerate, split } from "../utils/async_iterator.js";
 import { EventEmitter } from "../utils/event_emitter.js";
@@ -6,14 +6,13 @@ import { Trainer } from "./trainer.js";
 import { labeledDatasetToDataSplit } from "../dataset/data/helpers.js";
 /**
  * Top-level class handling distributed training from a client's perspective. It is meant to be
- * a convenient object providing a reduced yet complete API that wraps model training,
- * communication with nodes, logs and model memory.
+ * a convenient object providing a reduced yet complete API that wraps model training and
+ * communication with nodes.
  */
 export class Disco extends EventEmitter {
     trainer;
     #client;
     #logger;
-    #memory;
     #task;
     /**
      * Connect to the given task and get ready to train.
@@ -24,10 +23,9 @@ export class Disco extends EventEmitter {
      */
     constructor(task, clientConfig, config) {
         super();
-        const { scheme, logger, memory } = {
+        const { scheme, logger } = {
             scheme: task.trainingInformation.scheme,
             logger: new ConsoleLogger(),
-            memory: new EmptyMemory(),
             ...config,
         };
         let client;
@@ -49,7 +47,6 @@ export class Disco extends EventEmitter {
             throw new Error("client not setup for given task");
         this.#logger = logger;
         this.#client = client;
-        this.#memory = memory;
         this.#task = task;
         this.trainer = new Trainer(task, client);
         // Simply propagate the training status events emitted by the client
@@ -125,12 +122,6 @@ export class Disco extends EventEmitter {
                 }
                 return await returnedRoundLogs;
             }.bind(this)();
-            await this.#memory.updateWorkingModel({
-                type: "working",
-                taskID: this.#task.id,
-                name: this.#task.trainingInformation.modelID,
-                tensorBackend: this.#task.trainingInformation.tensorBackend,
-            }, this.trainer.model);
         }
         this.#logger.success("Training finished");
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epfml/discojs",
-  "version": "3.0.1-p20240902162912.0",
+  "version": "3.0.1-p20240904094219.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/dist/memory/base.d.ts

@@ -1,111 +0,0 @@
-import type { Model, TaskID } from '../index.js';
-/**
- * Type of models stored in memory. Stored models can either be a model currently
- * being trained ("working model") or a regular model saved in memory ("saved model").
- * There can only be a single working model for a given task.
- */
-type StoredModelType = 'saved' | 'working';
-/**
- * Model information which uniquely identifies a model in memory.
- */
-export interface ModelInfo {
-    type: StoredModelType;
-    version?: number;
-    taskID: TaskID;
-    name: string;
-    tensorBackend: 'gpt' | 'tfjs';
-}
-/**
- * A model source uniquely identifies a model stored in memory.
- * It can be in the form of either a model info object or an ID
- * (one-to-one mapping between the two)
- */
-export type ModelSource = ModelInfo | string;
-/**
- * Represents a model memory system, providing functions to fetch, save, delete and update models.
- * Stored models can either be a model currently being trained ("working model") or a regular model
- * saved in memory ("saved model"). There can only be a single working model for a given task.
- */
-export declare abstract class Memory {
-    /**
-     * Fetches the model identified by the given model source.
-     * @param source The model source
-     * @returns The model
-     */
-    abstract getModel(source: ModelSource): Promise<Model>;
-    /**
-     * Removes the model identified by the given model source from memory.
-     * @param source The model source
-     * @returns The model
-     */
-    abstract deleteModel(source: ModelSource): Promise<void>;
-    /**
-     * Replaces the corresponding working model with the saved model identified by the given model source.
-     * @param source The model source
-     */
-    abstract loadModel(source: ModelSource): Promise<void>;
-    /**
-     * Fetches metadata for the model identified by the given model source.
-     * If the model does not exist in memory, returns undefined.
-     * @param source The model source
-     * @returns The model metadata or undefined
-     */
-    abstract getModelMetadata(source: ModelSource): Promise<object | undefined>;
-    /**
-     * Replaces the working model identified by the given source with the newly provided model.
-     * @param source The model source
-     * @param model The new model
-     */
-    abstract updateWorkingModel(source: ModelSource, model: Model): Promise<void>;
-    /**
-     * Creates a saved model copy from the working model identified by the given model source.
-     * Returns the saved model's path.
-     * @param source The model source
-     * @returns The saved model's path
-     */
-    abstract saveWorkingModel(source: ModelSource): Promise<string | undefined>;
-    /**
-     * Saves the newly provided model to the given model source.
-     * Returns the saved model's path
-     * @param source The model source
-     * @param model The new model
-     * @returns The saved model's path
-     */
-    abstract saveModel(source: ModelSource, model: Model): Promise<string | undefined>;
-    /**
-     * Moves the model identified by the model source to a file system. This is platform-dependent.
-     * @param source The model source
-     */
-    abstract downloadModel(source: ModelSource): Promise<void>;
-    /**
-     * Checks whether the model memory contains the model identified by the given source.
-     * @param source The model source
-     * @returns True if the memory contains the model, false otherwise
-     */
-    abstract contains(source: ModelSource): Promise<boolean>;
-    /**
-     * Computes the path in memory corresponding to the given model source, be it a path or model information.
-     * This is used to easily switch between model path and information, which are both unique model identifiers
-     * with a one-to-one equivalence. Returns undefined instead if no path could be inferred from the given
-     * model source.
-     * @param source The model source
-     * @returns The model path
-     */
-    abstract getModelMemoryPath(source: ModelSource): string | undefined;
-    /**
-     * Computes the model information corresponding to the given model source, be it a path or model information.
-     * This is used to easily switch between model path and information, which are both unique model identifiers
-     * with a one-to-one equivalence. Returns undefined instead if no unique model information could be inferred
-     * from the given model source.
-     * @param source The model source
-     * @returns The model information
-     */
-    abstract getModelInfo(source: ModelSource): ModelInfo | undefined;
-    /**
-     * Computes the lowest version a model source can have without conflicting with model versions currently in memory.
-     * @param source The model source
-     * @returns The duplicated model source
-     */
-    abstract duplicateSource(source: ModelSource): Promise<ModelSource | undefined>;
-}
-export {};

package/dist/memory/base.js

@@ -1,9 +0,0 @@
-// only used browser-side
-// TODO: replace IO type
-/**
- * Represents a model memory system, providing functions to fetch, save, delete and update models.
- * Stored models can either be a model currently being trained ("working model") or a regular model
- * saved in memory ("saved model"). There can only be a single working model for a given task.
- */
-export class Memory {
-}

package/dist/memory/empty.d.ts

@@ -1,20 +0,0 @@
-import type { Model } from '../index.js';
-import type { ModelInfo } from './base.js';
-import { Memory } from './base.js';
-/**
- * Represents an empty model memory.
- */
-export declare class Empty extends Memory {
-    getModelMetadata(): Promise<undefined>;
-    contains(): Promise<boolean>;
-    getModel(): Promise<Model>;
-    loadModel(): Promise<void>;
-    updateWorkingModel(): Promise<void>;
-    saveWorkingModel(): Promise<undefined>;
-    saveModel(): Promise<undefined>;
-    deleteModel(): Promise<void>;
-    downloadModel(): Promise<void>;
-    getModelMemoryPath(): string;
-    getModelInfo(): ModelInfo;
-    duplicateSource(): Promise<undefined>;
-}

package/dist/memory/empty.js

@@ -1,43 +0,0 @@
-import { Memory } from './base.js';
-/**
- * Represents an empty model memory.
- */
-export class Empty extends Memory {
-    getModelMetadata() {
-        return Promise.resolve(undefined);
-    }
-    contains() {
-        return Promise.resolve(false);
-    }
-    getModel() {
-        return Promise.reject(new Error('empty'));
-    }
-    loadModel() {
-        return Promise.reject(new Error('empty'));
-    }
-    updateWorkingModel() {
-        // nothing to do
-        return Promise.resolve();
-    }
-    saveWorkingModel() {
-        return Promise.resolve(undefined);
-    }
-    saveModel() {
-        return Promise.resolve(undefined);
-    }
-    async deleteModel() {
-        // nothing to do
-    }
-    downloadModel() {
-        return Promise.reject(new Error('empty'));
-    }
-    getModelMemoryPath() {
-        throw new Error('empty');
-    }
-    getModelInfo() {
-        throw new Error('empty');
-    }
-    duplicateSource() {
-        return Promise.resolve(undefined);
-    }
-}

package/dist/memory/index.d.ts

@@ -1,2 +0,0 @@
-export { Empty } from './empty.js';
-export { Memory, type ModelInfo, type ModelSource } from './base.js';

package/dist/memory/index.js

@@ -1,2 +0,0 @@
-export { Empty } from './empty.js';
-export { Memory } from './base.js';