@epfml/discojs 3.0.1-p20250729132444.0 → 3.0.1-p20250924113522.0

package/dist/aggregator/get.d.ts

@@ -1,7 +1,7 @@
-import type { DataType, Task } from '../index.js';
+import type { DataType, Network, Task } from '../index.js';
 import { aggregator } from '../index.js';
 type AggregatorOptions = Partial<{
-    scheme: Task<DataType>["trainingInformation"]["scheme"];
+    scheme: Task<DataType, Network>["trainingInformation"]["scheme"];
     roundCutOff: number;
     threshold: number;
     thresholdType: 'relative' | 'absolute';
@@ -24,5 +24,5 @@ type AggregatorOptions = Partial<{
  * @param options Options passed down to the aggregator's constructor
  * @returns The aggregator
  */
-export declare function getAggregator(task: Task<DataType>, options?: AggregatorOptions): aggregator.Aggregator;
+export declare function getAggregator(task: Task<DataType, Network>, options?: AggregatorOptions): aggregator.Aggregator;
 export {};

package/dist/aggregator/get.js

@@ -18,9 +18,8 @@ import { aggregator } from '../index.js';
  * @returns The aggregator
  */
 export function getAggregator(task, options = {}) {
-    const aggregationStrategy = task.trainingInformation.aggregationStrategy ?? 'mean';
     const scheme = options.scheme ?? task.trainingInformation.scheme;
-    switch (aggregationStrategy) {
+    switch (task.trainingInformation.aggregationStrategy) {
         case 'mean':
             if (scheme === 'decentralized') {
                 // If options are not specified, we default to expecting a contribution from all peers, so we set the threshold to 100%

package/dist/client/client.d.ts

@@ -1,4 +1,4 @@
-import type { DataType, Model, RoundStatus, Task, WeightsContainer } from "../index.js";
+import type { DataType, Model, Network, RoundStatus, Task, WeightsContainer } from "../index.js";
 import type { NodeID } from './types.js';
 import type { EventConnection } from './event_connection.js';
 import type { Aggregator } from '../aggregator/index.js';
@@ -7,13 +7,13 @@ import { EventEmitter } from '../utils/event_emitter.js';
  * Main, abstract, class representing a Disco client in a network, which handles
  * communication with other nodes, be it peers or a server.
  */
-export declare abstract class Client extends EventEmitter<{
-    'status': RoundStatus;
-    'participants': number;
+export declare abstract class Client<N extends Network> extends EventEmitter<{
+    status: RoundStatus;
+    participants: number;
 }> {
     #private;
     readonly url: URL;
-    readonly task: Task<DataType>;
+    readonly task: Task<DataType, N>;
     readonly aggregator: Aggregator;
     protected _ownId?: NodeID;
     protected _server?: EventConnection;
@@ -25,7 +25,7 @@ export declare abstract class Client extends EventEmitter<{
      */
     protected promiseForMoreParticipants: Promise<void> | undefined;
     constructor(url: URL, // The network server's URL to connect to
-    task: Task<DataType>, // The client's corresponding task
+    task: Task<DataType, N>, // The client's corresponding task
     aggregator: Aggregator);
     /**
      * Communication callback called at the beginning of every training round.

package/dist/client/decentralized/decentralized_client.d.ts

@@ -6,7 +6,7 @@ import { Client } from '../client.js';
  * with the server is based off regular WebSockets, whereas peer-to-peer communication uses
  * WebRTC for Node.js.
  */
-export declare class DecentralizedClient extends Client {
+export declare class DecentralizedClient extends Client<"decentralized"> {
     #private;
     private get isDisconnected();
     private setAggregatorNodes;

package/dist/client/decentralized/peer_pool.d.ts

@@ -1,6 +1,6 @@
 import { Map, type Set } from 'immutable';
 import { type SignalData } from './peer.js';
-import { type NodeID } from '../types.js';
+import type { NodeID } from '../types.js';
 import { PeerConnection, type EventConnection } from '../event_connection.js';
 export declare class PeerPool {
     private readonly id;

package/dist/client/federated/federated_client.d.ts

@@ -4,7 +4,7 @@ import { Client } from "../client.js";
  * Client class that communicates with a centralized, federated server, when training
  * a specific task in the federated setting.
  */
-export declare class FederatedClient extends Client {
+export declare class FederatedClient extends Client<"federated"> {
     /**
      * Initializes the connection to the server, gets our node ID
      * as well as the latest training information: latest global model, current round and

package/dist/client/local_client.d.ts

@@ -4,7 +4,7 @@ import { Client } from "./client.js";
  * A LocalClient represents a Disco user training only on their local data without collaborating
  * with anyone. Thus LocalClient doesn't do anything during communication
  */
-export declare class LocalClient extends Client {
+export declare class LocalClient extends Client<"local"> {
     onRoundBeginCommunication(): Promise<void>;
     onRoundEndCommunication(weights: WeightsContainer): Promise<WeightsContainer>;
 }

package/dist/client/utils.d.ts

@@ -1,4 +1,4 @@
-import type { DataType, Task } from '../index.js';
+import type { DataType, Network, Task } from "../index.js";
 import { client as clients, type aggregator } from '../index.js';
 export declare function timeout(ms?: number, errorMsg?: string): Promise<never>;
-export declare function getClient(trainingScheme: Task<DataType>["trainingInformation"]["scheme"], serverURL: URL, task: Task<DataType>, aggregator: aggregator.Aggregator): clients.Client;
+export declare function getClient<D extends DataType, N extends Network>(scheme: N | "local", serverURL: URL, task: Task<D, N>, aggregator: aggregator.Aggregator): clients.Client<N>;

package/dist/client/utils.js

@@ -6,17 +6,26 @@ export async function timeout(ms = MAX_WAIT_PER_ROUND, errorMsg = 'timeout') {
         setTimeout(() => { reject(new Error(errorMsg)); }, ms);
     });
 }
-export function getClient(trainingScheme, serverURL, task, aggregator) {
-    switch (trainingScheme) {
-        case 'decentralized':
-            return new clients.decentralized.DecentralizedClient(serverURL, task, aggregator);
-        case 'federated':
-            return new clients.federated.FederatedClient(serverURL, task, aggregator);
-        case 'local':
-            return new clients.LocalClient(serverURL, task, aggregator);
+export function getClient(scheme, serverURL, task, aggregator) {
+    switch (scheme) {
+        case "decentralized": {
+            const t = task;
+            t.trainingInformation.scheme = scheme;
+            return new clients.decentralized.DecentralizedClient(serverURL, t, aggregator);
+        }
+        case "federated": {
+            const t = task;
+            t.trainingInformation.scheme = scheme;
+            return new clients.federated.FederatedClient(serverURL, t, aggregator);
+        }
+        case "local": {
+            const t = task;
+            t.trainingInformation.scheme = scheme;
+            return new clients.LocalClient(serverURL, t, aggregator);
+        }
         default: {
-            const _ = trainingScheme;
-            throw new Error('should never happen');
+            const _ = scheme;
+            throw new Error("should never happen");
         }
     }
 }

package/dist/default_tasks/cifar10.d.ts

@@ -1,2 +1,2 @@
-import type { TaskProvider } from '../index.js';
-export declare const cifar10: TaskProvider<'image'>;
+import type { TaskProvider } from "../index.js";
+export declare const cifar10: TaskProvider<"image", "decentralized">;

package/dist/default_tasks/cifar10.js

@@ -3,27 +3,28 @@ import { models } from '../index.js';
 import baseModel from '../models/mobileNet_v1_025_224.js';
 export const cifar10 = {
     getTask() {
-        return {
+        return Promise.resolve({
             id: 'cifar10',
+            dataType: "image",
             displayInformation: {
-                taskTitle: 'CIFAR10',
+                title: 'CIFAR10',
                 summary: {
                     preview: 'CIFAR-10 is a classic image classification task, and one of the most widely used datasets for machine learning research.',
                     overview: "The dataset contains 60,000 32x32 color images in 10 different classes: airplanes, cars, birds, cats, deer, dogs, frogs, horses, ships, and trucks. The official CIFAR-10 website can be found <a class='underline text-blue-400' href='https://www.cs.toronto.edu/~kriz/cifar.html' target='_blank'>here</a>. You can find a link to a sample dataset at the next step (Connect Your Data)."
                 },
                 model: 'The model is a pretrained <a  class="underline text-blue-400" target="_blank" href="https://github.com/tensorflow/tfjs-models/tree/master/mobilenet">MobileNetV1 model</a> trained in Tensorflow.js. The last output layer is replaced with a fully connected layer with softmax activation and one output neuron per CIFAR10 category. The data preprocessing reshapes images into 224x224 pixels and normalizes values between 0 and 1. The neural network is optimized via Stochastic Gradient Descent and a categorical Cross Entropy loss.',
                 dataFormatInformation: 'Images should be of .png format and of size 32x32. <br> The CSV file should start with the exact header "filename,label", and each row should contain an image filename (without extension) and its label.<br><br> For example if you have images: 0.png (of a frog) and 1.png (of a car) <br> The CSV file should be: <br>filename, label <br><br> 0, frog <br> 1, car',
-                dataExampleText: 'Below you can find 10 random examples from each of the 10 classes in the dataset.',
-                dataExampleImage: 'https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/cifar10-example.png',
-                sampleDatasetLink: 'https://storage.googleapis.com/deai-313515.appspot.com/example_training_data.tar.gz',
-                sampleDatasetInstructions: 'Opening the link should start downloading a zip file which you can unzip. To connect the data, use the CSV option below and select the file named "cifar10-labels.csv". You can now connect the images located in the "CIFAR10" folder. Note that there are only 24 images in this sample dataset which is far too few to successfully train a machine learning model.'
+                dataExample: "https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/cifar10-example.png",
+                sampleDataset: {
+                    link: "https://storage.googleapis.com/deai-313515.appspot.com/example_training_data.tar.gz",
+                    instructions: 'Opening the link should start downloading a zip file which you can unzip. To connect the data, use the CSV option below and select the file named "cifar10-labels.csv". You can now connect the images located in the "CIFAR10" folder. Note that there are only 24 images in this sample dataset which is far too few to successfully train a machine learning model.',
+                },
             },
             trainingInformation: {
                 epochs: 10,
                 roundDuration: 10,
                 validationSplit: 0.2,
                 batchSize: 10,
-                dataType: 'image',
                 IMAGE_H: 224,
                 IMAGE_W: 224,
                 LABEL_LIST: ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'],
@@ -34,7 +35,7 @@ export const cifar10 = {
                 maxShareValue: 100,
                 tensorBackend: 'tfjs'
             }
-        };
+        });
     },
     async getModel() {
         const mobilenet = await tf.loadLayersModel({

package/dist/default_tasks/lus_covid.d.ts

@@ -1,2 +1,2 @@
-import type { TaskProvider } from '../index.js';
-export declare const lusCovid: TaskProvider<'image'>;
+import type { TaskProvider } from "../index.js";
+export declare const lusCovid: TaskProvider<"image", "federated">;

package/dist/default_tasks/lus_covid.js

@@ -2,20 +2,22 @@ import * as tf from '@tensorflow/tfjs';
 import { models } from '../index.js';
 export const lusCovid = {
     getTask() {
-        return {
+        return Promise.resolve({
             id: 'lus_covid',
+            dataType: "image",
             displayInformation: {
-                taskTitle: 'Lung Ultrasound Image Classification',
+                title: 'Lung Ultrasound Image Classification',
                 summary: {
                     preview: "Medical images are a typical example of data that exists in huge quantity yet that can't be shared due to confidentiality reasons. Medical applications would immensely benefit from training on data currently locked. More data diversity leads to better generalization and bias mitigation.",
                     overview: "Disco allows data owners to collaboratively train machine learning models using their respective data without any privacy breach. This example problem is about diagnosing whether patients are positive or negative to COVID-19 from lung ultrasounds images. <br>Don't have a dataset of your own? You can find a link to a sample dataset at the next step."
                 },
                 model: "The model is a simple Convolutional Neural Network composed of two convolutional layers with ReLU activations and max pooling layers, followed by a fully connected output layer. The data preprocessing reshapes images into 100x100 pixels and normalizes values between 0 and 1",
                 dataFormatInformation: 'This model takes as input an image dataset of lung ultrasounds. The images are resized automatically.',
-                dataExampleText: 'Below you can find an example of an expected lung image.',
-                dataExampleImage: 'https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/2_QAID_1.masked.reshaped.squared.224.png',
-                sampleDatasetLink: 'https://drive.switch.ch/index.php/s/zM5ZrUWK3taaIly',
-                sampleDatasetInstructions: 'Opening the link will take you to a Switch Drive folder. You can click on the Download button in the top right corner. Unzip the file and you will get two subfolders: "COVID-" and "COVID+". You can connect the data by using the Group option and selecting each image group in its respective field.'
+                dataExample: "https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/2_QAID_1.masked.reshaped.squared.224.png",
+                sampleDataset: {
+                    link: "https://drive.switch.ch/index.php/s/zM5ZrUWK3taaIly",
+                    instructions: 'Opening the link will take you to a Switch Drive folder. You can click on the Download button in the top right corner. Unzip the file and you will get two subfolders: "COVID-" and "COVID+". You can connect the data by using the Group option and selecting each image group in its respective field.',
+                },
             },
             trainingInformation: {
                 epochs: 50,
@@ -25,13 +27,12 @@ export const lusCovid = {
                 IMAGE_H: 100,
                 IMAGE_W: 100,
                 LABEL_LIST: ['COVID-Positive', 'COVID-Negative'],
-                dataType: 'image',
                 scheme: 'federated',
                 aggregationStrategy: 'mean',
                 minNbOfParticipants: 2,
                 tensorBackend: 'tfjs'
             }
-        };
+        });
     },
     // Model architecture from tensorflow.js docs: 
     // https://codelabs.developers.google.com/codelabs/tfjs-training-classfication/index.html#4

package/dist/default_tasks/mnist.d.ts

@@ -1,2 +1,2 @@
-import type { TaskProvider } from '../index.js';
-export declare const mnist: TaskProvider<'image'>;
+import type { TaskProvider } from "../index.js";
+export declare const mnist: TaskProvider<"image", "decentralized">;

package/dist/default_tasks/mnist.js

@@ -2,27 +2,28 @@ import * as tf from '@tensorflow/tfjs';
 import { models } from '../index.js';
 export const mnist = {
     getTask() {
-        return {
+        return Promise.resolve({
             id: 'mnist',
+            dataType: "image",
             displayInformation: {
-                taskTitle: 'Handwritten Digit Recognition',
+                title: 'Handwritten Digit Recognition',
                 summary: {
                     preview: "The MNIST handwritten digit classification problem is a classic dataset used in computer vision and deep learning. The objective is to classify handwritten digits from 28x28 pixel images.",
                     overview: "Download the classic MNIST dataset of hand-written numbers <a class='underline text-blue-400' target='_blank' href='https://www.kaggle.com/scolianni/mnistasjpg'>here</a>. You can also find a sample dataset at the next step."
                 },
                 model: "The model is a simple Convolutional Neural Network composed of three convolutional layers with ReLU activations and max pooling layers, followed by two fully connected layers. The data preprocessing simply normalizes values between 0 and 1. The neural network is optimized via RMSProp and a categorical cross-entropy loss.",
                 dataFormatInformation: 'This model is trained on images corresponding to digits 0 to 9. You can connect your own images of each digit in the box corresponding to its label. The model takes images of size 28x28 as input.',
-                dataExampleText: 'Below you can find an example of an expected image representing the digit 9.',
-                dataExampleImage: 'http://storage.googleapis.com/deai-313515.appspot.com/example_training_data/9-mnist-example.png',
-                sampleDatasetLink: 'https://storage.googleapis.com/deai-313515.appspot.com/MNIST_samples.tar.gz',
-                sampleDatasetInstructions: 'Opening the link should start downloading a zip file which you can unzip. You can connect the data with the CSV option below using the CSV file named "mnist_labels.csv". After selecting in the CSV file, you will be able to connect the data under in the "images" folder.'
+                dataExample: "http://storage.googleapis.com/deai-313515.appspot.com/example_training_data/9-mnist-example.png",
+                sampleDataset: {
+                    link: "https://storage.googleapis.com/deai-313515.appspot.com/MNIST_samples.tar.gz",
+                    instructions: 'Opening the link should start downloading a zip file which you can unzip. You can connect the data with the CSV option below using the CSV file named "mnist_labels.csv". After selecting in the CSV file, you will be able to connect the data under in the "images" folder.',
+                },
             },
             trainingInformation: {
                 epochs: 20,
                 roundDuration: 2,
                 validationSplit: 0.2,
                 batchSize: 64,
-                dataType: 'image',
                 IMAGE_H: 28,
                 IMAGE_W: 28,
                 LABEL_LIST: ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'],
@@ -32,7 +33,7 @@ export const mnist = {
                 maxShareValue: 100,
                 tensorBackend: 'tfjs'
             }
-        };
+        });
     },
     getModel() {
         // Architecture from the PyTorch MNIST example (I made it slightly smaller, 650kB instead of 5MB)

package/dist/default_tasks/simple_face.d.ts

@@ -1,2 +1,2 @@
-import type { TaskProvider } from '../index.js';
-export declare const simpleFace: TaskProvider<'image'>;
+import type { TaskProvider } from "../index.js";
+export declare const simpleFace: TaskProvider<"image", "federated">;

package/dist/default_tasks/simple_face.js

@@ -3,26 +3,27 @@ import { models } from '../index.js';
 import baseModel from '../models/mobileNetV2_35_alpha_2_classes.js';
 export const simpleFace = {
     getTask() {
-        return {
+        return Promise.resolve({
             id: 'simple_face',
+            dataType: "image",
             displayInformation: {
-                taskTitle: 'Simple Face',
+                title: 'Simple Face',
                 summary: {
                     preview: 'Can you detect if the person in a picture is a child or an adult?',
                     overview: 'Simple face is a small subset of the public face_task dataset from Kaggle'
                 },
                 dataFormatInformation: '',
-                dataExampleText: 'Below you can find an example',
-                dataExampleImage: 'https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/simple_face-example.png',
-                sampleDatasetLink: "https://storage.googleapis.com/deai-313515.appspot.com/example_training_data.tar.gz",
-                sampleDatasetInstructions: 'Opening the link should start downloading a zip file which you can unzip. Inside the "example_training_data" directory you should find the "simple_face" folder which contains the "adult" and "child" folders. To connect the data, select the Group option below and connect adults and children image groups.'
+                dataExample: "https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/simple_face-example.png",
+                sampleDataset: {
+                    link: "https://storage.googleapis.com/deai-313515.appspot.com/example_training_data.tar.gz",
+                    instructions: 'Opening the link should start downloading a zip file which you can unzip. Inside the "example_training_data" directory you should find the "simple_face" folder which contains the "adult" and "child" folders. To connect the data, select the Group option below and connect adults and children image groups.',
+                },
             },
             trainingInformation: {
                 epochs: 50,
                 roundDuration: 1,
                 validationSplit: 0.2,
                 batchSize: 10,
-                dataType: 'image',
                 IMAGE_H: 200,
                 IMAGE_W: 200,
                 LABEL_LIST: ['child', 'adult'],
@@ -31,7 +32,7 @@ export const simpleFace = {
                 minNbOfParticipants: 2,
                 tensorBackend: 'tfjs'
             }
-        };
+        });
     },
     async getModel() {
         const model = await tf.loadLayersModel({

package/dist/default_tasks/tinder_dog.d.ts

@@ -1,2 +1,2 @@
 import type { TaskProvider } from '../index.js';
-export declare const tinderDog: TaskProvider<'image'>;
+export declare const tinderDog: TaskProvider<"image", "federated">;

package/dist/default_tasks/tinder_dog.js

@@ -2,27 +2,28 @@ import * as tf from '@tensorflow/tfjs';
 import { models } from '../index.js';
 export const tinderDog = {
     getTask() {
-        return {
+        return Promise.resolve({
             id: 'tinder_dog',
+            dataType: "image",
             displayInformation: {
-                taskTitle: 'GDHF 2024 | TinderDog',
+                title: "GDHF 2024 | TinderDog",
                 summary: {
                     preview: 'Which dog is the cutest....or not?',
                     overview: "Binary classification model for dog cuteness."
                 },
                 model: 'The model is a simple Convolutional Neural Network composed of two convolutional layers with ReLU activations and max pooling layers, followed by a fully connected output layer. The data preprocessing reshapes images into 64x64 pixels and normalizes values between 0 and 1',
                 dataFormatInformation: 'Accepted image formats are .png .jpg and .jpeg.',
-                dataExampleText: '',
-                dataExampleImage: 'https://storage.googleapis.com/deai-313515.appspot.com/tinder_dog_preview.png',
-                sampleDatasetLink: 'https://storage.googleapis.com/deai-313515.appspot.com/tinder_dog.zip',
-                sampleDatasetInstructions: 'Opening the link should start downloading a zip file which you can unzip. To connect the data, pick one of the data splits (the folder 0 for example) and use the CSV option below to select the file named "labels.csv". You can now connect the images located in the same folder.'
+                dataExample: "https://storage.googleapis.com/deai-313515.appspot.com/tinder_dog_preview.png",
+                sampleDataset: {
+                    link: "https://storage.googleapis.com/deai-313515.appspot.com/tinder_dog.zip",
+                    instructions: 'Opening the link should start downloading a zip file which you can unzip. To connect the data, pick one of the data splits (the folder 0 for example) and use the CSV option below to select the file named "labels.csv". You can now connect the images located in the same folder.',
+                },
             },
             trainingInformation: {
                 epochs: 10,
                 roundDuration: 2,
                 validationSplit: 0, // nicer plot for GDHF demo
                 batchSize: 10,
-                dataType: 'image',
                 IMAGE_H: 64,
                 IMAGE_W: 64,
                 LABEL_LIST: ['Cute dogs', 'Less cute dogs'],
@@ -31,12 +32,13 @@ export const tinderDog = {
                 minNbOfParticipants: 3,
                 tensorBackend: 'tfjs'
             }
-        };
+        });
     },
     async getModel() {
+        const task = await this.getTask();
         const seed = 42; // set a seed to ensure reproducibility during GDHF demo
-        const imageHeight = this.getTask().trainingInformation.IMAGE_H;
-        const imageWidth = this.getTask().trainingInformation.IMAGE_W;
+        const imageHeight = task.trainingInformation.IMAGE_H;
+        const imageWidth = task.trainingInformation.IMAGE_W;
         const imageChannels = 3;
         const model = tf.sequential();
         model.add(tf.layers.conv2d({

package/dist/default_tasks/titanic.d.ts

@@ -1,2 +1,2 @@
-import type { TaskProvider } from '../index.js';
-export declare const titanic: TaskProvider<'tabular'>;
+import type { TaskProvider } from "../index.js";
+export declare const titanic: TaskProvider<"tabular", "federated">;

package/dist/default_tasks/titanic.js

@@ -2,54 +2,41 @@ import * as tf from '@tensorflow/tfjs';
 import { models } from '../index.js';
 export const titanic = {
     getTask() {
-        return {
+        return Promise.resolve({
             id: 'titanic',
+            dataType: "tabular",
             displayInformation: {
-                taskTitle: 'Titanic Prediction',
+                title: 'Titanic Prediction',
                 summary: {
                     preview: "The Titanic classification task is one of the main entrypoints into machine learning. Using passenger data (name, age, gender, socio-economic class, etc), the goal is to identify who was more likely to survive the infamous shipwreck.",
                     overview: "The original competition can be found on <a  target='_blank' class='underline text-blue-400' href='https://www.kaggle.com/c/titanic'>Kaggle</a> and a link to the training set can be found here <a target='_blank' class='underline text-blue-400' href='https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/titanic_train.csv'>here</a>."
                 },
                 model: 'The model is a simple 5-layer feedforward network with ReLU activations. The model is optimized with Adam and binary cross-entropy loss. The preprocessing only fills missing value with a placeholder value (0).',
                 dataFormatInformation: 'The expected format for the tabular dataset is exactly the same as the sample data provided above or in the Kaggle competition. It is a CSV file with 12 columns. The features are general information about the passenger (sex, age, name, etc.) and specific related Titanic data such as the ticket class bought by the passenger, its cabin number, etc.<br>The first line of the CSV contains the header: "PassengerId, Survived, Pclass, Name, Sex, Age, SibSp, Parch, Ticket, Fare, Cabin, Embarked"<br>Each subsequent row contains passenger data.',
-                dataExampleText: "Here's an example of one data point:",
                 dataExample: [
-                    { columnName: 'PassengerId', columnData: '1' },
-                    { columnName: 'Survived', columnData: '0' },
-                    { columnName: 'Name', columnData: 'Braund, Mr. Owen Harris' },
-                    { columnName: 'Sex', columnData: 'male' },
-                    { columnName: 'Age', columnData: '22' },
-                    { columnName: 'SibSp', columnData: '1' },
-                    { columnName: 'Parch', columnData: '0' },
-                    { columnName: 'Ticket', columnData: '1/5 21171' },
-                    { columnName: 'Fare', columnData: '7.25' },
-                    { columnName: 'Cabin', columnData: 'E46' },
-                    { columnName: 'Embarked', columnData: 'S' },
-                    { columnName: 'Pclass', columnData: '3' }
+                    { name: "PassengerId", data: "1" },
+                    { name: "Survived", data: "0" },
+                    { name: "Name", data: "Braund, Mr. Owen Harris" },
+                    { name: "Sex", data: "male" },
+                    { name: "Age", data: "22" },
+                    { name: "SibSp", data: "1" },
+                    { name: "Parch", data: "0" },
+                    { name: "Ticket", data: "1/5 21171" },
+                    { name: "Fare", data: "7.25" },
+                    { name: "Cabin", data: "E46" },
+                    { name: "Embarked", data: "S" },
+                    { name: "Pclass", data: "3" },
                 ],
-                headers: [
-                    'PassengerId',
-                    'Survived',
-                    'Name',
-                    'Sex',
-                    'Age',
-                    'SibSp',
-                    'Parch',
-                    'Ticket',
-                    'Fare',
-                    'Cabin',
-                    'Embarked',
-                    'Pclass'
-                ],
-                sampleDatasetLink: "https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/titanic_train.csv",
-                sampleDatasetInstructions: 'Opening the link should start downloading a CSV file which you can drag and drop in the field below.'
+                sampleDataset: {
+                    link: "https://storage.googleapis.com/deai-313515.appspot.com/example_training_data/titanic_train.csv",
+                    instructions: "Opening the link should start downloading a CSV file which you can drag and drop in the field below.",
+                },
             },
             trainingInformation: {
                 epochs: 10,
                 roundDuration: 2,
                 validationSplit: 0.2,
                 batchSize: 30,
-                dataType: 'tabular',
                 inputColumns: [
                     'Age',
                     'SibSp',
@@ -63,7 +50,7 @@ export const titanic = {
                 minNbOfParticipants: 2,
                 tensorBackend: 'tfjs'
             }
-        };
+        });
     },
     getModel() {
         const model = tf.sequential();

package/dist/default_tasks/wikitext.d.ts

@@ -1,2 +1,2 @@
-import type { TaskProvider } from '../index.js';
-export declare const wikitext: TaskProvider<'text'>;
+import type { TaskProvider } from "../index.js";
+export declare const wikitext: TaskProvider<"text", "federated">;

package/dist/default_tasks/wikitext.js

@@ -1,10 +1,11 @@
-import { models } from '../index.js';
+import { Tokenizer, models } from "../index.js";
 export const wikitext = {
-    getTask() {
+    async getTask() {
         return {
             id: 'llm_task',
+            dataType: "text",
             displayInformation: {
-                taskTitle: 'GPT Language Modeling',
+                title: "GPT Language Modeling",
                 summary: {
                     preview: 'Train a language model (L)LM in your browser, collaboratively and from scratch.',
                     overview: "You can train a GPT-2 model in your browser and in a collaborative manner on any textual dataset. As an example, you can try the Wikitext-103 dataset, composed of Wikipedia articles, widely used in natural language modeling, which you can download <a class='underline text-blue-400' target='_blank' href='https://dax-cdn.cdn.appdomain.cloud/dax-wikitext-103/1.0.1/wikitext-103.tar.gz'>here</a>. More information on how to connect the dataset at the next step."
@@ -17,12 +18,13 @@ export const wikitext = {
                     "To accommodate all devices, the context length is currently kept at 128 and the batch size at 1.",
                 ].join(" "),
                 dataFormatInformation: 'You can use any natural language (text) dataset you like. For example the Wikitext-103 dataset is organized as a large text file, with each line representing a segment of raw text from Wikipedia articles.',
-                dataExampleText: 'An example excerpt from the dataset is: <i>"For the first twenty years of its existence , the only staged performances of Parsifal took place in the Bayreuth Festspielhaus , the venue for which Wagner conceived the work ( except eight private performances for Ludwig II at Munich in 1884 and 1885 ) ."</i>',
-                sampleDatasetLink: 'https://storage.googleapis.com/deai-313515.appspot.com/wikitext.zip',
-                sampleDatasetInstructions: 'Opening the link should start downloading a zip file. Unzip it and drag and drop the training set named "wiki.train.tokens" in the field below (or use the "Select File" button). Even though the file extension is ".tokens" it is indeed a text file. You can use "wiki.test.tokens" at the evaluation step after training a language model.'
+                dataExample: "For the first twenty years of its existence , the only staged performances of Parsifal took place in the Bayreuth Festspielhaus , the venue for which Wagner conceived the work ( except eight private performances for Ludwig II at Munich in 1884 and 1885 ) .",
+                sampleDataset: {
+                    link: "https://storage.googleapis.com/deai-313515.appspot.com/wikitext.zip",
+                    instructions: 'Opening the link should start downloading a zip file. Unzip it and drag and drop the training set named "wiki.train.tokens" in the field below (or use the "Select File" button). Even though the file extension is ".tokens" it is indeed a text file. You can use "wiki.test.tokens" at the evaluation step after training a language model.',
+                },
             },
             trainingInformation: {
-                dataType: 'text',
                 scheme: 'federated',
                 aggregationStrategy: 'mean',
                 minNbOfParticipants: 2,
@@ -32,15 +34,16 @@ export const wikitext = {
                 validationSplit: 0.1,
                 roundDuration: 2,
                 batchSize: 8, // If set too high firefox raises a WebGL error
-                tokenizer: 'Xenova/gpt2',
+                tokenizer: await Tokenizer.from_pretrained("Xenova/gpt2"),
                 contextLength: 64,
                 tensorBackend: 'gpt'
             }
         };
     },
-    getModel() {
-        return Promise.resolve(new models.GPT({
-            contextLength: this.getTask().trainingInformation.contextLength,
-        }));
-    }
+    async getModel() {
+        const task = await this.getTask();
+        return new models.GPT({
+            contextLength: task.trainingInformation.contextLength,
+        });
+    },
 };

package/dist/index.d.ts

@@ -8,7 +8,7 @@ export { WeightsContainer, aggregation } from './weights/index.js';
 export { Logger, ConsoleLogger } from './logging/index.js';
 export { Disco, RoundLogs, RoundStatus } from './training/index.js';
 export { Validator } from './validator.js';
-export { Model, BatchLogs, EpochLogs, ValidationMetrics } from './models/index.js';
+export { Model, BatchLogs, EpochLogs, Tokenizer, ValidationMetrics, } from "./models/index.js";
 export * as models from './models/index.js';
 export * from './task/index.js';
 export * as defaultTasks from './default_tasks/index.js';

package/dist/index.js

@@ -8,7 +8,7 @@ export { WeightsContainer, aggregation } from './weights/index.js';
 export { ConsoleLogger } from './logging/index.js';
 export { Disco } from './training/index.js';
 export { Validator } from './validator.js';
-export { Model, EpochLogs } from './models/index.js';
+export { Model, EpochLogs, Tokenizer, } from "./models/index.js";
 export * as models from './models/index.js';
 export * from './task/index.js';
 export * as defaultTasks from './default_tasks/index.js';

package/dist/models/gpt/config.d.ts

@@ -1,5 +1,5 @@
 type GPTModelType = 'gpt2' | 'gpt2-medium' | 'gpt2-large' | 'gpt2-xl' | 'gpt-mini' | 'gpt-micro' | 'gpt-nano';
-export interface GPTConfig {
+export type GPTConfig = {
     lr: number;
     contextLength: number;
     vocabSize?: number;
@@ -19,7 +19,7 @@ export interface GPTConfig {
     nHead?: number;
     nEmbd?: number;
     seed?: number;
-}
+};
 export declare const DefaultGPTConfig: Required<GPTConfig>;
 export type ModelSize = {
     nLayer: number;