@epfml/discojs 2.1.1 → 2.1.2-p20240506085037.0

package/dist/dataset/data_loader/image_loader.js

@@ -1,26 +1,101 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebImageLoader = void 0;
-var tslib_1 = require("tslib");
-var __1 = require("../..");
-var WebImageLoader = /** @class */ (function (_super) {
-    (0, tslib_1.__extends)(WebImageLoader, _super);
-    function WebImageLoader() {
-        return _super !== null && _super.apply(this, arguments) || this;
+import { Range } from 'immutable';
+import * as tf from '@tensorflow/tfjs';
+import { ImageData } from '../data/index.js';
+import { DataLoader } from '../data_loader/index.js';
+/**
+ * Image data loader whose instantiable implementation is delegated by the platform-dependent Disco subprojects, namely,
+ * @epfml/discojs-web and @epfml/discojs-node.
+ * Load labels and correctly match them with their respective images, with the following constraints:
+ * 1. Images are given as 1 image/1 file;
+ * 2. Labels are given as multiple labels/1 file, each label file can contain a different amount of labels.
+ */
+export class ImageLoader extends DataLoader {
+    task;
+    constructor(task) {
+        super();
+        this.task = task;
     }
-    WebImageLoader.prototype.readImageFrom = function (source) {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var _a, _b;
-            return (0, tslib_1.__generator)(this, function (_c) {
-                switch (_c.label) {
-                    case 0:
-                        _b = (_a = __1.tf.browser).fromPixels;
-                        return [4 /*yield*/, createImageBitmap(source)];
-                    case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])];
+    async load(image, config) {
+        let tensorContainer;
+        if (config?.labels === undefined) {
+            tensorContainer = await this.readImageFrom(image, config?.channels);
+        }
+        else {
+            tensorContainer = {
+                xs: await this.readImageFrom(image, config?.channels),
+                ys: config.labels[0]
+            };
+        }
+        return tf.data.array([tensorContainer]);
+    }
+    async buildDataset(images, labels, indices, config) {
+        // Can't use arrow function for generator and need access to 'this'
+        // eslint-disable-next-line
+        const self = this;
+        async function* dataGenerator() {
+            const withLabels = config?.labels !== undefined;
+            let index = 0;
+            while (index < indices.length) {
+                const sample = await self.readImageFrom(images[indices[index]], config?.channels);
+                const label = withLabels ? labels[indices[index]] : undefined;
+                const value = withLabels ? { xs: sample, ys: label } : sample;
+                index++;
+                yield value;
+            }
+        }
+        // @ts-expect-error: For some reasons typescript refuses async generator but tensorflow do work with them
+        const dataset = tf.data.generator(dataGenerator);
+        return await ImageData.init(dataset, this.task, indices.length);
+    }
+    async loadAll(images, config) {
+        let labels = [];
+        const indices = Range(0, images.length).toArray();
+        if (config?.labels !== undefined) {
+            const labelList = this.task.trainingInformation?.LABEL_LIST;
+            if (labelList === undefined || !Array.isArray(labelList)) {
+                throw new Error('LABEL_LIST should be specified in the task training information');
+            }
+            const numberOfClasses = labelList.length;
+            // Map label strings to integer
+            const label_to_int = new Map(labelList.map((label_name, idx) => [label_name, idx]));
+            if (label_to_int.size != numberOfClasses) {
+                throw new Error("Input labels aren't matching the task LABEL_LIST");
+            }
+            labels = config.labels.map(label_name => {
+                const label_int = label_to_int.get(label_name);
+                if (label_int === undefined) {
+                    throw new Error(`Found input label ${label_name} not specified in task LABEL_LIST`);
                 }
+                return label_int;
             });
-        });
-    };
-    return WebImageLoader;
-}(__1.data.ImageLoader));
-exports.WebImageLoader = WebImageLoader;
+            labels = await tf.oneHot(tf.tensor1d(labels, 'int32'), numberOfClasses).array();
+        }
+        if (config?.shuffle === undefined || config?.shuffle) {
+            this.shuffle(indices);
+        }
+        if (config?.validationSplit === undefined || config?.validationSplit === 0) {
+            const dataset = await this.buildDataset(images, labels, indices, config);
+            return {
+                train: dataset,
+                validation: undefined
+            };
+        }
+        const trainSize = Math.floor(images.length * (1 - config.validationSplit));
+        const trainIndices = indices.slice(0, trainSize);
+        const valIndices = indices.slice(trainSize);
+        const trainDataset = await this.buildDataset(images, labels, trainIndices, config);
+        const valDataset = await this.buildDataset(images, labels, valIndices, config);
+        return {
+            train: trainDataset,
+            validation: valDataset
+        };
+    }
+    shuffle(array) {
+        for (let i = 0; i < array.length; i++) {
+            const j = Math.floor(Math.random() * i);
+            const swap = array[i];
+            array[i] = array[j];
+            array[j] = swap;
+        }
+    }
+}

package/dist/dataset/data_loader/index.d.ts

@@ -1,2 +1,5 @@
-export { WebImageLoader } from './image_loader';
-export { WebTabularLoader } from './tabular_loader';
+export type { DataConfig } from './data_loader.js';
+export { DataLoader } from './data_loader.js';
+export { ImageLoader } from './image_loader.js';
+export { TabularLoader } from './tabular_loader.js';
+export { TextLoader } from './text_loader.js';

package/dist/dataset/data_loader/index.js

@@ -1,7 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebTabularLoader = exports.WebImageLoader = void 0;
-var image_loader_1 = require("./image_loader");
-Object.defineProperty(exports, "WebImageLoader", { enumerable: true, get: function () { return image_loader_1.WebImageLoader; } });
-var tabular_loader_1 = require("./tabular_loader");
-Object.defineProperty(exports, "WebTabularLoader", { enumerable: true, get: function () { return tabular_loader_1.WebTabularLoader; } });
+export { DataLoader } from './data_loader.js';
+export { ImageLoader } from './image_loader.js';
+export { TabularLoader } from './tabular_loader.js';
+export { TextLoader } from './text_loader.js';

package/dist/dataset/data_loader/tabular_loader.d.ts

@@ -1,4 +1,35 @@
-import { tf, data } from '../..';
-export declare class WebTabularLoader extends data.TabularLoader<File> {
-    loadTabularDatasetFrom(source: File, csvConfig: Record<string, unknown>): tf.data.CSVDataset;
+import type { Task } from '../../index.js';
+import type { Dataset, DataSplit } from '../index.js';
+import type { DataConfig } from './index.js';
+import { DataLoader } from './index.js';
+/**
+ * Tabular data loader whose instantiable implementation is delegated by the platform-dependent Disco subprojects, namely,
+ * @epfml/discojs-web and @epfml/discojs-node. Loads data from files whose entries are line-separated and consist of
+ * character-separated features and label(s). Such files typically have the .csv extension.
+ */
+export declare abstract class TabularLoader<Source> extends DataLoader<Source> {
+    private readonly task;
+    readonly delimiter: string;
+    constructor(task: Task, delimiter?: string);
+    /**
+     * Creates a CSV dataset object based off the given source.
+     * @param source File object, URL string or local file system path.
+     * @param csvConfig Object expected by TF.js to create a CSVDataset.
+     * @returns The CSVDataset object built upon the given source.
+     */
+    abstract loadDatasetFrom(source: Source, csvConfig: Record<string, unknown>): Promise<Dataset>;
+    /**
+     * Expects delimiter-separated tabular data made of N columns. The data may be
+     * potentially split among several sources. Every source should contain N-1
+     * feature columns and 1 single label column.
+     * @param source List of File objects, URLs or file system paths.
+     * @param config
+     * @returns A TF.js dataset built upon read tabular data stored in the given sources.
+     */
+    load(source: Source, config?: DataConfig): Promise<Dataset>;
+    /**
+     * Creates the CSV datasets based off the given sources, then fuses them into a single CSV
+     * dataset.
+     */
+    loadAll(sources: Source[], config: DataConfig): Promise<DataSplit>;
 }

package/dist/dataset/data_loader/tabular_loader.js

@@ -1,16 +1,76 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebTabularLoader = void 0;
-var tslib_1 = require("tslib");
-var __1 = require("../..");
-var WebTabularLoader = /** @class */ (function (_super) {
-    (0, tslib_1.__extends)(WebTabularLoader, _super);
-    function WebTabularLoader() {
-        return _super !== null && _super.apply(this, arguments) || this;
+import { List, Map, Set } from 'immutable';
+import { TabularData } from '../index.js';
+import { DataLoader } from './index.js';
+// Window size from which the dataset shuffling will sample
+const BUFFER_SIZE = 1000;
+/**
+ * Tabular data loader whose instantiable implementation is delegated by the platform-dependent Disco subprojects, namely,
+ * @epfml/discojs-web and @epfml/discojs-node. Loads data from files whose entries are line-separated and consist of
+ * character-separated features and label(s). Such files typically have the .csv extension.
+ */
+export class TabularLoader extends DataLoader {
+    task;
+    delimiter;
+    constructor(task, delimiter = ',') {
+        super();
+        this.task = task;
+        this.delimiter = delimiter;
     }
-    WebTabularLoader.prototype.loadTabularDatasetFrom = function (source, csvConfig) {
-        return new __1.tf.data.CSVDataset(new __1.tf.data.FileDataSource(source), csvConfig);
-    };
-    return WebTabularLoader;
-}(__1.data.TabularLoader));
-exports.WebTabularLoader = WebTabularLoader;
+    /**
+     * Expects delimiter-separated tabular data made of N columns. The data may be
+     * potentially split among several sources. Every source should contain N-1
+     * feature columns and 1 single label column.
+     * @param source List of File objects, URLs or file system paths.
+     * @param config
+     * @returns A TF.js dataset built upon read tabular data stored in the given sources.
+     */
+    async load(source, config) {
+        /**
+         * Prepare the CSV config object based off the given features and labels.
+         * If labels is empty, then the returned dataset is comprised of samples only.
+         * Otherwise, each entry is of the form `{ xs, ys }` with `xs` as features and `ys`
+         * as labels.
+         */
+        if (config?.features === undefined) {
+            // TODO @s314cy
+            throw new Error('Not implemented');
+        }
+        const columnConfigs = Map(Set(config.features).map((feature) => [feature, { required: false, isLabel: false }])).merge(Set(config.labels).map((label) => [label, { required: true, isLabel: true }]));
+        const csvConfig = {
+            hasHeader: true,
+            columnConfigs: columnConfigs.toObject(),
+            configuredColumnsOnly: true,
+            delimiter: this.delimiter
+        };
+        const dataset = (await this.loadDatasetFrom(source, csvConfig)).map((t) => {
+            if (typeof t === 'object') {
+                if (('xs' in t) && ('ys' in t)) {
+                    const { xs, ys } = t;
+                    return {
+                        xs: Object.values(xs),
+                        ys: Object.values(ys)
+                    };
+                }
+                else {
+                    return t;
+                }
+            }
+            throw new TypeError('Expected TensorContainerObject');
+        });
+        return (config?.shuffle === undefined || config?.shuffle) ? dataset.shuffle(BUFFER_SIZE) : dataset;
+    }
+    /**
+     * Creates the CSV datasets based off the given sources, then fuses them into a single CSV
+     * dataset.
+     */
+    async loadAll(sources, config) {
+        const datasets = await Promise.all(sources.map(async (source) => await this.load(source, { ...config, shuffle: false })));
+        let dataset = List(datasets).reduce((acc, dataset) => acc.concatenate(dataset));
+        dataset = config?.shuffle === true ? dataset.shuffle(BUFFER_SIZE) : dataset;
+        const data = await TabularData.init(dataset, this.task);
+        // TODO: Implement validation split for tabular data (tricky due to streaming)
+        return {
+            train: data
+        };
+    }
+}

package/dist/dataset/data_loader/text_loader.d.ts

@@ -0,0 +1,14 @@
+import type { Task } from '../../index.js';
+import type { DataSplit, Dataset } from '../index.js';
+import { DataLoader, DataConfig } from './index.js';
+/**
+ * Text data loader whose instantiable implementation is delegated by the platform-dependent Disco subprojects, namely,
+ * @epfml/discojs-web and @epfml/discojs-node.
+ */
+export declare abstract class TextLoader<S> extends DataLoader<S> {
+    private readonly task;
+    constructor(task: Task);
+    abstract loadDatasetFrom(source: S): Promise<Dataset>;
+    load(source: S, config?: DataConfig): Promise<Dataset>;
+    loadAll(sources: S[], config?: DataConfig): Promise<DataSplit>;
+}

package/dist/dataset/data_loader/text_loader.js

@@ -0,0 +1,25 @@
+import { TextData } from '../index.js';
+import { DataLoader } from './index.js';
+/**
+ * Text data loader whose instantiable implementation is delegated by the platform-dependent Disco subprojects, namely,
+ * @epfml/discojs-web and @epfml/discojs-node.
+ */
+export class TextLoader extends DataLoader {
+    task;
+    constructor(task) {
+        super();
+        this.task = task;
+    }
+    async load(source, config) {
+        const dataset = await this.loadDatasetFrom(source);
+        // 1st arg: Stream shuffling buffer size
+        return (config?.shuffle === undefined || config?.shuffle) ? dataset.shuffle(1000, undefined, true) : dataset;
+    }
+    async loadAll(sources, config) {
+        const concatenated = (await Promise.all(sources.map(async (src) => await this.load(src, config))))
+            .reduce((acc, dataset) => acc.concatenate(dataset));
+        return {
+            train: await TextData.init(concatenated, this.task)
+        };
+    }
+}

package/dist/dataset/dataset.d.ts

@@ -0,0 +1,5 @@
+import type tf from '@tensorflow/tfjs';
+/**
+ * Convenient type for the common dataset type used in TF.js.
+ */
+export type Dataset = tf.data.Dataset<tf.TensorContainer>;

package/dist/dataset/dataset.js

@@ -0,0 +1 @@
+export {};

package/dist/dataset/dataset_builder.d.ts

@@ -0,0 +1,60 @@
+import type { Task } from '../index.js';
+import type { DataSplit } from './data/index.js';
+import type { DataConfig, DataLoader } from './data_loader/data_loader.js';
+/**
+ * Incrementally builds a dataset from the provided file sources. The sources may
+ * either be file blobs or regular file system paths.
+ */
+export declare class DatasetBuilder<Source> {
+    /**
+     * The data loader used to load the data contained in the provided files.
+     */
+    private readonly dataLoader;
+    /**
+     * The task for which the dataset should be built.
+     */
+    readonly task: Task;
+    /**
+     * The buffer of unlabelled file sources.
+     */
+    private _sources;
+    /**
+     * The buffer of labelled file sources.
+     */
+    private labelledSources;
+    /**
+     * Whether a dataset was already produced.
+     */
+    private _built;
+    constructor(
+    /**
+     * The data loader used to load the data contained in the provided files.
+     */
+    dataLoader: DataLoader<Source>, 
+    /**
+     * The task for which the dataset should be built.
+     */
+    task: Task);
+    /**
+     * Adds the given file sources to the builder's buffer. Sources may be provided a label in the case
+     * of supervised learning.
+     * @param sources The array of file sources
+     * @param label The file sources label
+     */
+    addFiles(sources: Source[], label?: string): void;
+    /**
+     * Clears the file sources buffers. If a label is provided, only the file sources
+     * corresponding to the given label will be removed.
+     * @param label The file sources label
+     */
+    clearFiles(label?: string): void;
+    private resetBuiltState;
+    private getLabels;
+    build(config?: DataConfig): Promise<DataSplit>;
+    /**
+     * Whether the dataset builder has already been consumed to produce a dataset.
+     */
+    get built(): boolean;
+    get size(): number;
+    get sources(): Source[];
+}

package/dist/dataset/dataset_builder.js

@@ -0,0 +1,142 @@
+import { Map } from 'immutable';
+/**
+ * Incrementally builds a dataset from the provided file sources. The sources may
+ * either be file blobs or regular file system paths.
+ */
+export class DatasetBuilder {
+    dataLoader;
+    task;
+    /**
+     * The buffer of unlabelled file sources.
+     */
+    _sources;
+    /**
+     * The buffer of labelled file sources.
+     */
+    labelledSources;
+    /**
+     * Whether a dataset was already produced.
+     */
+    // TODO useless, responsibility on callers
+    _built;
+    constructor(
+    /**
+     * The data loader used to load the data contained in the provided files.
+     */
+    dataLoader, 
+    /**
+     * The task for which the dataset should be built.
+     */
+    task) {
+        this.dataLoader = dataLoader;
+        this.task = task;
+        this._sources = [];
+        this.labelledSources = Map();
+        this._built = false;
+    }
+    /**
+     * Adds the given file sources to the builder's buffer. Sources may be provided a label in the case
+     * of supervised learning.
+     * @param sources The array of file sources
+     * @param label The file sources label
+     */
+    addFiles(sources, label) {
+        if (this.built) {
+            this.resetBuiltState();
+        }
+        if (label === undefined) {
+            this._sources = this._sources.concat(sources);
+        }
+        else {
+            const currentSources = this.labelledSources.get(label);
+            if (currentSources === undefined) {
+                this.labelledSources = this.labelledSources.set(label, sources);
+            }
+            else {
+                this.labelledSources = this.labelledSources.set(label, currentSources.concat(sources));
+            }
+        }
+    }
+    /**
+     * Clears the file sources buffers. If a label is provided, only the file sources
+     * corresponding to the given label will be removed.
+     * @param label The file sources label
+     */
+    clearFiles(label) {
+        if (this.built) {
+            this.resetBuiltState();
+        }
+        if (label === undefined) {
+            this._sources = [];
+        }
+        else {
+            this.labelledSources = this.labelledSources.delete(label);
+        }
+    }
+    // If files are added or removed, then this should be called since the latest
+    // version of the dataset_builder has not yet been built.
+    resetBuiltState() {
+        this._built = false;
+    }
+    getLabels() {
+        // We need to duplicate the labels as we need one for each source.
+        // Say for label A we have sources [img1, img2, img3], then we
+        // need labels [A, A, A].
+        let labels = [];
+        this.labelledSources.forEach((sources, label) => {
+            const sourcesLabels = Array.from({ length: sources.length }, (_) => label);
+            labels = labels.concat(sourcesLabels);
+        });
+        return labels.flat();
+    }
+    async build(config) {
+        // Require that at least one source collection is non-empty, but not both
+        if ((this._sources.length > 0) === (this.labelledSources.size > 0)) {
+            throw new Error('Please provide dataset input files');
+        }
+        let dataTuple;
+        if (this._sources.length > 0) {
+            let defaultConfig = {};
+            if (config?.inference === true) {
+                // Inferring model, no labels needed
+                defaultConfig = {
+                    features: this.task.trainingInformation.inputColumns,
+                    shuffle: false
+                };
+            }
+            else {
+                // Labels are contained in the given sources
+                defaultConfig = {
+                    features: this.task.trainingInformation.inputColumns,
+                    labels: this.task.trainingInformation.outputColumns,
+                    shuffle: false
+                };
+            }
+            dataTuple = await this.dataLoader.loadAll(this._sources, { ...defaultConfig, ...config });
+        }
+        else {
+            // Labels are inferred from the file selection boxes
+            const defaultConfig = {
+                labels: this.getLabels(),
+                shuffle: false
+            };
+            const sources = this.labelledSources.valueSeq().toArray().flat();
+            dataTuple = await this.dataLoader.loadAll(sources, { ...defaultConfig, ...config });
+        }
+        // TODO @s314cy: Support .csv labels for image datasets (supervised training or testing)
+        this._built = true;
+        return dataTuple;
+    }
+    /**
+     * Whether the dataset builder has already been consumed to produce a dataset.
+     */
+    get built() {
+        return this._built;
+    }
+    get size() {
+        return Math.max(this._sources.length, this.labelledSources.size);
+    }
+    get sources() {
+        return this._sources.length > 0 ? this._sources : this.labelledSources.valueSeq().toArray().flat();
+    }
+}

package/dist/dataset/index.d.ts

@@ -0,0 +1,5 @@
+export type { Dataset } from './dataset.js';
+export { DatasetBuilder } from './dataset_builder.js';
+export { ImageLoader, TabularLoader, DataLoader, TextLoader } from './data_loader/index.js';
+export type { DataSplit } from './data/index.js';
+export { Data, TabularData, ImageData, TextData, ImagePreprocessing, TabularPreprocessing, TextPreprocessing, IMAGE_PREPROCESSING, TABULAR_PREPROCESSING, TEXT_PREPROCESSING } from './data/index.js';

package/dist/dataset/index.js

@@ -0,0 +1,3 @@
+export { DatasetBuilder } from './dataset_builder.js';
+export { ImageLoader, TabularLoader, DataLoader, TextLoader } from './data_loader/index.js';
+export { Data, TabularData, ImageData, TextData, ImagePreprocessing, TabularPreprocessing, TextPreprocessing, IMAGE_PREPROCESSING, TABULAR_PREPROCESSING, TEXT_PREPROCESSING } from './data/index.js';

package/dist/default_tasks/cifar10/index.d.ts

@@ -0,0 +1,2 @@
+import type { TaskProvider } from '../../index.js';
+export declare const cifar10: TaskProvider;

package/dist/{core/default_tasks/cifar10.js → default_tasks/cifar10/index.js}

@@ -1,12 +1,10 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.cifar10 = void 0;
-var tslib_1 = require("tslib");
-var __1 = require("..");
-exports.cifar10 = {
-    getTask: function () {
+import * as tf from '@tensorflow/tfjs';
+import { data, models } from '../../index.js';
+import baseModel from './model.js';
+export const cifar10 = {
+    getTask() {
         return {
-            taskID: 'cifar10',
+            id: 'cifar10',
             displayInformation: {
                 taskTitle: 'CIFAR10',
                 summary: {
@@ -25,17 +23,12 @@ exports.cifar10 = {
                 roundDuration: 10,
                 validationSplit: 0.2,
                 batchSize: 10,
-                modelCompileData: {
-                    optimizer: 'sgd',
-                    loss: 'categoricalCrossentropy',
-                    metrics: ['accuracy']
-                },
                 dataType: 'image',
-                IMAGE_H: 32,
-                IMAGE_W: 32,
-                preprocessingFunctions: [],
+                preprocessingFunctions: [data.ImagePreprocessing.Resize],
+                IMAGE_H: 224,
+                IMAGE_W: 224,
                 LABEL_LIST: ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'],
-                scheme: 'Decentralized',
+                scheme: 'decentralized',
                 noiseScale: undefined,
                 clippingRadius: 20,
                 decentralizedSecure: true,
@@ -44,25 +37,24 @@ exports.cifar10 = {
             }
         };
     },
-    getModel: function () {
-        return (0, tslib_1.__awaiter)(this, void 0, void 0, function () {
-            var mobilenet, x, predictions;
-            return (0, tslib_1.__generator)(this, function (_a) {
-                switch (_a.label) {
-                    case 0: return [4 /*yield*/, __1.tf.loadLayersModel('https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_0.25_224/model.json')];
-                    case 1:
-                        mobilenet = _a.sent();
-                        x = mobilenet.getLayer('global_average_pooling2d_1');
-                        predictions = __1.tf.layers
-                            .dense({ units: 10, activation: 'softmax', name: 'denseModified' })
-                            .apply(x.output);
-                        return [2 /*return*/, __1.tf.model({
-                                inputs: mobilenet.input,
-                                outputs: predictions,
-                                name: 'modelModified'
-                            })];
-                }
-            });
+    async getModel() {
+        const mobilenet = await tf.loadLayersModel({
+            load: async () => Promise.resolve(baseModel),
+        });
+        const x = mobilenet.getLayer('global_average_pooling2d_1');
+        const predictions = tf.layers
+            .dense({ units: 10, activation: 'softmax', name: 'denseModified' })
+            .apply(x.output);
+        const model = tf.model({
+            inputs: mobilenet.input,
+            outputs: predictions,
+            name: 'modelModified'
+        });
+        model.compile({
+            optimizer: 'sgd',
+            loss: 'categoricalCrossentropy',
+            metrics: ['accuracy']
         });
+        return new models.TFJS(model);
     }
 };