@epfml/discojs 2.0.0 → 2.1.2-p20240506085037.0

package/dist/client/local.js

@@ -0,0 +1,3 @@
+import { Base } from './base.js';
+export class Local extends Base {
+}

package/dist/client/messages.d.ts

@@ -0,0 +1,30 @@
+import type * as decentralized from './decentralized/messages.js';
+import type * as federated from './federated/messages.js';
+import { type NodeID } from './types.js';
+export declare enum type {
+    ClientConnected = 0,
+    AssignNodeID = 1,
+    SignalForPeer = 2,
+    PeerIsReady = 3,
+    PeersForRound = 4,
+    Payload = 5,
+    SendPayload = 6,
+    ReceiveServerMetadata = 7,
+    ReceiveServerPayload = 8,
+    RequestServerStatistics = 9,
+    ReceiveServerStatistics = 10
+}
+export interface ClientConnected {
+    type: type.ClientConnected;
+}
+export interface AssignNodeID {
+    type: type.AssignNodeID;
+    id: NodeID;
+}
+export type Message = decentralized.MessageFromServer | decentralized.MessageToServer | decentralized.PeerMessage | federated.MessageFederated;
+export type NarrowMessage<D> = Extract<Message, {
+    type: D;
+}>;
+export declare function hasMessageType(raw: unknown): raw is {
+    type: type;
+} & Record<string, unknown>;

package/dist/client/messages.js

@@ -0,0 +1,26 @@
+export var type;
+(function (type) {
+    type[type["ClientConnected"] = 0] = "ClientConnected";
+    type[type["AssignNodeID"] = 1] = "AssignNodeID";
+    // Decentralized
+    type[type["SignalForPeer"] = 2] = "SignalForPeer";
+    type[type["PeerIsReady"] = 3] = "PeerIsReady";
+    type[type["PeersForRound"] = 4] = "PeersForRound";
+    type[type["Payload"] = 5] = "Payload";
+    // Federated
+    type[type["SendPayload"] = 6] = "SendPayload";
+    type[type["ReceiveServerMetadata"] = 7] = "ReceiveServerMetadata";
+    type[type["ReceiveServerPayload"] = 8] = "ReceiveServerPayload";
+    type[type["RequestServerStatistics"] = 9] = "RequestServerStatistics";
+    type[type["ReceiveServerStatistics"] = 10] = "ReceiveServerStatistics";
+})(type || (type = {}));
+export function hasMessageType(raw) {
+    if (typeof raw !== 'object' || raw === null) {
+        return false;
+    }
+    const o = raw;
+    if (!('type' in o && typeof o.type === 'number' && o.type in type)) {
+        return false;
+    }
+    return true;
+}

package/dist/client/types.d.ts

@@ -0,0 +1,2 @@
+export type NodeID = string;
+export declare function isNodeID(raw: unknown): raw is NodeID;

package/dist/client/types.js

@@ -0,0 +1,4 @@
+// TODO @s314cy: regexp test just like server-side
+export function isNodeID(raw) {
+    return typeof raw === 'string';
+}

package/dist/client/utils.d.ts

@@ -0,0 +1,2 @@
+export declare const MAX_WAIT_PER_ROUND = 15000;
+export declare function timeout(ms?: number): Promise<never>;

package/dist/client/utils.js

@@ -0,0 +1,7 @@
+// Time to wait for the others in milliseconds.
+export const MAX_WAIT_PER_ROUND = 15_000;
+export async function timeout(ms = MAX_WAIT_PER_ROUND) {
+    return await new Promise((_, reject) => {
+        setTimeout(() => { reject(new Error('timeout')); }, ms);
+    });
+}

package/dist/dataset/data/data.d.ts

@@ -0,0 +1,48 @@
+import type tf from '@tensorflow/tfjs';
+import type { List } from 'immutable';
+import type { Task } from '../../index.js';
+import type { Dataset } from '../index.js';
+import type { PreprocessingFunction } from './preprocessing/base.js';
+/**
+ * Abstract class representing an immutable Disco dataset, including a TF.js dataset,
+ * Disco task and set of preprocessing functions.
+ */
+export declare abstract class Data {
+    readonly dataset: Dataset;
+    readonly task: Task;
+    readonly size?: number | undefined;
+    abstract readonly availablePreprocessing: List<PreprocessingFunction>;
+    protected constructor(dataset: Dataset, task: Task, size?: number | undefined);
+    static init(_dataset: Dataset, _task: Task, _size?: number): Promise<Data>;
+    /**
+     * Callable abstract method instead of constructor.
+     */
+    protected abstract create(dataset: Dataset, task: Task, size?: number): Data;
+    /**
+     * Creates a new Disco data object containing the batched TF.js dataset, according to the
+     * task's parameters.
+     * @returns The batched Disco data
+     */
+    batch(): Data;
+    /**
+     * The TF.js dataset batched according to the task's parameters.
+     */
+    get batchedDataset(): Dataset;
+    /**
+     * Creates a new Disco data object containing the preprocessed TF.js dataset,
+     * according to the defined set of preprocessing functions and the task's parameters.
+     * @returns The preprocessed Disco data
+     */
+    preprocess(): Data;
+    /**
+     * Creates a higher level preprocessing function applying the specified set of preprocessing
+     * functions in a series. The preprocessing functions are chained according to their defined
+     * priority.
+     */
+    get preprocessing(): (entry: tf.TensorContainer) => Promise<tf.TensorContainer>;
+    /**
+     * The TF.js dataset preprocessing according to the set of preprocessing functions and the task's
+     * parameters.
+     */
+    get preprocessedDataset(): Dataset;
+}

package/dist/dataset/data/data.js

@@ -0,0 +1,72 @@
+/**
+ * Abstract class representing an immutable Disco dataset, including a TF.js dataset,
+ * Disco task and set of preprocessing functions.
+ */
+export class Data {
+    dataset;
+    task;
+    size;
+    constructor(dataset, task, size) {
+        this.dataset = dataset;
+        this.task = task;
+        this.size = size;
+    }
+    static init(_dataset, _task, _size) {
+        return Promise.reject(new Error('abstract'));
+    }
+    /**
+     * Creates a new Disco data object containing the batched TF.js dataset, according to the
+     * task's parameters.
+     * @returns The batched Disco data
+     */
+    batch() {
+        return this.create(this.batchedDataset, this.task, this.size);
+    }
+    /**
+     * The TF.js dataset batched according to the task's parameters.
+     */
+    get batchedDataset() {
+        const batchSize = this.task.trainingInformation.batchSize;
+        return batchSize === undefined
+            ? this.dataset
+            : this.dataset.batch(batchSize);
+    }
+    /**
+     * Creates a new Disco data object containing the preprocessed TF.js dataset,
+     * according to the defined set of preprocessing functions and the task's parameters.
+     * @returns The preprocessed Disco data
+     */
+    preprocess() {
+        return this.create(this.preprocessedDataset, this.task, this.size);
+    }
+    /**
+     * Creates a higher level preprocessing function applying the specified set of preprocessing
+     * functions in a series. The preprocessing functions are chained according to their defined
+     * priority.
+     */
+    get preprocessing() {
+        const params = this.task.trainingInformation;
+        const taskPreprocessing = params.preprocessingFunctions;
+        if (taskPreprocessing === undefined ||
+            taskPreprocessing.length === 0 ||
+            this.availablePreprocessing === undefined ||
+            this.availablePreprocessing.size === 0) {
+            return x => Promise.resolve(x);
+        }
+        const applyPreprocessing = this.availablePreprocessing
+            .filter((e) => e.type in taskPreprocessing)
+            .map((e) => e.apply);
+        if (applyPreprocessing.size === 0) {
+            return x => Promise.resolve(x);
+        }
+        const preprocessingChain = applyPreprocessing.reduce((acc, fn) => x => fn(acc(x), this.task), (x) => x);
+        return x => preprocessingChain(Promise.resolve(x));
+    }
+    /**
+     * The TF.js dataset preprocessing according to the set of preprocessing functions and the task's
+     * parameters.
+     */
+    get preprocessedDataset() {
+        return this.dataset.mapAsync(this.preprocessing);
+    }
+}

package/dist/dataset/data/data_split.d.ts

@@ -0,0 +1,8 @@
+import type { Data } from './data.js';
+/**
+ * Train-validation split of Disco data.
+ */
+export interface DataSplit {
+    train: Data;
+    validation?: Data;
+}

package/dist/dataset/data/data_split.js

@@ -0,0 +1 @@
+export {};

package/dist/dataset/data/image_data.d.ts

@@ -0,0 +1,11 @@
+import type { Task } from '../../index.js';
+import type { Dataset } from '../dataset.js';
+import { Data } from './data.js';
+/**
+ * Disco data made of image samples (.jpg, .png, etc.).
+ */
+export declare class ImageData extends Data {
+    readonly availablePreprocessing: import("immutable").List<import("./preprocessing/base.js").PreprocessingFunction>;
+    static init(dataset: Dataset, task: Task, size?: number): Promise<Data>;
+    protected create(dataset: Dataset, task: Task, size: number): ImageData;
+}

package/dist/dataset/data/image_data.js

@@ -0,0 +1,38 @@
+import { Data } from './data.js';
+import { ImagePreprocessing, IMAGE_PREPROCESSING } from './preprocessing/index.js';
+/**
+ * Disco data made of image samples (.jpg, .png, etc.).
+ */
+export class ImageData extends Data {
+    availablePreprocessing = IMAGE_PREPROCESSING;
+    static async init(dataset, task, size) {
+        // Here we do our best to check data format before proceeding to training, for
+        // better error handling. An incorrectly formatted image in the dataset might still
+        // cause an error during training, because of the lazy aspect of the dataset; we only
+        // verify the first sample.
+        if (task.trainingInformation.preprocessingFunctions?.includes(ImagePreprocessing.Resize) !== true) {
+            const sample = (await dataset.take(1).toArray())[0];
+            // TODO: We suppose the presence of labels
+            // TODO: Typing (discojs-node/src/dataset/data_loader/image_loader.spec.ts)
+            if (typeof sample !== 'object' || sample === null || sample === undefined) {
+                throw new Error("Image is undefined or is not an object");
+            }
+            let shape;
+            if ('xs' in sample && 'ys' in sample) {
+                shape = sample.xs.shape;
+            }
+            else {
+                shape = sample.shape;
+            }
+            const { IMAGE_H, IMAGE_W } = task.trainingInformation;
+            if (IMAGE_W !== undefined && IMAGE_H !== undefined &&
+                (shape[0] !== IMAGE_W || shape[1] !== IMAGE_H)) {
+                throw new Error(`Image doesn't have the dimensions specified in the task's training information. Expected ${IMAGE_H}x${IMAGE_W} but got ${shape[0]}x${shape[1]}.`);
+            }
+        }
+        return new ImageData(dataset, task, size);
+    }
+    create(dataset, task, size) {
+        return new ImageData(dataset, task, size);
+    }
+}

package/dist/dataset/data/index.d.ts

@@ -0,0 +1,6 @@
+export type { DataSplit } from './data_split.js';
+export { Data } from './data.js';
+export { ImageData } from './image_data.js';
+export { TabularData } from './tabular_data.js';
+export { TextData } from './text_data.js';
+export { ImagePreprocessing, TabularPreprocessing, TextPreprocessing, IMAGE_PREPROCESSING, TABULAR_PREPROCESSING, TEXT_PREPROCESSING } from './preprocessing/index.js';

package/dist/dataset/data/index.js

@@ -0,0 +1,5 @@
+export { Data } from './data.js';
+export { ImageData } from './image_data.js';
+export { TabularData } from './tabular_data.js';
+export { TextData } from './text_data.js';
+export { ImagePreprocessing, TabularPreprocessing, TextPreprocessing, IMAGE_PREPROCESSING, TABULAR_PREPROCESSING, TEXT_PREPROCESSING } from './preprocessing/index.js';

package/dist/dataset/data/preprocessing/base.d.ts

@@ -0,0 +1,16 @@
+import type tf from '@tensorflow/tfjs';
+import type { Task } from '../../../index.js';
+import type { ImagePreprocessing } from './image_preprocessing.js';
+import type { TabularPreprocessing } from './tabular_preprocessing.js';
+import type { TextPreprocessing } from './text_preprocessing.js';
+/**
+ * All available preprocessing type enums.
+ */
+export type Preprocessing = ImagePreprocessing | TextPreprocessing | TabularPreprocessing;
+/**
+ * Preprocessing function associating a preprocessing type enum to a sample transformation.
+ */
+export interface PreprocessingFunction {
+    type: Preprocessing;
+    apply: (x: Promise<tf.TensorContainer>, task: Task) => Promise<tf.TensorContainer>;
+}

package/dist/dataset/data/preprocessing/base.js

@@ -0,0 +1 @@
+export {};

package/dist/dataset/data/preprocessing/image_preprocessing.d.ts

@@ -0,0 +1,13 @@
+import { List } from 'immutable';
+import type { PreprocessingFunction } from './base.js';
+/**
+ * Available image preprocessing types.
+ */
+export declare enum ImagePreprocessing {
+    Resize = 0,
+    Normalize = 1
+}
+/**
+ * Available image preprocessing functions.
+ */
+export declare const AVAILABLE_PREPROCESSING: List<PreprocessingFunction>;

package/dist/dataset/data/preprocessing/image_preprocessing.js

@@ -0,0 +1,40 @@
+import { List } from 'immutable';
+import * as tf from '@tensorflow/tfjs';
+/**
+ * Available image preprocessing types.
+ */
+export var ImagePreprocessing;
+(function (ImagePreprocessing) {
+    ImagePreprocessing[ImagePreprocessing["Resize"] = 0] = "Resize";
+    ImagePreprocessing[ImagePreprocessing["Normalize"] = 1] = "Normalize";
+})(ImagePreprocessing || (ImagePreprocessing = {}));
+const resize = {
+    type: ImagePreprocessing.Resize,
+    apply: async (entry, task) => {
+        const { xs, ys } = await entry;
+        const params = task.trainingInformation;
+        return {
+            xs: params.IMAGE_W !== undefined && params.IMAGE_H !== undefined
+                ? xs.resizeBilinear([params.IMAGE_H, params.IMAGE_W])
+                : xs,
+            ys
+        };
+    }
+};
+const normalize = {
+    type: ImagePreprocessing.Normalize,
+    apply: async (entry) => {
+        const { xs, ys } = await entry;
+        return {
+            xs: xs.div(tf.scalar(255)),
+            ys
+        };
+    }
+};
+/**
+ * Available image preprocessing functions.
+ */
+export const AVAILABLE_PREPROCESSING = List([
+    resize,
+    normalize
+]).sortBy((e) => e.type);

package/dist/dataset/data/preprocessing/index.d.ts

@@ -0,0 +1,4 @@
+export type { Preprocessing, PreprocessingFunction } from './base.js';
+export { AVAILABLE_PREPROCESSING as IMAGE_PREPROCESSING, ImagePreprocessing } from './image_preprocessing.js';
+export { AVAILABLE_PREPROCESSING as TABULAR_PREPROCESSING, TabularPreprocessing } from './tabular_preprocessing.js';
+export { AVAILABLE_PREPROCESSING as TEXT_PREPROCESSING, TextPreprocessing } from './text_preprocessing.js';

package/dist/dataset/data/preprocessing/index.js

@@ -0,0 +1,3 @@
+export { AVAILABLE_PREPROCESSING as IMAGE_PREPROCESSING, ImagePreprocessing } from './image_preprocessing.js';
+export { AVAILABLE_PREPROCESSING as TABULAR_PREPROCESSING, TabularPreprocessing } from './tabular_preprocessing.js';
+export { AVAILABLE_PREPROCESSING as TEXT_PREPROCESSING, TextPreprocessing } from './text_preprocessing.js';

package/dist/dataset/data/preprocessing/tabular_preprocessing.d.ts

@@ -0,0 +1,13 @@
+import { List } from 'immutable';
+import type { PreprocessingFunction } from './base.js';
+/**
+ * Available tabular preprocessing types.
+ */
+export declare enum TabularPreprocessing {
+    Sanitize = 0,
+    Normalize = 1
+}
+/**
+ * Available tabular preprocessing functions.
+ */
+export declare const AVAILABLE_PREPROCESSING: List<PreprocessingFunction>;

package/dist/dataset/data/preprocessing/tabular_preprocessing.js

@@ -0,0 +1,45 @@
+import { List } from 'immutable';
+/**
+ * Available tabular preprocessing types.
+ */
+export var TabularPreprocessing;
+(function (TabularPreprocessing) {
+    TabularPreprocessing[TabularPreprocessing["Sanitize"] = 0] = "Sanitize";
+    TabularPreprocessing[TabularPreprocessing["Normalize"] = 1] = "Normalize";
+})(TabularPreprocessing || (TabularPreprocessing = {}));
+const sanitize = {
+    type: TabularPreprocessing.Sanitize,
+    apply: async (entry) => {
+        const entryContainer = await entry;
+        // if preprocessing a dataset without labels, then the entry is an array of numbers
+        if (Array.isArray(entryContainer)) {
+            const entry = entryContainer;
+            return entry.map((i) => i ?? 0);
+            // if it is an object
+        }
+        else if (typeof entryContainer === 'object' && entry !== null) {
+            // if the object is a tensor container with features xs and labels ys
+            if (Object.hasOwn(entryContainer, 'xs')) {
+                const { xs, ys } = entryContainer;
+                return {
+                    xs: xs.map(i => i ?? 0),
+                    ys
+                };
+                // if the object contains features as a dict of feature names-values
+            }
+            else {
+                const entry = Object.values(entryContainer);
+                return entry.map((i) => i ?? 0);
+            }
+        }
+        else {
+            throw new Error('Unrecognized format during tabular preprocessing');
+        }
+    }
+};
+/**
+ * Available tabular preprocessing functions.
+ */
+export const AVAILABLE_PREPROCESSING = List([
+    sanitize
+]).sortBy((e) => e.type);

package/dist/dataset/data/preprocessing/text_preprocessing.d.ts

@@ -0,0 +1,13 @@
+import { List } from 'immutable';
+import type { PreprocessingFunction } from './base.js';
+/**
+ * Available text preprocessing types.
+ */
+export declare enum TextPreprocessing {
+    Tokenize = 0,
+    LeftPadding = 1
+}
+/**
+ * Available text preprocessing functions.
+ */
+export declare const AVAILABLE_PREPROCESSING: List<PreprocessingFunction>;

package/dist/dataset/data/preprocessing/text_preprocessing.js

@@ -0,0 +1,85 @@
+import { List } from 'immutable';
+import * as tf from '@tensorflow/tfjs';
+import { models } from '../../../index.js';
+/**
+ * Available text preprocessing types.
+ */
+export var TextPreprocessing;
+(function (TextPreprocessing) {
+    TextPreprocessing[TextPreprocessing["Tokenize"] = 0] = "Tokenize";
+    TextPreprocessing[TextPreprocessing["LeftPadding"] = 1] = "LeftPadding";
+})(TextPreprocessing || (TextPreprocessing = {}));
+/**
+ * LeftPadding pads all incoming inputs to be a fixed length, which should be specified
+ * in `task.trainingInformation.maxSequenceLength`.
+ *
+ * We are currently only implementing left padding for text generation
+ * https://huggingface.co/docs/transformers/en/llm_tutorial#wrong-padding-side
+ * The function can easily be extended to support right padding if needed
+ *
+ * Once Transformers.js supports left padding, it will be possible to pad inputs
+ * directly when tokenizing
+ * https://github.com/xenova/transformers.js/blob/8804c36591d11d8456788d1bb4b16489121b3be2/src/tokenizers.js#L2517
+ */
+const leftPadding = {
+    type: TextPreprocessing.LeftPadding,
+    apply: async (x, task) => {
+        if (x === undefined || !Array.isArray(x) || x.length == 0 || typeof (x[0] != 'number')) {
+            new Error("The leftPadding preprocessing expects a non empty 1D array of number");
+        }
+        const { tokens } = await x;
+        const tokenizer = await models.getTaskTokenizer(task);
+        return tf.tidy(() => {
+            // maxLength is the final length of xs
+            // Because ys the contains the tokens in xs shifted by one (to predict the next token), we need
+            // to include one more token than maxSequenceLength in order to have the next token's label of the maxSequenceLength'th token
+            const maxLength = task.trainingInformation.maxSequenceLength ?? tokenizer.model_max_length;
+            const maxLengthPlusLabel = maxLength + 1;
+            let fixedLengthTokens = tf.tensor(tokens, undefined, 'int32'); // cast tokens from float to int for gpt-tfjs
+            if (fixedLengthTokens.size > maxLengthPlusLabel) { // Should never happen because tokenization truncates inputs
+                throw Error("There are more tokens than expected after tokenization and truncation");
+            }
+            else if (fixedLengthTokens.size < maxLengthPlusLabel) { // Pad inputs to fixed length
+                const paddingToken = tokenizer.pad_token_id;
+                fixedLengthTokens = fixedLengthTokens.pad([[Math.max(0, maxLengthPlusLabel - fixedLengthTokens.size), 0]], paddingToken);
+            }
+            // if tokens.size == maxLengthPlusLabel we can leave it as it is
+            // ys is a one-hot encoding of the next token (i.e. xs shifted by one)
+            const ys = tf.oneHot(fixedLengthTokens.slice([1]), tokenizer.model.vocab.length + 1);
+            // remove the extra token now that ys is created
+            const xs = fixedLengthTokens.slice([0], maxLength);
+            return { xs, ys };
+        });
+    }
+};
+/**
+ * Tokenize and truncates input strings
+ */
+const tokenize = {
+    type: TextPreprocessing.Tokenize,
+    apply: async (x, task) => {
+        if (typeof x != 'string') {
+            new Error("The tokenize preprocessing expects a string as input");
+        }
+        const xs = await x; // tf.TextLineDataset yields strings
+        const tokenizer = await models.getTaskTokenizer(task);
+        // Add plus one to include the next token label of the last token in the input sequence
+        // The inputs are truncated down to exactly maxSequenceLength in leftPadding
+        const maxLength = task.trainingInformation.maxSequenceLength ?? tokenizer.model_max_length;
+        const maxLengthPlusLabel = maxLength + 1;
+        const { input_ids: tokens } = tokenizer(xs, {
+            // Transformers.js currently only supports right padding while we need left for text generation
+            // Right padding should be supported in the future, once it is, we can directly pad while tokenizing
+            // https://github.com/xenova/transformers.js/blob/8804c36591d11d8456788d1bb4b16489121b3be2/src/tokenizers.js#L2517
+            padding: false,
+            truncation: true,
+            return_tensor: false,
+            max_length: maxLengthPlusLabel,
+        });
+        return { tokens };
+    }
+};
+/**
+ * Available text preprocessing functions.
+ */
+export const AVAILABLE_PREPROCESSING = List.of(tokenize, leftPadding).sortBy((e) => e.type);

package/dist/dataset/data/tabular_data.d.ts

@@ -0,0 +1,11 @@
+import type { Task } from '../../index.js';
+import type { Dataset } from '../dataset.js';
+import { Data } from './data.js';
+/**
+ * Disco data made of tabular (.csv, .tsv, etc.) files.
+ */
+export declare class TabularData extends Data {
+    readonly availablePreprocessing: import("immutable").List<import("./preprocessing/base.js").PreprocessingFunction>;
+    static init(dataset: Dataset, task: Task, size?: number): Promise<TabularData>;
+    protected create(dataset: Dataset, task: Task, size: number): TabularData;
+}

package/dist/dataset/data/tabular_data.js

@@ -0,0 +1,25 @@
+import { Data } from './data.js';
+import { TABULAR_PREPROCESSING } from './preprocessing/index.js';
+/**
+ * Disco data made of tabular (.csv, .tsv, etc.) files.
+ */
+export class TabularData extends Data {
+    availablePreprocessing = TABULAR_PREPROCESSING;
+    static async init(dataset, task, size) {
+        // Force the check of the data column format (among other things) before proceeding
+        // to training, for better error handling. An incorrectly formatted line might still
+        // cause an error during training, because of the lazy aspect of the dataset; we only
+        // load/read the tabular file's lines on training.
+        try {
+            await dataset.iterator();
+        }
+        catch (e) {
+            console.error('Data input format is not compatible with the chosen task.');
+            throw (e);
+        }
+        return new TabularData(dataset, task, size);
+    }
+    create(dataset, task, size) {
+        return new TabularData(dataset, task, size);
+    }
+}

package/dist/dataset/data/text_data.d.ts

@@ -0,0 +1,11 @@
+import type { Task } from '../../index.js';
+import type { Dataset } from '../dataset.js';
+import { Data } from './data.js';
+/**
+ * Disco data made of textual samples.
+ */
+export declare class TextData extends Data {
+    readonly availablePreprocessing: import("immutable").List<import("./preprocessing/base.js").PreprocessingFunction>;
+    static init(dataset: Dataset, task: Task, size?: number): Promise<TextData>;
+    protected create(dataset: Dataset, task: Task, size?: number): TextData;
+}

package/dist/dataset/data/text_data.js

@@ -0,0 +1,14 @@
+import { Data } from './data.js';
+import { TEXT_PREPROCESSING } from './preprocessing/index.js';
+/**
+ * Disco data made of textual samples.
+ */
+export class TextData extends Data {
+    availablePreprocessing = TEXT_PREPROCESSING;
+    static init(dataset, task, size) {
+        return Promise.resolve(new TextData(dataset, task, size));
+    }
+    create(dataset, task, size) {
+        return new TextData(dataset, task, size);
+    }
+}

package/dist/{core/dataset → dataset}/data_loader/data_loader.d.ts

@@ -1,15 +1,13 @@
-import { Task } from '../..';
-import { Dataset } from '../dataset';
-import { DataSplit } from '../data';
+import type { DataSplit, Dataset } from '../index.js';
 export interface DataConfig {
     features?: string[];
     labels?: string[];
     shuffle?: boolean;
     validationSplit?: number;
+    inference?: boolean;
+    channels?: number;
 }
 export declare abstract class DataLoader<Source> {
-    protected task: Task;
-    constructor(task: Task);
     abstract load(source: Source, config: DataConfig): Promise<Dataset>;
     abstract loadAll(sources: Source[], config: DataConfig): Promise<DataSplit>;
 }

package/dist/dataset/data_loader/data_loader.js

@@ -0,0 +1,2 @@
+export class DataLoader {
+}

package/dist/dataset/data_loader/image_loader.d.ts

@@ -1,4 +1,21 @@
-import { tf, data } from '../..';
-export declare class WebImageLoader extends data.ImageLoader<File> {
-    readImageFrom(source: File): Promise<tf.Tensor3D>;
+import * as tf from '@tensorflow/tfjs';
+import type { Task } from '../../index.js';
+import type { Dataset, DataSplit } from '../index.js';
+import type { DataConfig } from '../data_loader/index.js';
+import { DataLoader } from '../data_loader/index.js';
+/**
+ * Image data loader whose instantiable implementation is delegated by the platform-dependent Disco subprojects, namely,
+ * @epfml/discojs-web and @epfml/discojs-node.
+ * Load labels and correctly match them with their respective images, with the following constraints:
+ * 1. Images are given as 1 image/1 file;
+ * 2. Labels are given as multiple labels/1 file, each label file can contain a different amount of labels.
+ */
+export declare abstract class ImageLoader<Source> extends DataLoader<Source> {
+    private readonly task;
+    abstract readImageFrom(source: Source, channels?: number): Promise<tf.Tensor3D>;
+    constructor(task: Task);
+    load(image: Source, config?: DataConfig): Promise<Dataset>;
+    private buildDataset;
+    loadAll(images: Source[], config?: DataConfig): Promise<DataSplit>;
+    shuffle(array: number[]): void;
 }