@epfml/discojs 3.0.1-p20241025115642.0 → 3.0.1-p20241107104659.0

package/dist/training/disco.d.ts

@@ -1,11 +1,19 @@
-import { client as clients, BatchLogs, EpochLogs, Logger, Task, TrainingInformation } from "../index.js";
-import type { TypedLabeledDataset } from "../index.js";
+import { client as clients, BatchLogs, EpochLogs, Logger, TrainingInformation, Dataset } from "../index.js";
+import type { DataFormat, DataType, Task } from "../index.js";
 import type { Aggregator } from "../aggregator/index.js";
 import { EventEmitter } from "../utils/event_emitter.js";
 import { RoundLogs, Trainer } from "./trainer.js";
 interface DiscoConfig {
-    scheme: TrainingInformation["scheme"];
+    scheme: TrainingInformation<DataType>["scheme"];
     logger: Logger;
+    /**
+     * keep preprocessed dataset in memory while training
+     *
+     * `Dataset` is cached anyway but this cache can get evicted.
+     * if your system has enough memory to keep the whole preprocessed `Dataset` around,
+     * you can switch this on to only do it once, trading memory for speed.
+     */
+    preprocessOnce: boolean;
 }
 export type RoundStatus = 'not enough participants' | // Server notification to wait for more participants
 'updating model' | // fetching/aggregating local updates into a global model
@@ -16,11 +24,11 @@ export type RoundStatus = 'not enough participants' | // Server notification to
  * a convenient object providing a reduced yet complete API that wraps model training and
  * communication with nodes.
  */
-export declare class Disco extends EventEmitter<{
-    'status': RoundStatus;
+export declare class Disco<D extends DataType> extends EventEmitter<{
+    status: RoundStatus;
 }> {
     #private;
-    readonly trainer: Trainer;
+    readonly trainer: Trainer<D>;
     /**
      * Connect to the given task and get ready to train.
      *
@@ -28,25 +36,25 @@ export declare class Disco extends EventEmitter<{
      * @param clientConfig client to connect with or parameters on how to create one.
      * @param config the DiscoConfig
      */
-    constructor(task: Task, clientConfig: clients.Client | URL | {
+    constructor(task: Task<D>, clientConfig: clients.Client | URL | {
         aggregator: Aggregator;
         url: URL;
     }, config: Partial<DiscoConfig>);
     /** Train on dataset, yielding logs of every round. */
-    trainByRound(dataset: TypedLabeledDataset): AsyncGenerator<RoundLogs>;
+    trainByRound(dataset: Dataset<DataFormat.Raw[D]>): AsyncGenerator<RoundLogs>;
     /** Train on dataset, yielding logs of every epoch. */
-    trainByEpoch(dataset: TypedLabeledDataset): AsyncGenerator<EpochLogs>;
+    trainByEpoch(dataset: Dataset<DataFormat.Raw[D]>): AsyncGenerator<EpochLogs>;
     /** Train on dataset, yielding logs of every batch. */
-    trainByBatch(dataTuple: TypedLabeledDataset): AsyncGenerator<BatchLogs>;
+    trainByBatch(dataset: Dataset<DataFormat.Raw[D]>): AsyncGenerator<BatchLogs>;
     /** Run whole train on dataset. */
-    trainFully(dataTuple: TypedLabeledDataset): Promise<void>;
+    trainFully(dataset: Dataset<DataFormat.Raw[D]>): Promise<void>;
     /**
      * Train on dataset, yield the nested steps.
      *
      * Don't forget to await the yielded generator otherwise nothing will progress.
      * If you don't care about the whole process, use one of the other train methods.
      **/
-    train(dataset: TypedLabeledDataset): AsyncGenerator<AsyncGenerator<AsyncGenerator<BatchLogs, EpochLogs>, RoundLogs>>;
+    train(dataset: Dataset<DataFormat.Raw[D]>): AsyncGenerator<AsyncGenerator<AsyncGenerator<BatchLogs, EpochLogs>, RoundLogs>>;
     /**
      * Completely stops the ongoing training instance.
      */

package/dist/training/disco.js

@@ -1,9 +1,8 @@
-import { async_iterator, client as clients, ConsoleLogger, } from "../index.js";
+import { async_iterator, client as clients, ConsoleLogger, processing, Dataset, } from "../index.js";
 import { getAggregator } from "../aggregator/index.js";
 import { enumerate, split } from "../utils/async_iterator.js";
 import { EventEmitter } from "../utils/event_emitter.js";
 import { Trainer } from "./trainer.js";
-import { labeledDatasetToDataSplit } from "../dataset/data/helpers.js";
 /**
  * Top-level class handling distributed training from a client's perspective. It is meant to be
  * a convenient object providing a reduced yet complete API that wraps model training and
@@ -14,6 +13,7 @@ export class Disco extends EventEmitter {
     #client;
     #logger;
     #task;
+    #preprocessOnce;
     /**
      * Connect to the given task and get ready to train.
      *
@@ -23,9 +23,10 @@ export class Disco extends EventEmitter {
      */
     constructor(task, clientConfig, config) {
         super();
-        const { scheme, logger } = {
+        const { scheme, logger, preprocessOnce } = {
             scheme: task.trainingInformation.scheme,
             logger: new ConsoleLogger(),
+            preprocessOnce: false,
             ...config,
         };
         let client;
@@ -46,11 +47,12 @@ export class Disco extends EventEmitter {
         if (client.task !== task)
             throw new Error("client not setup for given task");
         this.#logger = logger;
+        this.#preprocessOnce = preprocessOnce;
         this.#client = client;
         this.#task = task;
         this.trainer = new Trainer(task, client);
         // Simply propagate the training status events emitted by the client
-        this.#client.on('status', status => this.emit('status', status));
+        this.#client.on("status", (status) => this.emit("status", status));
     }
     /** Train on dataset, yielding logs of every round. */
     async *trainByRound(dataset) {
@@ -74,14 +76,14 @@ export class Disco extends EventEmitter {
         }
     }
     /** Train on dataset, yielding logs of every batch. */
-    async *trainByBatch(dataTuple) {
-        for await (const round of this.train(dataTuple))
+    async *trainByBatch(dataset) {
+        for await (const round of this.train(dataset))
             for await (const epoch of round)
                 yield* epoch;
     }
     /** Run whole train on dataset. */
-    async trainFully(dataTuple) {
-        for await (const round of this.train(dataTuple))
+    async trainFully(dataset) {
+        for await (const round of this.train(dataset))
             for await (const epoch of round)
                 for await (const _ of epoch)
                     ;
@@ -94,12 +96,11 @@ export class Disco extends EventEmitter {
      **/
     async *train(dataset) {
         this.#logger.success("Training started");
-        const data = await labeledDatasetToDataSplit(this.#task, dataset);
-        const trainData = data.train.preprocess().batch().dataset;
-        const validationData = data.validation?.preprocess().batch().dataset ?? trainData;
+        const [trainingDataset, validationDataset] = await this.#preprocessSplitAndBatch(dataset);
         // the client fetches the latest weights upon connection
-        this.trainer.model = await this.#client.connect();
-        for await (const [round, epochs] of enumerate(this.trainer.train(trainData, validationData))) {
+        // TODO unsafe cast
+        this.trainer.model = (await this.#client.connect());
+        for await (const [round, epochs] of enumerate(this.trainer.train(trainingDataset, validationDataset))) {
             yield async function* () {
                 const [gen, returnedRoundLogs] = split(epochs);
                 for await (const [epoch, batches] of enumerate(gen)) {
@@ -131,4 +132,22 @@ export class Disco extends EventEmitter {
     async close() {
         await this.#client.disconnect();
     }
+    async #preprocessSplitAndBatch(dataset) {
+        const { batchSize, validationSplit } = this.#task.trainingInformation;
+        const preprocessed = await processing.preprocess(this.#task, dataset);
+        const [training, validation] = (this.#preprocessOnce
+            ? new Dataset(await arrayFromAsync(preprocessed))
+            : preprocessed).split(validationSplit);
+        return [
+            training.batch(batchSize).cached(),
+            validation.batch(batchSize).cached(),
+        ];
+    }
+}
+// Array.fromAsync not yet widely used (2024)
+async function arrayFromAsync(iter) {
+    const ret = [];
+    for await (const e of iter)
+        ret.push(e);
+    return ret;
 }

package/dist/training/trainer.d.ts

@@ -1,17 +1,16 @@
-import * as tf from "@tensorflow/tfjs";
 import { List } from "immutable";
-import type { BatchLogs, EpochLogs, Model, Task } from "../index.js";
+import type { Batched, BatchLogs, Dataset, DataFormat, DataType, EpochLogs, Model, Task } from "../index.js";
 import { Client } from "../client/index.js";
 export interface RoundLogs {
     epochs: List<EpochLogs>;
     participants: number;
 }
 /** Train a model and exchange with others **/
-export declare class Trainer {
+export declare class Trainer<D extends DataType> {
     #private;
-    get model(): Model;
-    set model(model: Model);
-    constructor(task: Task, client: Client);
+    get model(): Model<D>;
+    set model(model: Model<D>);
+    constructor(task: Task<D>, client: Client);
     stopTraining(): Promise<void>;
-    train(dataset: tf.data.Dataset<tf.TensorContainer>, valDataset: tf.data.Dataset<tf.TensorContainer>): AsyncGenerator<AsyncGenerator<AsyncGenerator<BatchLogs, EpochLogs>, RoundLogs>, void>;
+    train(dataset: Dataset<Batched<DataFormat.ModelEncoded[D]>>, validationDataset?: Dataset<Batched<DataFormat.ModelEncoded[D]>>): AsyncGenerator<AsyncGenerator<AsyncGenerator<BatchLogs, EpochLogs>, RoundLogs>, void>;
 }

package/dist/training/trainer.js

@@ -29,23 +29,23 @@ export class Trainer {
     async stopTraining() {
         await this.#training?.return();
     }
-    async *train(dataset, valDataset) {
+    async *train(dataset, validationDataset) {
         if (this.#training !== undefined)
             throw new Error("training already running, stop it before launching a new one");
         try {
-            this.#training = this.#runRounds(dataset, valDataset);
+            this.#training = this.#runRounds(dataset, validationDataset);
             yield* this.#training;
         }
         finally {
             this.#training = undefined;
         }
     }
-    async *#runRounds(dataset, valDataset) {
+    async *#runRounds(dataset, validationDataset) {
         const totalRound = Math.trunc(this.#epochs / this.#roundDuration);
         let previousRoundWeights;
         for (let round = 0; round < totalRound; round++) {
             await this.#client.onRoundBeginCommunication();
-            yield this.#runRound(dataset, valDataset);
+            yield this.#runRound(dataset, validationDataset);
             let localWeights = this.model.weights;
             if (this.#privacy !== undefined)
                 localWeights = await applyPrivacy(previousRoundWeights, localWeights, this.#privacy);
@@ -53,10 +53,10 @@ export class Trainer {
             this.model.weights = previousRoundWeights = networkWeights;
         }
     }
-    async *#runRound(dataset, valDataset) {
+    async *#runRound(dataset, validationDataset) {
         let epochsLogs = List();
         for (let epoch = 0; epoch < this.#roundDuration; epoch++) {
-            const [gen, epochLogs] = async_iterator.split(this.model.train(dataset, valDataset));
+            const [gen, epochLogs] = async_iterator.split(this.model.train(dataset, validationDataset));
             yield gen;
             epochsLogs = epochsLogs.push(await epochLogs);
         }

package/dist/types/data_format.d.ts

@@ -0,0 +1,40 @@
+import { List } from "immutable";
+import type { Image, processing, Tabular, Text } from "../index.js";
+/**
+ * The data & label format goes through various stages.
+ * Raw* is preprocessed into ModelEncoded.
+ * ModelEncoded's labels are postprocess into Inferred.
+ *
+ * Raw* -> ModelEncoded -> Inferred
+ */
+/** what gets ingested by Disco */
+export interface Raw {
+    image: [Image, label: string];
+    tabular: Tabular;
+    text: Text;
+}
+/** what gets ingested by the Validator */
+export interface RawWithoutLabel {
+    image: Image;
+    tabular: Tabular;
+    text: Text;
+}
+type Token = number;
+/**
+ * what model can understand
+ *
+ * training needs data & label input
+ * prediction needs data input and outputs label
+ **/
+export interface ModelEncoded {
+    image: [image: processing.NormalizedImage<3>, label: number];
+    tabular: [row: List<number>, number];
+    text: [line: List<Token>, next: Token];
+}
+/** what gets outputted by the Validator, for humans */
+export interface Inferred {
+    image: string;
+    tabular: number;
+    text: string;
+}
+export {};

package/dist/types/index.d.ts

@@ -0,0 +1,2 @@
+export * as DataFormat from "./data_format.js";
+export type DataType = "image" | "tabular" | "text";

package/dist/types/index.js

@@ -0,0 +1 @@
+export * as DataFormat from "./data_format.js";

package/dist/validator.d.ts

@@ -0,0 +1,10 @@
+import type { Dataset, DataFormat, DataType, Model, Task } from "./index.js";
+export declare class Validator<D extends DataType> {
+    #private;
+    readonly task: Task<D>;
+    constructor(task: Task<D>, model: Model<D>);
+    /** infer every line of the dataset and check that it is as labelled */
+    test(dataset: Dataset<DataFormat.Raw[D]>): AsyncGenerator<boolean, void>;
+    /** use the model to predict every line of the dataset */
+    infer(dataset: Dataset<DataFormat.RawWithoutLabel[D]>): AsyncGenerator<DataFormat.Inferred[D], void>;
+}

package/dist/validator.js

@@ -0,0 +1,30 @@
+import { processing } from "./index.js";
+export class Validator {
+    task;
+    #model;
+    constructor(task, model) {
+        this.task = task;
+        this.#model = model;
+    }
+    /** infer every line of the dataset and check that it is as labelled */
+    async *test(dataset) {
+        const results = (await processing.preprocess(this.task, dataset))
+            .batch(this.task.trainingInformation.batchSize)
+            .map(async (batch) => (await this.#model.predict(batch.map(([inputs, _]) => inputs)))
+            .zip(batch.map(([_, outputs]) => outputs))
+            .map(([inferred, truth]) => inferred === truth))
+            .unbatch();
+        for await (const e of results)
+            yield e;
+    }
+    /** use the model to predict every line of the dataset */
+    async *infer(dataset) {
+        const modelPredictions = (await processing.preprocessWithoutLabel(this.task, dataset))
+            .batch(this.task.trainingInformation.batchSize)
+            .map((batch) => this.#model.predict(batch))
+            .unbatch();
+        const predictions = await processing.postprocess(this.task, modelPredictions);
+        for await (const e of predictions)
+            yield e;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epfml/discojs",
-  "version": "3.0.1-p20241025115642.0",
+  "version": "3.0.1-p20241107104659.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -19,6 +19,8 @@
   },
   "homepage": "https://github.com/epfml/disco#readme",
   "dependencies": {
+    "@jimp/core": "1",
+    "@jimp/plugin-resize": "1",
     "@msgpack/msgpack": "^3.0.0-beta2",
     "@tensorflow/tfjs": "4",
     "@xenova/transformers": "2",
@@ -31,7 +33,7 @@
   },
   "devDependencies": {
     "@tensorflow/tfjs-node": "4",
-    "@types/chai": "4",
+    "@types/chai": "5",
     "@types/mocha": "10",
     "@types/simple-peer": "9",
     "chai": "5",

package/dist/dataset/data/data.d.ts

@@ -1,47 +0,0 @@
-import * as tf from '@tensorflow/tfjs';
-import type { List } from 'immutable';
-import type { Task } from '../../index.js';
-import type { PreprocessingFunction } from './preprocessing/base.js';
-/**
- * Abstract class representing an immutable Disco dataset, including a TF.js dataset,
- * Disco task and set of preprocessing functions.
- */
-export declare abstract class Data {
-    readonly dataset: tf.data.Dataset<tf.TensorContainer>;
-    readonly task: Task;
-    readonly size?: number | undefined;
-    abstract readonly availablePreprocessing: List<PreprocessingFunction>;
-    protected constructor(dataset: tf.data.Dataset<tf.TensorContainer>, task: Task, size?: number | undefined);
-    static init(_dataset: tf.data.Dataset<tf.TensorContainer>, _task: Task, _size?: number): Promise<Data>;
-    /**
-     * Callable abstract method instead of constructor.
-     */
-    protected abstract create(dataset: tf.data.Dataset<tf.TensorContainer>, task: Task, size?: number): Data;
-    /**
-     * Creates a new Disco data object containing the batched TF.js dataset, according to the
-     * task's parameters.
-     * @returns The batched Disco data
-     */
-    batch(): Data;
-    /**
-     * The TF.js dataset batched according to the task's parameters.
-     */
-    get batchedDataset(): tf.data.Dataset<tf.TensorContainer>;
-    /**
-     * Creates a new Disco data object containing the preprocessed TF.js dataset,
-     * according to the defined set of preprocessing functions and the task's parameters.
-     * @returns The preprocessed Disco data
-     */
-    preprocess(): Data;
-    /**
-     * Creates a higher level preprocessing function applying the specified set of preprocessing
-     * functions in a series. The preprocessing functions are chained according to their defined
-     * priority.
-     */
-    get preprocessing(): (entry: tf.TensorContainer) => Promise<tf.TensorContainer>;
-    /**
-     * The TF.js dataset preprocessing according to the set of preprocessing functions and the task's
-     * parameters.
-     */
-    get preprocessedDataset(): tf.data.Dataset<tf.TensorContainer>;
-}

package/dist/dataset/data/data.js

@@ -1,88 +0,0 @@
-import * as tf from '@tensorflow/tfjs';
-/**
- * Abstract class representing an immutable Disco dataset, including a TF.js dataset,
- * Disco task and set of preprocessing functions.
- */
-export class Data {
-    dataset;
-    task;
-    size;
-    constructor(dataset, task, size) {
-        this.dataset = dataset;
-        this.task = task;
-        this.size = size;
-    }
-    static init(_dataset, _task, _size) {
-        return Promise.reject(new Error('abstract'));
-    }
-    /**
-     * Creates a new Disco data object containing the batched TF.js dataset, according to the
-     * task's parameters.
-     * @returns The batched Disco data
-     */
-    batch() {
-        return this.create(this.batchedDataset, this.task, this.size);
-    }
-    /**
-     * The TF.js dataset batched according to the task's parameters.
-     */
-    get batchedDataset() {
-        const batchSize = this.task.trainingInformation.batchSize;
-        return batchSize === undefined
-            ? this.dataset
-            : this.dataset.batch(batchSize);
-    }
-    /**
-     * Creates a new Disco data object containing the preprocessed TF.js dataset,
-     * according to the defined set of preprocessing functions and the task's parameters.
-     * @returns The preprocessed Disco data
-     */
-    preprocess() {
-        return this.create(this.preprocessedDataset, this.task, this.size);
-    }
-    /**
-     * Creates a higher level preprocessing function applying the specified set of preprocessing
-     * functions in a series. The preprocessing functions are chained according to their defined
-     * priority.
-     */
-    get preprocessing() {
-        const params = this.task.trainingInformation;
-        const taskPreprocessing = params.preprocessingFunctions;
-        if (taskPreprocessing === undefined ||
-            taskPreprocessing.length === 0 ||
-            this.availablePreprocessing === undefined ||
-            this.availablePreprocessing.size === 0) {
-            return x => Promise.resolve(x);
-        }
-        const applyPreprocessing = this.availablePreprocessing
-            .filter((e) => e.type in taskPreprocessing)
-            .map((e) => e.apply);
-        const preprocessingChain = async (input) => {
-            let currentContainer = await input; // Start with the initial tensor container
-            for (const fn of applyPreprocessing) {
-                const next = await fn(Promise.resolve(currentContainer), this.task);
-                // dirty but kinda working way to dispose of converted tensors
-                if (typeof currentContainer === "object" && typeof next === "object") {
-                    if ("xs" in currentContainer &&
-                        "xs" in next &&
-                        currentContainer.xs !== next.xs)
-                        tf.dispose(currentContainer.xs);
-                    if ("ys" in currentContainer &&
-                        "ys" in next &&
-                        currentContainer.ys !== next.ys)
-                        tf.dispose(currentContainer.ys);
-                }
-                currentContainer = next;
-            }
-            return currentContainer; // Return the final tensor container
-        };
-        return async (entry) => await preprocessingChain(Promise.resolve(entry));
-    }
-    /**
-     * The TF.js dataset preprocessing according to the set of preprocessing functions and the task's
-     * parameters.
-     */
-    get preprocessedDataset() {
-        return this.dataset.mapAsync(this.preprocessing);
-    }
-}

package/dist/dataset/data/data_split.d.ts

@@ -1,8 +0,0 @@
-import type { Data } from './data.js';
-/**
- * Train-validation split of Disco data.
- */
-export interface DataSplit {
-    train: Data;
-    validation?: Data;
-}

package/dist/dataset/data/helpers.d.ts

@@ -1,10 +0,0 @@
-/** Internal functions to help with Dataset to Data/DataSplit conversion
- *
- * @todo rm when fully using Dataset
- */
-import type { Task, TypedDataset, TypedLabeledDataset } from "../../index.js";
-import { Data } from "./index.js";
-import { DataSplit } from "./data_split.js";
-export declare function datasetToData(task: Task, [t, dataset]: TypedDataset): Promise<Data>;
-export declare function labeledDatasetToData(task: Task, [t, dataset]: TypedLabeledDataset): Promise<Data>;
-export declare function labeledDatasetToDataSplit(task: Task, [t, dataset]: TypedLabeledDataset): Promise<DataSplit>;

package/dist/dataset/data/helpers.js

@@ -1,97 +0,0 @@
-/** Internal functions to help with Dataset to Data/DataSplit conversion
- *
- * @todo rm when fully using Dataset
- */
-import { List } from "immutable";
-import * as tf from "@tensorflow/tfjs";
-import { processing } from "../../index.js";
-import { ImageData, TabularData, TextData } from "./index.js";
-function intoTFDataset(iter) {
-    // @ts-expect-error generator
-    return tf.data.generator(async function* () {
-        yield* iter;
-    });
-}
-function imageToTensor(image) {
-    return tf.tensor3d(image.data, [image.width, image.height, 3], "int32");
-}
-function tabularToNumbers(columns, row) {
-    return List(columns)
-        .map((column) => processing.extractColumn(row, column))
-        .map((v) => (v !== "" ? v : "0")) // TODO how to specify defaults?
-        .map(processing.convertToNumber)
-        .toArray();
-}
-export async function datasetToData(task, [t, dataset]) {
-    switch (t) {
-        case "image": {
-            const converted = dataset
-                .map(processing.removeAlpha)
-                .map((image) => processing.expandToMulticolor(image))
-                .map((image) => ({
-                xs: imageToTensor(image),
-            }));
-            return await ImageData.init(intoTFDataset(converted), task);
-        }
-        case "tabular": {
-            const inputColumns = task.trainingInformation.inputColumns;
-            if (inputColumns === undefined)
-                throw new Error("tabular task without input columns");
-            const converted = dataset.map((row) => ({
-                xs: tabularToNumbers(inputColumns, row),
-            }));
-            return await TabularData.init(intoTFDataset(converted), task);
-        }
-        case "text":
-            return await TextData.init(intoTFDataset(dataset), task);
-    }
-}
-export async function labeledDatasetToData(task, [t, dataset]) {
-    switch (t) {
-        case "image": {
-            const labels = List(task.trainingInformation.LABEL_LIST);
-            const converted = dataset
-                .map(([image, label]) => [
-                processing.expandToMulticolor(processing.removeAlpha(image)),
-                processing.indexInList(label, labels),
-            ])
-                .map(([image, label]) => ({
-                xs: imageToTensor(image),
-                ys: tf.oneHot(label, labels.size, 1, 0, "int32"),
-            }));
-            return await ImageData.init(intoTFDataset(converted), task);
-        }
-        case "tabular": {
-            const { inputColumns, outputColumns } = task.trainingInformation;
-            if (inputColumns === undefined || outputColumns === undefined)
-                throw new Error("tabular task without input and output columns");
-            const converted = dataset.map((row) => ({
-                xs: tabularToNumbers(inputColumns, row),
-                ys: tf.tensor1d(tabularToNumbers(outputColumns, row)),
-            }));
-            return await TabularData.init(intoTFDataset(converted), task);
-        }
-        case "text":
-            return await TextData.init(intoTFDataset(dataset), task);
-    }
-}
-export async function labeledDatasetToDataSplit(task, [t, dataset]) {
-    const split = task.trainingInformation.validationSplit;
-    let train;
-    let validation;
-    switch (t) {
-        case "image": {
-            [train, validation] = await Promise.all(dataset.split(split).map((d) => labeledDatasetToData(task, [t, d])));
-            break;
-        }
-        case "tabular": {
-            [train, validation] = await Promise.all(dataset.split(split).map((d) => labeledDatasetToData(task, [t, d])));
-            break;
-        }
-        case "text": {
-            [train, validation] = await Promise.all(dataset.split(split).map((d) => labeledDatasetToData(task, [t, d])));
-            break;
-        }
-    }
-    return { train, validation };
-}

package/dist/dataset/data/image_data.d.ts

@@ -1,11 +0,0 @@
-import * as tf from '@tensorflow/tfjs';
-import type { Task } from '../../index.js';
-import { Data } from './data.js';
-/**
- * Disco data made of image samples (.jpg, .png, etc.).
- */
-export declare class ImageData extends Data {
-    readonly availablePreprocessing: import("immutable").List<import("./preprocessing/base.js").PreprocessingFunction>;
-    static init(dataset: tf.data.Dataset<tf.TensorContainer>, task: Task, size?: number): Promise<Data>;
-    protected create(dataset: tf.data.Dataset<tf.TensorContainer>, task: Task, size: number): ImageData;
-}