@epfml/discojs 3.0.1-p20241025115642.0 → 3.0.1-p20241107104659.0

package/dist/processing/image.js

@@ -0,0 +1,75 @@
+import { Repeat, Seq } from "immutable";
+import { createJimp } from "@jimp/core";
+import * as jimpResize from "@jimp/plugin-resize";
+import { Image } from "../index.js";
+/** Image where intensity is represented in the range 0..1 */
+export class NormalizedImage {
+    data;
+    width;
+    height;
+    depth;
+    // private as it doesn't check that array content is valid
+    constructor(data, width, height, depth) {
+        this.data = data;
+        this.width = width;
+        this.height = height;
+        this.depth = depth;
+        if (data.length != width * height * depth)
+            throw new Error("data isn't of expected size");
+    }
+    static from(image) {
+        return new NormalizedImage(Float32Array.from(image.data).map((v) => v / 255), image.width, image.height, image.depth);
+    }
+}
+/** Add a full opaque alpha channel to an image */
+function addAlpha(image) {
+    switch (image.depth) {
+        case 3:
+            return new Image(Uint8Array.from(
+            // we are adding a channel, so for every 3 byte in the base image,
+            // we need to add a fourth. we choose to "expand" the last channel
+            // to two value, the channel base value and the transparency.
+            // let's say we want to add a byte A to the bytestring RGB
+            // [R, G, B] -> [[R], [G], [B, A]] -> [R, G, B, A]
+            Seq(image.data).flatMap((v, i) => {
+                const OPAQUE = 0xff;
+                if (i % 3 !== 2)
+                    return [v];
+                else
+                    return [v, OPAQUE];
+            })), image.width, image.height, 4);
+        case 4:
+            return new Image(image.data, image.width, image.height, image.depth);
+    }
+}
+export function removeAlpha(image) {
+    switch (image.depth) {
+        case 1:
+        case 3:
+            return new Image(image.data, image.width, image.height, image.depth);
+        case 4:
+            return new Image(image.data.filter((_, i) => i % 4 !== 3), image.width, image.height, 3);
+    }
+}
+export function expandToMulticolor(image) {
+    switch (image.depth) {
+        case 1:
+            return new Image(Uint8Array.from(Seq(image.data).flatMap((v) => Repeat(v, 3))), image.width, image.height, 3);
+        case 3:
+        case 4:
+            return new Image(image.data, image.width, image.height, image.depth);
+    }
+}
+export function resize(width, height, image) {
+    const Jimp = createJimp({
+        plugins: [jimpResize.methods],
+    });
+    const resized = new Jimp(addAlpha(expandToMulticolor(image))).resize({
+        w: width,
+        h: height,
+    });
+    return new Image(new Uint8Array(resized.bitmap.data), width, height, 4);
+}
+export function normalize(image) {
+    return NormalizedImage.from(image);
+}

package/dist/processing/index.d.ts

@@ -0,0 +1,8 @@
+/** Dataset shapers, convenient to map with */
+import type { Dataset, DataFormat, DataType, Task } from "../index.js";
+export * from "./image.js";
+export * from "./tabular.js";
+export * from "./text.js";
+export declare function preprocess<D extends DataType>(task: Task<D>, dataset: Dataset<DataFormat.Raw[D]>): Promise<Dataset<DataFormat.ModelEncoded[D]>>;
+export declare function preprocessWithoutLabel<D extends DataType>(task: Task<D>, dataset: Dataset<DataFormat.RawWithoutLabel[D]>): Promise<Dataset<DataFormat.ModelEncoded[D][0]>>;
+export declare function postprocess<D extends DataType>(task: Task<D>, dataset: Dataset<DataFormat.ModelEncoded[D][1]>): Promise<Dataset<DataFormat.Inferred[D]>>;

package/dist/processing/index.js

@@ -0,0 +1,106 @@
+/** Dataset shapers, convenient to map with */
+import { List } from "immutable";
+import { models } from "../index.js";
+import * as processing from "./index.js";
+export * from "./image.js";
+export * from "./tabular.js";
+export * from "./text.js";
+export async function preprocess(task, dataset) {
+    switch (task.trainingInformation.dataType) {
+        case "image": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            const { IMAGE_H, IMAGE_W, LABEL_LIST } = task.trainingInformation;
+            return d.map(([image, label]) => [
+                processing.normalize(processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image))),
+                processing.indexInList(label, LABEL_LIST),
+            ]);
+        }
+        case "tabular": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            const { inputColumns, outputColumn } = task.trainingInformation;
+            return d.map((row) => {
+                const output = processing.extractColumn(row, outputColumn);
+                return [
+                    extractToNumbers(inputColumns, row),
+                    // TODO sanitization doesn't care about column distribution
+                    output !== "" ? processing.convertToNumber(output) : 0,
+                ];
+            });
+        }
+        case "text": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            const t = task;
+            const tokenizer = await models.getTaskTokenizer(t);
+            const totalTokenCount = task.trainingInformation.maxSequenceLength ??
+                tokenizer.model_max_length;
+            return d
+                .map((line) => processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount))
+                .map((tokens) => [tokens.pop(), tokens.last()]);
+        }
+    }
+}
+export async function preprocessWithoutLabel(task, dataset) {
+    switch (task.trainingInformation.dataType) {
+        case "image": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            const { IMAGE_H, IMAGE_W } = task.trainingInformation;
+            return d.map((image) => processing.normalize(processing.removeAlpha(processing.resize(IMAGE_W, IMAGE_H, image))));
+        }
+        case "tabular": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            const { inputColumns } = task.trainingInformation;
+            return d.map((row) => extractToNumbers(inputColumns, row));
+        }
+        case "text": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            const t = task;
+            const tokenizer = await models.getTaskTokenizer(t);
+            const totalTokenCount = t.trainingInformation.maxSequenceLength ??
+                tokenizer.model_max_length;
+            return d
+                .map((line) => processing.tokenizeAndLeftPad(line, tokenizer, totalTokenCount))
+                .map((tokens) => tokens.pop());
+        }
+    }
+}
+export async function postprocess(task, dataset) {
+    switch (task.trainingInformation.dataType) {
+        case "image": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            const { LABEL_LIST } = task.trainingInformation;
+            const labels = List(LABEL_LIST);
+            return d.map((index) => {
+                const v = labels.get(index);
+                if (v === undefined)
+                    throw new Error("index not found in labels");
+                return v;
+            });
+        }
+        case "tabular": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            return d;
+        }
+        case "text": {
+            // cast as typescript doesn't reduce generic type
+            const d = dataset;
+            const t = task;
+            const tokenizer = await models.getTaskTokenizer(t);
+            return d.map((token) => tokenizer.decode([token]));
+        }
+    }
+}
+function extractToNumbers(columns, row) {
+    return (List(columns)
+        .map((column) => processing.extractColumn(row, column))
+        // TODO sanitization doesn't care about column distribution
+        .map((v) => (v !== "" ? v : "0"))
+        .map(processing.convertToNumber));
+}

package/dist/processing/tabular.d.ts

@@ -0,0 +1,19 @@
+import { List } from "immutable";
+/**
+ * Convert a string to a number
+ *
+ * @throws if it isn't written as a number
+ */
+export declare function convertToNumber(raw: string): number;
+/**
+ * Return the named field of an object with string values
+ *
+ * @throws if the named field isn't there
+ */
+export declare function extractColumn(row: Partial<Record<string, string>>, column: string): string;
+/**
+ * Return the index of the element in the given list
+ *
+ * @throws if not found
+ */
+export declare function indexInList(element: string, elements: List<string> | Array<string>): number;

package/dist/processing/tabular.js

@@ -0,0 +1,33 @@
+/**
+ * Convert a string to a number
+ *
+ * @throws if it isn't written as a number
+ */
+export function convertToNumber(raw) {
+    const num = Number.parseFloat(raw);
+    if (Number.isNaN(num))
+        throw new Error(`unable to parse "${raw}" as number`);
+    return num;
+}
+/**
+ * Return the named field of an object with string values
+ *
+ * @throws if the named field isn't there
+ */
+export function extractColumn(row, column) {
+    const raw = row[column];
+    if (raw === undefined)
+        throw new Error(`${column} not found in row`);
+    return raw;
+}
+/**
+ * Return the index of the element in the given list
+ *
+ * @throws if not found
+ */
+export function indexInList(element, elements) {
+    const ret = elements.indexOf(element);
+    if (ret === -1)
+        throw new Error(`${element} not found in list`);
+    return ret;
+}

package/dist/processing/text.d.ts

@@ -0,0 +1,11 @@
+import { List } from "immutable";
+import { PreTrainedTokenizer } from "@xenova/transformers";
+type Token = number;
+/**
+ * Tokenize and truncates input strings
+ *
+ * @param length number of tokens
+ * @returns encoded string in an array of token, size of max_length
+ */
+export declare function tokenizeAndLeftPad(line: string, tokenizer: PreTrainedTokenizer, length: number): List<Token>;
+export {};

package/dist/processing/text.js

@@ -0,0 +1,33 @@
+import { Repeat } from "immutable";
+function isArrayOfNumber(raw) {
+    return Array.isArray(raw) && raw.every((e) => typeof e === "number");
+}
+/**
+ * Tokenize and truncates input strings
+ *
+ * @param length number of tokens
+ * @returns encoded string in an array of token, size of max_length
+ */
+export function tokenizeAndLeftPad(line, tokenizer, length) {
+    if (!Number.isInteger(length))
+        throw new Error("length should be an integer");
+    // Transformers.js currently only supports right padding while we need left for text generation
+    // Right padding should be supported in the future, once it is, we can directly pad while tokenizing
+    // https://github.com/xenova/transformers.js/blob/8804c36591d11d8456788d1bb4b16489121b3be2/src/tokenizers.js#L2517
+    const tokenized = tokenizer(line, {
+        padding: false,
+        truncation: true,
+        return_tensor: false,
+        max_length: length,
+    });
+    if (typeof tokenized !== "object" ||
+        tokenized === null ||
+        !("input_ids" in tokenized) ||
+        !isArrayOfNumber(tokenized.input_ids))
+        throw new Error("tokenizer returns unexpected type");
+    const tokens = tokenized.input_ids;
+    const paddingSize = length - tokens.length;
+    if (paddingSize < 0)
+        throw new Error("tokenized returned more token than expected");
+    return Repeat(tokenizer.pad_token_id, paddingSize).concat(tokens).toList();
+}

package/dist/serialization/model.d.ts

@@ -1,4 +1,4 @@
-import type { Model } from '../index.js';
+import type { DataType, Model } from '../index.js';
 import { Encoded } from "./coder.js";
-export declare function encode(model: Model): Promise<Encoded>;
-export declare function decode(encoded: unknown): Promise<Model>;
+export declare function encode(model: Model<DataType>): Promise<Encoded>;
+export declare function decode(encoded: unknown): Promise<Model<DataType>>;

package/dist/serialization/model.js

@@ -9,7 +9,7 @@ export async function encode(model) {
     switch (true) {
         case model instanceof models.TFJS: {
             const serialized = await model.serialize();
-            return coder.encode([Type.TFJS, serialized]);
+            return coder.encode([Type.TFJS, ...serialized]);
         }
         case model instanceof models.GPT: {
             const { weights, config } = model.serialize();
@@ -33,12 +33,25 @@ export async function decode(encoded) {
     }
     const rawModel = raw[1];
     switch (type) {
-        case Type.TFJS:
-            if (raw.length !== 2) {
-                throw new Error('invalid encoding, TFJS model encoding should be an array of length 2');
+        case Type.TFJS: {
+            if (raw.length !== 3)
+                throw new Error("invalid TFJS model encoding: should be an array of length 3");
+            const [rawDatatype, rawModel] = raw.slice(1);
+            let datatype;
+            switch (rawDatatype) {
+                case "image":
+                case "tabular":
+                    datatype = rawDatatype;
+                    break;
+                default:
+                    throw new Error("invalid TFJS model encoding: invalid DataType");
             }
-            // TODO totally unsafe casting
-            return await models.TFJS.deserialize(rawModel);
+            return await models.TFJS.deserialize([
+                datatype,
+                // TODO totally unsafe casting
+                rawModel,
+            ]);
+        }
         case Type.GPT: {
             let config;
             if (raw.length == 2) {

package/dist/task/task.d.ts

@@ -1,10 +1,11 @@
+import { DataType } from "../index.js";
 import { type DisplayInformation } from './display_information.js';
 import { type TrainingInformation } from './training_information.js';
 export type TaskID = string;
-export interface Task {
+export interface Task<D extends DataType> {
     id: TaskID;
     displayInformation: DisplayInformation;
-    trainingInformation: TrainingInformation;
+    trainingInformation: TrainingInformation<D>;
 }
 export declare function isTaskID(obj: unknown): obj is TaskID;
-export declare function isTask(raw: unknown): raw is Task;
+export declare function isTask(raw: unknown): raw is Task<DataType>;

package/dist/task/task.js

@@ -13,8 +13,10 @@ export function isTask(raw) {
         !isTrainingInformation(trainingInformation)) {
         return false;
     }
-    const repack = { id, displayInformation, trainingInformation };
-    const _correct = repack;
-    const _total = repack;
+    const _ = {
+        id,
+        displayInformation,
+        trainingInformation,
+    };
     return true;
 }

package/dist/task/task_handler.d.ts

@@ -1,5 +1,5 @@
 import { Map } from "immutable";
-import type { Model } from "../index.js";
+import type { DataType, Model } from "../index.js";
 import type { Task, TaskID } from "./task.js";
-export declare function pushTask(base: URL, task: Task, model: Model): Promise<void>;
-export declare function fetchTasks(base: URL): Promise<Map<TaskID, Task>>;
+export declare function pushTask<D extends DataType>(base: URL, task: Task<D>, model: Model<D>): Promise<void>;
+export declare function fetchTasks(base: URL): Promise<Map<TaskID, Task<DataType>>>;

package/dist/task/task_provider.d.ts

@@ -1,5 +1,5 @@
-import type { Model, Task } from '../index.js';
-export interface TaskProvider {
-    getTask: () => Task;
-    getModel: () => Promise<Model>;
+import type { DataType, Model, Task } from "../index.js";
+export interface TaskProvider<D extends DataType> {
+    getTask(): Task<D>;
+    getModel(): Promise<Model<D>>;
 }

package/dist/task/training_information.d.ts

@@ -1,29 +1,38 @@
-import type { Preprocessing } from '../dataset/data/preprocessing/index.js';
-import { PreTrainedTokenizer } from '@xenova/transformers';
+import { PreTrainedTokenizer } from "@xenova/transformers";
+import { DataType } from "../index.js";
 interface Privacy {
     clippingRadius?: number;
     noiseScale?: number;
 }
-export interface TrainingInformation {
+export type TrainingInformation<D extends DataType> = {
     epochs: number;
     roundDuration: number;
     validationSplit: number;
     batchSize: number;
-    preprocessingFunctions?: Preprocessing[];
-    dataType: 'image' | 'tabular' | 'text';
-    inputColumns?: string[];
-    outputColumns?: string[];
-    IMAGE_H?: number;
-    IMAGE_W?: number;
-    LABEL_LIST?: string[];
-    scheme: 'decentralized' | 'federated' | 'local';
+    scheme: "decentralized" | "federated" | "local";
     privacy?: Privacy;
     maxShareValue?: number;
     minNbOfParticipants: number;
-    aggregationStrategy?: 'mean' | 'secure';
-    tokenizer?: string | PreTrainedTokenizer;
-    maxSequenceLength?: number;
-    tensorBackend: 'tfjs' | 'gpt';
+    aggregationStrategy?: "mean" | "secure";
+    tensorBackend: "tfjs" | "gpt";
+} & DataTypeToTrainingInformation[D];
+interface DataTypeToTrainingInformation {
+    image: {
+        dataType: "image";
+        LABEL_LIST: string[];
+        IMAGE_H: number;
+        IMAGE_W: number;
+    };
+    tabular: {
+        dataType: "tabular";
+        inputColumns: string[];
+        outputColumn: string;
+    };
+    text: {
+        dataType: "text";
+        tokenizer: string | PreTrainedTokenizer;
+        maxSequenceLength?: number;
+    };
 }
-export declare function isTrainingInformation(raw: unknown): raw is TrainingInformation;
+export declare function isTrainingInformation(raw: unknown): raw is TrainingInformation<DataType>;
 export {};

package/dist/task/training_information.js

@@ -1,11 +1,4 @@
-import { PreTrainedTokenizer } from '@xenova/transformers';
-function isStringArray(raw) {
-    if (!Array.isArray(raw)) {
-        return false;
-    }
-    const arr = raw; // isArray is unsafely guarding with any[]
-    return arr.every((e) => typeof e === 'string');
-}
+import { PreTrainedTokenizer } from "@xenova/transformers";
 function isPrivacy(raw) {
     if (typeof raw !== "object" || raw === null) {
         return false;
@@ -21,89 +14,100 @@ function isPrivacy(raw) {
     return true;
 }
 export function isTrainingInformation(raw) {
-    if (typeof raw !== 'object' || raw === null) {
+    if (typeof raw !== "object" || raw === null) {
         return false;
     }
-    const { IMAGE_H, IMAGE_W, LABEL_LIST, aggregationStrategy, batchSize, dataType, privacy, epochs, inputColumns, maxShareValue, minNbOfParticipants, outputColumns, preprocessingFunctions, roundDuration, scheme, validationSplit, tokenizer, maxSequenceLength, tensorBackend } = raw;
-    if (typeof dataType !== 'string' ||
-        typeof epochs !== 'number' ||
-        typeof batchSize !== 'number' ||
-        typeof roundDuration !== 'number' ||
-        typeof validationSplit !== 'number' ||
-        typeof minNbOfParticipants !== 'number' ||
-        (tokenizer !== undefined && typeof tokenizer !== 'string' && !(tokenizer instanceof PreTrainedTokenizer)) ||
-        (maxSequenceLength !== undefined && typeof maxSequenceLength !== 'number') ||
-        (aggregationStrategy !== undefined && typeof aggregationStrategy !== 'string') ||
+    const { aggregationStrategy, batchSize, dataType, privacy, epochs, maxShareValue, minNbOfParticipants, roundDuration, scheme, validationSplit, tensorBackend, } = raw;
+    if (typeof epochs !== "number" ||
+        typeof batchSize !== "number" ||
+        typeof roundDuration !== "number" ||
+        typeof validationSplit !== "number" ||
+        typeof minNbOfParticipants !== "number" ||
         (privacy !== undefined && !isPrivacy(privacy)) ||
-        (maxShareValue !== undefined && typeof maxShareValue !== 'number') ||
-        (IMAGE_H !== undefined && typeof IMAGE_H !== 'number') ||
-        (IMAGE_W !== undefined && typeof IMAGE_W !== 'number') ||
-        (LABEL_LIST !== undefined && !isStringArray(LABEL_LIST)) ||
-        (inputColumns !== undefined && !isStringArray(inputColumns)) ||
-        (outputColumns !== undefined && !isStringArray(outputColumns)) ||
-        (preprocessingFunctions !== undefined && !Array.isArray(preprocessingFunctions))) {
+        (maxShareValue !== undefined && typeof maxShareValue !== "number")) {
         return false;
     }
-    if (aggregationStrategy !== undefined) {
-        switch (aggregationStrategy) {
-            case 'mean': break;
-            case 'secure': break;
-            default: return false;
-        }
-    }
-    switch (dataType) {
-        case 'image': break;
-        case 'tabular': break;
-        case 'text': break;
-        default: return false;
-    }
-    // interdependencies on data type
-    if (dataType === 'image') {
-        if (typeof IMAGE_H !== 'number' || typeof IMAGE_W !== 'number') {
-            return false;
-        }
-    }
-    else if (dataType in ['text', 'tabular']) {
-        if (!(Array.isArray(inputColumns) && inputColumns.every((e) => typeof e === 'string'))) {
-            return false;
-        }
-        if (!(Array.isArray(outputColumns) && outputColumns.every((e) => typeof e === 'string'))) {
+    switch (aggregationStrategy) {
+        case undefined:
+        case "mean":
+        case "secure":
+            break;
+        default:
             return false;
-        }
     }
     switch (tensorBackend) {
-        case 'tfjs': break;
-        case 'gpt': break;
-        default: return false;
+        case "tfjs":
+        case "gpt":
+            break;
+        default:
+            return false;
     }
     switch (scheme) {
-        case 'decentralized': break;
-        case 'federated': break;
-        case 'local': break;
-        default: return false;
+        case "decentralized":
+        case "federated":
+        case "local":
+            break;
+        default:
+            return false;
     }
     const repack = {
-        IMAGE_W,
-        IMAGE_H,
-        LABEL_LIST,
         aggregationStrategy,
         batchSize,
-        dataType,
-        privacy,
         epochs,
-        inputColumns,
         maxShareValue,
         minNbOfParticipants,
-        outputColumns,
-        preprocessingFunctions,
+        privacy,
         roundDuration,
         scheme,
+        tensorBackend,
         validationSplit,
-        tokenizer,
-        maxSequenceLength,
-        tensorBackend
     };
-    const _correct = repack;
-    const _total = repack;
-    return true;
+    switch (dataType) {
+        case "image": {
+            const { LABEL_LIST, IMAGE_W, IMAGE_H } = raw;
+            if (!(Array.isArray(LABEL_LIST) &&
+                LABEL_LIST.every((e) => typeof e === "string")) ||
+                typeof IMAGE_H !== "number" ||
+                typeof IMAGE_W !== "number")
+                return false;
+            const _ = {
+                ...repack,
+                dataType,
+                LABEL_LIST,
+                IMAGE_W,
+                IMAGE_H,
+            };
+            return true;
+        }
+        case "tabular": {
+            const { inputColumns, outputColumn } = raw;
+            if (!(Array.isArray(inputColumns) &&
+                inputColumns.every((e) => typeof e === "string")) ||
+                typeof outputColumn !== "string")
+                return false;
+            const _ = {
+                ...repack,
+                dataType,
+                inputColumns,
+                outputColumn,
+            };
+            return true;
+        }
+        case "text": {
+            const { maxSequenceLength, tokenizer, } = raw;
+            if ((typeof tokenizer !== "string" &&
+                !(tokenizer instanceof PreTrainedTokenizer)) ||
+                (maxSequenceLength !== undefined &&
+                    typeof maxSequenceLength !== "number"))
+                return false;
+            const _ = {
+                ...repack,
+                dataType,
+                maxSequenceLength,
+                tokenizer,
+            };
+            return true;
+        }
+    }
+    return false;
 }