@epfml/discojs-web 2.1.2-p20240506085037.0

package/dist/data/image_loader.d.ts

@@ -0,0 +1,5 @@
+import * as tf from '@tensorflow/tfjs';
+import { data } from '@epfml/discojs';
+export declare class ImageLoader extends data.ImageLoader<File> {
+    readImageFrom(source: File): Promise<tf.Tensor3D>;
+}

package/dist/data/image_loader.js

@@ -0,0 +1,7 @@
+import * as tf from '@tensorflow/tfjs';
+import { data } from '@epfml/discojs';
+export class ImageLoader extends data.ImageLoader {
+    async readImageFrom(source) {
+        return tf.browser.fromPixels(await createImageBitmap(source));
+    }
+}

package/dist/data/index.d.ts

@@ -0,0 +1,3 @@
+export { ImageLoader as WebImageLoader } from './image_loader.js';
+export { TabularLoader as WebTabularLoader } from './tabular_loader.js';
+export { TextLoader as WebTextLoader } from './text_loader.js';

package/dist/data/index.js

@@ -0,0 +1,3 @@
+export { ImageLoader as WebImageLoader } from './image_loader.js';
+export { TabularLoader as WebTabularLoader } from './tabular_loader.js';
+export { TextLoader as WebTextLoader } from './text_loader.js';

package/dist/data/tabular_loader.d.ts

@@ -0,0 +1,4 @@
+import { data } from '@epfml/discojs';
+export declare class TabularLoader extends data.TabularLoader<File> {
+    loadDatasetFrom(source: File, csvConfig: Record<string, unknown>): Promise<data.Dataset>;
+}

package/dist/data/tabular_loader.js

@@ -0,0 +1,8 @@
+import * as tf from '@tensorflow/tfjs';
+import { data } from '@epfml/discojs';
+export class TabularLoader extends data.TabularLoader {
+    async loadDatasetFrom(source, csvConfig) {
+        const file = new tf.data.FileDataSource(source);
+        return Promise.resolve(new tf.data.CSVDataset(file, csvConfig));
+    }
+}

package/dist/data/text_loader.d.ts

@@ -0,0 +1,4 @@
+import { data } from '@epfml/discojs';
+export declare class TextLoader extends data.TextLoader<File> {
+    loadDatasetFrom(source: File): Promise<data.Dataset>;
+}

package/dist/data/text_loader.js

@@ -0,0 +1,9 @@
+import * as tf from '@tensorflow/tfjs';
+import { data } from '@epfml/discojs';
+export class TextLoader extends data.TextLoader {
+    loadDatasetFrom(source) {
+        const file = new tf.data.FileDataSource(source);
+        const dataset = new tf.data.TextLineDataset(file).filter(s => s != ' '); // newline creates empty strings
+        return Promise.resolve(dataset);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export * from './data/index.js';
+export * from './memory/index.js';

package/dist/index.js

@@ -0,0 +1,2 @@
+export * from './data/index.js';
+export * from './memory/index.js';

package/dist/memory/index.d.ts

@@ -0,0 +1 @@
+export { IndexedDB } from './memory.js';

package/dist/memory/index.js

@@ -0,0 +1 @@
+export { IndexedDB } from './memory.js';

package/dist/memory/memory.d.ts

@@ -0,0 +1,31 @@
+import * as tf from '@tensorflow/tfjs';
+import type { Path, Model, ModelInfo, ModelSource } from '@epfml/discojs';
+import { Memory } from '@epfml/discojs';
+export declare class IndexedDB extends Memory {
+    pathFor(source: ModelSource): Path;
+    infoFor(source: ModelSource): ModelInfo;
+    getModelMetadata(source: ModelSource): Promise<tf.io.ModelArtifactsInfo | undefined>;
+    contains(source: ModelSource): Promise<boolean>;
+    getModel(source: ModelSource): Promise<Model>;
+    deleteModel(source: ModelSource): Promise<void>;
+    loadModel(source: ModelSource): Promise<void>;
+    /**
+     * Saves the working model to the source.
+     * @param source the destination
+     * @param model the model
+     */
+    updateWorkingModel(source: ModelSource, model: Model): Promise<void>;
+    /**
+   * Creates a saved copy of the working model corresponding to the source.
+   * @param source the source
+   */
+    saveWorkingModel(source: ModelSource): Promise<Path>;
+    saveModel(source: ModelSource, model: Model): Promise<Path>;
+    /**
+   * Downloads the model corresponding to the source.
+   * @param source the source
+   */
+    downloadModel(source: ModelSource): Promise<void>;
+    latestDuplicate(source: ModelSource): Promise<number | undefined>;
+    duplicateSource(source: ModelSource): Promise<ModelInfo>;
+}

package/dist/memory/memory.js

@@ -0,0 +1,134 @@
+/**
+ * Helper functions used to load and save TFJS models from IndexedDB. The
+ * working model is the model currently being trained for a task. Saved models
+ * are models that were explicitly saved to IndexedDB. The two working/ and saved/
+ * folders are invisible to the user. The user only interacts with the saved/
+ * folder via the model library. The working/ folder is only used by the backend.
+ * The working model is loaded from IndexedDB for training (model.fit) only.
+ */
+import { Map } from 'immutable';
+import * as tf from '@tensorflow/tfjs';
+import { Memory, ModelType, models } from '@epfml/discojs';
+export class IndexedDB extends Memory {
+    pathFor(source) {
+        if (typeof source === 'string') {
+            return source;
+        }
+        if (source.type === undefined || source.taskID === undefined || source.name === undefined) {
+            throw new TypeError('source incomplete');
+        }
+        const version = source.version ?? 0;
+        return `indexeddb://${source.type}/${source.taskID}/${source.name}@${version}`;
+    }
+    infoFor(source) {
+        if (typeof source !== 'string') {
+            return source;
+        }
+        const [stringType, taskID, fullName] = source.split('/').splice(2);
+        const type = stringType === 'working' ? ModelType.WORKING : ModelType.SAVED;
+        const [name, versionSuffix] = fullName.split('@');
+        const version = versionSuffix === undefined ? 0 : Number(versionSuffix);
+        return { type, taskID, name, version };
+    }
+    async getModelMetadata(source) {
+        const models = await tf.io.listModels();
+        return models[this.pathFor(source)];
+    }
+    async contains(source) {
+        return await this.getModelMetadata(source) !== undefined;
+    }
+    async getModel(source) {
+        return new models.TFJS(await tf.loadLayersModel(this.pathFor(source)));
+    }
+    async deleteModel(source) {
+        await tf.io.removeModel(this.pathFor(source));
+    }
+    async loadModel(source) {
+        const src = this.infoFor(source);
+        if (src.type === ModelType.WORKING) {
+            // Model is already loaded
+            return;
+        }
+        await tf.io.copyModel(this.pathFor(src), this.pathFor({ ...src, type: ModelType.WORKING, version: 0 }));
+    }
+    /**
+     * Saves the working model to the source.
+     * @param source the destination
+     * @param model the model
+     */
+    async updateWorkingModel(source, model) {
+        const src = this.infoFor(source);
+        if (src.type !== undefined && src.type !== ModelType.WORKING) {
+            throw new Error('expected working model');
+        }
+        if (model instanceof models.TFJS) {
+            await model.extract().save(this.pathFor({ ...src, type: ModelType.WORKING, version: 0 }), { includeOptimizer: true });
+        }
+        else {
+            throw new Error('unknown model type');
+        }
+        // Enforce version 0 to always keep a single working model at a time
+    }
+    /**
+   * Creates a saved copy of the working model corresponding to the source.
+   * @param source the source
+   */
+    async saveWorkingModel(source) {
+        const src = this.infoFor(source);
+        if (src.type !== undefined && src.type !== ModelType.WORKING) {
+            throw new Error('expected working model');
+        }
+        const dst = this.pathFor(await this.duplicateSource({ ...src, type: ModelType.SAVED }));
+        await tf.io.copyModel(this.pathFor({ ...src, type: ModelType.WORKING }), dst);
+        return dst;
+    }
+    async saveModel(source, model) {
+        const src = this.infoFor(source);
+        if (src.type !== undefined && src.type !== ModelType.SAVED) {
+            throw new Error('expected saved model');
+        }
+        const dst = this.pathFor(await this.duplicateSource({ ...src, type: ModelType.SAVED }));
+        if (model instanceof models.TFJS) {
+            await model.extract().save(dst, { includeOptimizer: true });
+        }
+        else {
+            throw new Error('unknown model type');
+        }
+        return dst;
+    }
+    /**
+   * Downloads the model corresponding to the source.
+   * @param source the source
+   */
+    async downloadModel(source) {
+        const src = this.infoFor(source);
+        await tf.io.copyModel(this.pathFor(source), `downloads://${src.taskID}_${src.name}`);
+    }
+    async latestDuplicate(source) {
+        if (typeof source !== 'string') {
+            source = this.pathFor({ ...source, version: 0 });
+        }
+        // perform a single memory read
+        const paths = Map(await tf.io.listModels());
+        if (!paths.has(source)) {
+            return undefined;
+        }
+        const latest = Map(paths)
+            .keySeq()
+            .toList()
+            .map((p) => this.infoFor(p).version)
+            .max();
+        if (latest === undefined) {
+            return 0;
+        }
+        return latest;
+    }
+    async duplicateSource(source) {
+        const latestDuplicate = await this.latestDuplicate(source);
+        source = this.infoFor(source);
+        if (latestDuplicate === undefined) {
+            return source;
+        }
+        return { ...source, version: latestDuplicate + 1 };
+    }
+}

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@epfml/discojs-web",
+  "version": "2.1.2-p20240506085037.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "watch": "nodemon --ext ts --ignore dist --watch ../discojs/dist --watch . --exec npm run",
+    "build": "tsc",
+    "lint": "npx eslint .",
+    "test": ": nothing"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/epfml/disco.git"
+  },
+  "bugs": {
+    "url": "https://github.com/epfml/disco/issues"
+  },
+  "homepage": "https://github.com/epfml/disco#readme",
+  "dependencies": {
+    "@epfml/discojs": "*",
+    "@tensorflow/tfjs": "4"
+  },
+  "devDependencies": {
+    "nodemon": "3"
+  }
+}