@epfml/discojs-web 3.0.1-p20240902162912.0 → 3.0.1-p20240904094219.0

package/dist/index.d.ts

@@ -1,2 +1 @@
 export * from "./loaders/index.js";
-export * from "./memory/index.js";

package/dist/index.js

@@ -1,2 +1 @@
 export * from "./loaders/index.js";
-export * from "./memory/index.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epfml/discojs-web",
-  "version": "3.0.1-p20240902162912.0",
+  "version": "3.0.1-p20240904094219.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/dist/memory/index.d.ts

@@ -1 +0,0 @@
-export { IndexedDB } from './memory.js';

package/dist/memory/index.js

@@ -1 +0,0 @@
-export { IndexedDB } from './memory.js';

package/dist/memory/memory.d.ts

@@ -1,31 +0,0 @@
-import * as tf from '@tensorflow/tfjs';
-import type { Model, ModelInfo, ModelSource } from '@epfml/discojs';
-import { Memory } from '@epfml/discojs';
-export declare class IndexedDB extends Memory {
-    getModelMemoryPath(source: ModelSource): string;
-    getModelInfo(source: ModelSource): ModelInfo;
-    getModelMetadata(source: ModelSource): Promise<tf.io.ModelArtifactsInfo | undefined>;
-    contains(source: ModelSource): Promise<boolean>;
-    getModel(source: ModelSource): Promise<Model>;
-    deleteModel(source: ModelSource): Promise<void>;
-    loadModel(source: ModelSource): Promise<void>;
-    /**
-     * Saves the working model to the source.
-     * @param source the destination
-     * @param model the model
-     */
-    updateWorkingModel(source: ModelSource, model: Model): Promise<void>;
-    /**
-   * Creates a saved copy of the working model corresponding to the source.
-   * @param source the source
-   */
-    saveWorkingModel(source: ModelSource): Promise<string>;
-    saveModel(source: ModelSource, model: Model): Promise<string>;
-    /**
-   * Downloads the model corresponding to the source.
-   * @param source the source
-   */
-    downloadModel(source: ModelSource): Promise<void>;
-    latestDuplicate(source: ModelSource): Promise<number | undefined>;
-    duplicateSource(source: ModelSource): Promise<ModelInfo>;
-}

package/dist/memory/memory.js

@@ -1,165 +0,0 @@
-/**
- * Helper functions used to load and save TFJS models from IndexedDB. The
- * working model is the model currently being trained for a task. Saved models
- * are models that were explicitly saved to IndexedDB. The two working/ and saved/
- * folders are invisible to the user. The user only interacts with the saved/
- * folder via the model library. The working/ folder is only used by the backend.
- * The working model is loaded from IndexedDB for training (model.fit) only.
- */
-import { Map } from 'immutable';
-import createDebug from "debug";
-import * as tf from '@tensorflow/tfjs';
-import { Memory, models } from '@epfml/discojs';
-const debug = createDebug('discojs-web:memory');
-export class IndexedDB extends Memory {
-    getModelMemoryPath(source) {
-        if (typeof source === 'string') {
-            return source;
-        }
-        const version = source.version ?? 0;
-        return `indexeddb://${source.type}/${source.tensorBackend}/${source.taskID}/${source.name}@${version}`;
-    }
-    getModelInfo(source) {
-        if (typeof source !== 'string') {
-            return source;
-        }
-        const [type, tensorBackend, taskID, fullName] = source.split('/').splice(2);
-        if (type !== 'working' && type !== 'saved') {
-            throw Error("Unknown memory model type");
-        }
-        const [name, versionSuffix] = fullName.split('@');
-        const version = versionSuffix === undefined ? 0 : Number(versionSuffix);
-        if (tensorBackend !== 'tfjs' && tensorBackend !== 'gpt') {
-            throw Error("Unknown tensor backend");
-        }
-        return { type, taskID, name, version, tensorBackend };
-    }
-    async getModelMetadata(source) {
-        const models = await tf.io.listModels();
-        return models[this.getModelMemoryPath(source)];
-    }
-    async contains(source) {
-        return await this.getModelMetadata(source) !== undefined;
-    }
-    async getModel(source) {
-        const layersModel = await tf.loadLayersModel(this.getModelMemoryPath(source));
-        const tensorBackend = this.getModelInfo(source).tensorBackend;
-        switch (tensorBackend) {
-            case 'tfjs':
-                return new models.TFJS(layersModel);
-            case 'gpt':
-                return new models.GPT(undefined, layersModel);
-            default: {
-                const _ = tensorBackend;
-                throw new Error('should never happen');
-            }
-        }
-    }
-    async deleteModel(source) {
-        await tf.io.removeModel(this.getModelMemoryPath(source));
-    }
-    async loadModel(source) {
-        const src = this.getModelInfo(source);
-        if (src.type === 'working') {
-            // Model is already loaded
-            return;
-        }
-        await tf.io.copyModel(this.getModelMemoryPath(src), this.getModelMemoryPath({ ...src, type: 'working', version: 0 }));
-    }
-    /**
-     * Saves the working model to the source.
-     * @param source the destination
-     * @param model the model
-     */
-    async updateWorkingModel(source, model) {
-        const src = this.getModelInfo(source);
-        if (src.type !== 'working') {
-            throw new Error('expected working type model');
-        }
-        // Enforce version 0 to always keep a single working model at a time
-        const modelInfo = { ...src, type: 'working', version: 0 };
-        let includeOptimizer;
-        if (model instanceof models.TFJS) {
-            modelInfo['tensorBackend'] = 'tfjs';
-            includeOptimizer = true;
-        }
-        else if (model instanceof models.GPT) {
-            modelInfo['tensorBackend'] = 'gpt';
-            includeOptimizer = false; // true raises an error
-        }
-        else {
-            debug('unknown working model type %o', model);
-            throw new Error(`unknown model type while updating working model`);
-        }
-        const indexedDBURL = this.getModelMemoryPath(modelInfo);
-        await model.extract().save(indexedDBURL, { includeOptimizer });
-    }
-    /**
-   * Creates a saved copy of the working model corresponding to the source.
-   * @param source the source
-   */
-    async saveWorkingModel(source) {
-        const src = this.getModelInfo(source);
-        if (src.type !== 'working') {
-            throw new Error('expected working type model');
-        }
-        const dst = this.getModelMemoryPath(await this.duplicateSource({ ...src, type: 'saved' }));
-        await tf.io.copyModel(this.getModelMemoryPath({ ...src, type: 'working' }), dst);
-        return dst;
-    }
-    async saveModel(source, model) {
-        const src = this.getModelInfo(source);
-        if (src.type !== 'saved') {
-            throw new Error('expected saved type model');
-        }
-        const modelInfo = await this.duplicateSource({ ...src, type: 'saved' });
-        let includeOptimizer;
-        if (model instanceof models.TFJS) {
-            modelInfo['tensorBackend'] = 'tfjs';
-            includeOptimizer = true;
-        }
-        else if (model instanceof models.GPT) {
-            modelInfo['tensorBackend'] = 'gpt';
-            includeOptimizer = false; // true raises an error
-        }
-        else {
-            debug('unknown saved model type %o', model);
-            throw new Error('unknown model type while saving model');
-        }
-        const indexedDBURL = this.getModelMemoryPath(modelInfo);
-        await model.extract().save(indexedDBURL, { includeOptimizer });
-        return indexedDBURL;
-    }
-    /**
-   * Downloads the model corresponding to the source.
-   * @param source the source
-   */
-    async downloadModel(source) {
-        const src = this.getModelInfo(source);
-        await tf.io.copyModel(this.getModelMemoryPath(source), `downloads://${src.taskID}_${src.name}`);
-    }
-    async latestDuplicate(source) {
-        if (typeof source !== 'string') {
-            source = this.getModelMemoryPath({ ...source, version: 0 });
-        }
-        // perform a single memory read
-        const paths = Map(await tf.io.listModels());
-        if (!paths.has(source)) {
-            return undefined;
-        }
-        const latest = Map(paths).keySeq().toList()
-            .map((p) => this.getModelInfo(p).version).max();
-        if (latest === undefined) {
-            return 0;
-        }
-        return latest;
-    }
-    async duplicateSource(source) {
-        const latestDuplicate = await this.latestDuplicate(source);
-        source = this.getModelInfo(source);
-        if (latestDuplicate === undefined) {
-            return source;
-        }
-        return { ...source, version: latestDuplicate + 1 };
-    }
-}