@epfml/discojs-web 3.0.1-p20240821133014.0 → 3.0.1-p20240826092658.0

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-export * from './data/index.js';
-export * from './memory/index.js';
+export * from "./loaders/index.js";
+export * from "./memory/index.js";

package/dist/index.js

@@ -1,2 +1,2 @@
-export * from './data/index.js';
-export * from './memory/index.js';
+export * from "./loaders/index.js";
+export * from "./memory/index.js";

package/dist/loaders/csv.d.ts

@@ -0,0 +1,2 @@
+import { Dataset } from "@epfml/discojs";
+export declare function load(file: File): Dataset<Partial<Record<string, string>>>;

package/dist/loaders/csv.js

@@ -0,0 +1,36 @@
+import Papa from "papaparse";
+import { Dataset } from "@epfml/discojs";
+function isRecordOfString(raw) {
+    if (typeof raw !== "object" || raw === null)
+        return false;
+    const record = raw;
+    for (const v of Object.values(record))
+        if (typeof v !== "string")
+            return false;
+    return true;
+}
+export function load(file) {
+    return new Dataset(async function* () {
+        // papaparse uses callback for streams and can't easily be converted to async generator
+        // maybe another library does it better but I didn't find one at the time
+        yield* await new Promise((resolve, reject) => {
+            Papa.parse(file, {
+                header: true,
+                dynamicTyping: false,
+                skipEmptyLines: true, // TODO needed to avoid parsing last empty line
+                complete(results) {
+                    if (results.errors.length > 0) {
+                        reject(results.errors);
+                        return;
+                    }
+                    const rows = results.data.map((row) => {
+                        if (!isRecordOfString(row))
+                            throw new Error("excepted object of string to string");
+                        return row;
+                    });
+                    resolve(rows);
+                },
+            });
+        });
+    });
+}

package/dist/loaders/image.d.ts

@@ -0,0 +1,2 @@
+import { Image as DiscoImage } from "@epfml/discojs";
+export declare function load(file: Blob): Promise<DiscoImage<4>>;

package/dist/loaders/image.js

@@ -0,0 +1,15 @@
+import { Image as DiscoImage } from "@epfml/discojs";
+export async function load(file) {
+    const image = new Image();
+    const url = URL.createObjectURL(file);
+    image.src = url;
+    await image.decode();
+    URL.revokeObjectURL(url);
+    const [width, height] = [image.naturalWidth, image.naturalHeight];
+    const context = new OffscreenCanvas(width, height).getContext("2d");
+    if (context === null)
+        throw new Error("unable to setup image convertor");
+    context.drawImage(image, 0, 0);
+    const data = new Uint8Array(context.getImageData(0, 0, width, height).data);
+    return new DiscoImage(data, width, height, 4);
+}

package/dist/loaders/index.d.ts

@@ -0,0 +1,3 @@
+export { load as loadCSV } from "./csv.js";
+export { load as loadImage } from "./image.js";
+export { load as loadText } from "./text.js";

package/dist/loaders/index.js

@@ -0,0 +1,3 @@
+export { load as loadCSV } from "./csv.js";
+export { load as loadImage } from "./image.js";
+export { load as loadText } from "./text.js";

package/dist/loaders/text.d.ts

@@ -0,0 +1,2 @@
+import { Dataset, Text } from "@epfml/discojs";
+export declare function load(file: Blob): Dataset<Text>;

package/dist/loaders/text.js

@@ -0,0 +1,37 @@
+import { Dataset } from "@epfml/discojs";
+class LineStream extends TransformStream {
+    constructor() {
+        let current_line = "";
+        super({
+            transform: (chunk, controller) => {
+                const [head, ...lines] = chunk.split(/\r\n|\r|\n/);
+                const first_line = current_line + head;
+                if (lines.length === 0) {
+                    current_line = first_line;
+                    return;
+                }
+                controller.enqueue(first_line);
+                for (const line of lines.slice(0, -1))
+                    controller.enqueue(line);
+                current_line = lines[lines.length - 1];
+            },
+            flush: (controller) => controller.enqueue(current_line),
+        });
+    }
+}
+export function load(file) {
+    return new Dataset(async function* () {
+        const reader = file
+            .stream()
+            .pipeThrough(new TextDecoderStream())
+            .pipeThrough(new LineStream())
+            .getReader();
+        while (true) {
+            const { value: chunk, done } = await reader.read();
+            if (chunk !== undefined)
+                yield chunk;
+            if (done)
+                break;
+        }
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epfml/discojs-web",
-  "version": "3.0.1-p20240821133014.0",
+  "version": "3.0.1-p20240826092658.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,7 +8,7 @@
     "watch": "nodemon --ext ts --ignore dist --watch ../discojs/dist --watch . --exec npm run",
     "build": "tsc",
     "lint": "npx eslint .",
-    "test": ": nothing"
+    "test": "vitest --run"
   },
   "repository": {
     "type": "git",
@@ -20,9 +20,13 @@
   "homepage": "https://github.com/epfml/disco#readme",
   "dependencies": {
     "@epfml/discojs": "*",
-    "@tensorflow/tfjs": "4"
+    "@tensorflow/tfjs": "4",
+    "papaparse": "5"
   },
   "devDependencies": {
-    "nodemon": "3"
+    "@types/papaparse": "5",
+    "jsdom": "24",
+    "nodemon": "3",
+    "vitest": "1"
   }
 }

package/dist/data/image_loader.d.ts

@@ -1,5 +0,0 @@
-import * as tf from '@tensorflow/tfjs';
-import { data } from '@epfml/discojs';
-export declare class ImageLoader extends data.ImageLoader<File> {
-    readImageFrom(source: File): Promise<tf.Tensor3D>;
-}

package/dist/data/image_loader.js

@@ -1,7 +0,0 @@
-import * as tf from '@tensorflow/tfjs';
-import { data } from '@epfml/discojs';
-export class ImageLoader extends data.ImageLoader {
-    async readImageFrom(source) {
-        return tf.browser.fromPixels(await createImageBitmap(source));
-    }
-}

package/dist/data/index.d.ts

@@ -1,3 +0,0 @@
-export { ImageLoader as WebImageLoader } from './image_loader.js';
-export { TabularLoader as WebTabularLoader } from './tabular_loader.js';
-export { TextLoader as WebTextLoader } from './text_loader.js';

package/dist/data/index.js

@@ -1,3 +0,0 @@
-export { ImageLoader as WebImageLoader } from './image_loader.js';
-export { TabularLoader as WebTabularLoader } from './tabular_loader.js';
-export { TextLoader as WebTextLoader } from './text_loader.js';

package/dist/data/tabular_loader.d.ts

@@ -1,4 +0,0 @@
-import { data } from '@epfml/discojs';
-export declare class TabularLoader extends data.TabularLoader<File> {
-    loadDatasetFrom(source: File, csvConfig: Record<string, unknown>): Promise<data.Dataset>;
-}

package/dist/data/tabular_loader.js

@@ -1,8 +0,0 @@
-import * as tf from '@tensorflow/tfjs';
-import { data } from '@epfml/discojs';
-export class TabularLoader extends data.TabularLoader {
-    async loadDatasetFrom(source, csvConfig) {
-        const file = new tf.data.FileDataSource(source);
-        return Promise.resolve(new tf.data.CSVDataset(file, csvConfig));
-    }
-}

package/dist/data/text_loader.d.ts

@@ -1,4 +0,0 @@
-import { data } from '@epfml/discojs';
-export declare class TextLoader extends data.TextLoader<File> {
-    loadDatasetFrom(source: File): Promise<data.Dataset>;
-}

package/dist/data/text_loader.js

@@ -1,9 +0,0 @@
-import * as tf from '@tensorflow/tfjs';
-import { data } from '@epfml/discojs';
-export class TextLoader extends data.TextLoader {
-    loadDatasetFrom(source) {
-        const file = new tf.data.FileDataSource(source);
-        const dataset = new tf.data.TextLineDataset(file).filter(s => s !== ' '); // newline creates empty strings
-        return Promise.resolve(dataset);
-    }
-}