@epfml/discojs-node 3.0.1-p20240821133014.0 → 3.0.1-p20240826092658.0

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-export * from './data/index.js';
-export { saveModelToDisk, loadModelFromDisk } from './models/model_loader.js';
+export * from './loaders/index.js';
+export { saveModelToDisk, loadModelFromDisk } from './model_loader.js';

package/dist/index.js

@@ -1,2 +1,2 @@
-export * from './data/index.js';
-export { saveModelToDisk, loadModelFromDisk } from './models/model_loader.js';
+export * from './loaders/index.js';
+export { saveModelToDisk, loadModelFromDisk } from './model_loader.js';

package/dist/loaders/csv.d.ts

@@ -0,0 +1,2 @@
+import { Dataset } from "@epfml/discojs";
+export declare function load(path: string): Dataset<Partial<Record<string, string>>>;

package/dist/loaders/csv.js

@@ -0,0 +1,24 @@
+import * as fs from "node:fs/promises";
+import { parse as csvParser } from "csv-parse";
+import { Dataset } from "@epfml/discojs";
+function isRecordOfString(raw) {
+    if (typeof raw !== "object" || raw === null)
+        return false;
+    const record = raw;
+    for (const [k, v] of Object.entries(record))
+        if (typeof k !== "string" || typeof v !== "string")
+            return false;
+    return true;
+}
+export function load(path) {
+    return new Dataset(async function* () {
+        const stream = (await fs.open(path))
+            .createReadStream()
+            .pipe(csvParser({ columns: true }));
+        for await (const row of stream) {
+            if (!isRecordOfString(row))
+                throw new Error("excepted object of string to string");
+            yield row;
+        }
+    });
+}

package/dist/loaders/image.d.ts

@@ -0,0 +1,3 @@
+import { Dataset, Image } from "@epfml/discojs";
+export declare function load(path: string): Promise<Image<1 | 3 | 4>>;
+export declare function loadAllInDir(dir: string): Promise<Dataset<Image>>;

package/dist/loaders/image.js

@@ -0,0 +1,17 @@
+import sharp from "sharp";
+import * as path from "node:path";
+import * as fs from "node:fs/promises";
+import { Dataset, Image } from "@epfml/discojs";
+export async function load(path) {
+    const { data, info } = await sharp(path).raw().toBuffer({
+        resolveWithObject: true,
+    });
+    if (info.channels === 2)
+        throw new Error("unsupported channel count");
+    return new Image(data, info.width, info.height, info.channels);
+}
+export async function loadAllInDir(dir) {
+    const filenames = await fs.readdir(dir);
+    const paths = filenames.map((f) => path.join(dir, f));
+    return new Dataset(paths).map(load);
+}

package/dist/loaders/index.d.ts

@@ -0,0 +1,3 @@
+export { load as loadCSV } from "./csv.js";
+export { load as loadImage, loadAllInDir as loadImagesInDir, } from "./image.js";
+export { load as loadText } from "./text.js";

package/dist/loaders/index.js

@@ -0,0 +1,3 @@
+export { load as loadCSV } from "./csv.js";
+export { load as loadImage, loadAllInDir as loadImagesInDir, } from "./image.js";
+export { load as loadText } from "./text.js";

package/dist/loaders/text.d.ts

@@ -0,0 +1,2 @@
+import { Dataset, Text } from "@epfml/discojs";
+export declare function load(path: string): Dataset<Text>;

package/dist/loaders/text.js

@@ -0,0 +1,11 @@
+import * as fs from "node:fs/promises";
+import * as readline from "node:readline/promises";
+import { Dataset } from "@epfml/discojs";
+export function load(path) {
+    return new Dataset(async function* () {
+        const input = (await fs.open(path)).createReadStream({ encoding: "utf8" });
+        // `readline` is a bit overkill but seems standard
+        // https://nodejs.org/api/readline.html#example-read-file-stream-line-by-line
+        yield* readline.createInterface({ input, crlfDelay: Infinity });
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epfml/discojs-node",
-  "version": "3.0.1-p20240821133014.0",
+  "version": "3.0.1-p20240826092658.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -21,11 +21,14 @@
   "dependencies": {
     "@epfml/discojs": "*",
     "@koush/wrtc": "0.5",
-    "@tensorflow/tfjs-node": "4"
+    "@tensorflow/tfjs-node": "4",
+    "csv-parse": "5",
+    "sharp": "0.33"
   },
   "devDependencies": {
     "@types/node": "22",
     "nodemon": "3",
+    "tmp-promise": "3",
     "ts-node": "10"
   }
 }

package/dist/data/image_loader.d.ts

@@ -1,5 +0,0 @@
-import type tf from '@tensorflow/tfjs';
-import { data } from '@epfml/discojs';
-export declare class ImageLoader extends data.ImageLoader<string> {
-    readImageFrom(source: string, channels?: number): Promise<tf.Tensor3D>;
-}

package/dist/data/image_loader.js

@@ -1,11 +0,0 @@
-import fs from 'node:fs/promises';
-import { node as tfNode } from '@tensorflow/tfjs-node';
-import { data } from '@epfml/discojs';
-export class ImageLoader extends data.ImageLoader {
-    async readImageFrom(source, channels) {
-        // We allow specifying the number of channels because the default number of channels
-        // differs between web and node for the same image 
-        // E.g. lus covid images have 1 channel with fs.readFile but 3 when loaded with discojs-web
-        return tfNode.decodeImage(await fs.readFile(source), channels);
-    }
-}

package/dist/data/index.d.ts

@@ -1,3 +0,0 @@
-export { ImageLoader as NodeImageLoader } from './image_loader.js';
-export { TabularLoader as NodeTabularLoader } from './tabular_loader.js';
-export { TextLoader as NodeTextLoader } from './text_loader.js';

package/dist/data/index.js

@@ -1,3 +0,0 @@
-export { ImageLoader as NodeImageLoader } from './image_loader.js';
-export { TabularLoader as NodeTabularLoader } from './tabular_loader.js';
-export { TextLoader as NodeTextLoader } from './text_loader.js';

package/dist/data/tabular_loader.d.ts

@@ -1,4 +0,0 @@
-import { data } from '@epfml/discojs';
-export declare class TabularLoader extends data.TabularLoader<string> {
-    loadDatasetFrom(source: string, csvConfig: Record<string, unknown>): Promise<data.Dataset>;
-}

package/dist/data/tabular_loader.js

@@ -1,11 +0,0 @@
-import { data as tfData } from '@tensorflow/tfjs-node';
-import { data } from '@epfml/discojs';
-export class TabularLoader extends data.TabularLoader {
-    loadDatasetFrom(source, csvConfig) {
-        const prefix = 'file://';
-        if (source.slice(0, 7) !== prefix) {
-            source = prefix + source;
-        }
-        return Promise.resolve(tfData.csv(source, csvConfig));
-    }
-}

package/dist/data/text_loader.d.ts

@@ -1,4 +0,0 @@
-import { data } from '@epfml/discojs';
-export declare class TextLoader extends data.TextLoader<string> {
-    loadDatasetFrom(source: string): Promise<data.Dataset>;
-}

package/dist/data/text_loader.js

@@ -1,14 +0,0 @@
-import { data as tfData } from '@tensorflow/tfjs-node';
-import fs from 'node:fs/promises';
-import { data } from '@epfml/discojs';
-export class TextLoader extends data.TextLoader {
-    async loadDatasetFrom(source) {
-        // TODO: reads all the file at once, 
-        // inputting the file path to FileDataSource isn't supported anymore
-        const inputFile = await fs.readFile(source);
-        const file = new tfData.FileDataSource(inputFile, { chunkSize: 1024 });
-        // TODO: reading files line by line is an issue for LLM tokenization
-        const dataset = new tfData.TextLineDataset(file).filter(s => s !== ' '); // newline creates empty strings
-        return Promise.resolve(dataset);
-    }
-}