@teselagen/file-utils 0.3.0 → 0.3.2

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "@teselagen/file-utils",
-  "version": "0.3.0",
-  "type": "module",
+  "version": "0.3.2",
+  "type": "commonjs",
   "dependencies": {
     "bluebird": "^3.7.2",
     "jszip": "^3.10.1",
     "lodash": "^4.17.21",
     "papaparse": "^5.4.1"
   },
-  "module": "index.js",
-  "main": "index.js"
-}
+  "devDependencies": {
+    "mock-fs": "^5.2.0"
+  },
+  "license": "MIT"
+}

package/src/file-utils.js

@@ -0,0 +1,354 @@
+/* Copyright (C) 2018 TeselaGen Biotechnology, Inc. */
+import { camelCase, flatMap, remove, startsWith, snakeCase } from "lodash";
+import { loadAsync } from "jszip";
+import Promise from "bluebird";
+import { parse, unparse } from "papaparse";
+
+const debug = false;
+const logDebug = (...args) => {
+  if (debug) {
+    // eslint-disable-next-line no-console
+    console.log(...args);
+  }
+};
+
+export const allowedCsvFileTypes = [".csv", ".txt", ".xlsx"];
+
+export const isZipFile = (file) => {
+  if (getExt(file) === "zip") return true;
+  if (getExt(file) === "geneious") return false;
+  const type = file.mimetype || file.type;
+  return type === "application/zip" || type === "application/x-zip-compressed";
+};
+
+export const getExt = (file) => file?.name?.split(".").pop();
+export const isExcelFile = (file) => getExt(file) === "xlsx";
+export const isCsvFile = (file) => getExt(file) === "csv";
+export const isTextFile = (file) => ["text", "txt"].includes(getExt(file));
+
+export const isCsvOrExcelFile = (file) => isCsvFile(file) || isExcelFile(file);
+
+export const extractZipFiles = async (allFiles) => {
+  if (!Array.isArray(allFiles)) allFiles = [allFiles];
+  // make a copy so we don't mutate the form value
+  allFiles = [...allFiles];
+  const zipFiles = remove(allFiles, isZipFile);
+  if (!zipFiles.length) return allFiles;
+  const zipFilesArray = Array.isArray(zipFiles) ? zipFiles : [zipFiles];
+  const parsedZips = await Promise.map(zipFilesArray, (file) =>
+    loadAsync(file instanceof Blob ? file : file.originFileObj)
+  );
+  const zippedFiles = flatMap(parsedZips, (zip) =>
+    Object.keys(zip.files).map((key) => zip.files[key])
+  );
+  const unzippedFiles = await Promise.map(zippedFiles, (file) => {
+    // converts the compressed file to a string of its contents
+    return file.async("blob").then(function (fileData) {
+      const newFileObj = new File([fileData], file.name);
+      return {
+        name: file.name,
+        originFileObj: newFileObj,
+        originalFileObj: newFileObj
+      };
+    });
+  });
+  if (unzippedFiles.length) {
+    return allFiles.concat(
+      unzippedFiles.filter(
+        ({ name, originFileObj }) =>
+          !name.includes("__MACOSX") &&
+          !name.includes(".DS_Store") &&
+          originFileObj.size !== 0
+      )
+    );
+  } else {
+    return allFiles;
+  }
+};
+
+const defaultCsvParserOptions = {
+  header: true,
+  skipEmptyLines: "greedy",
+  trimHeaders: true
+};
+export const setupCsvParserOptions = (parserOptions = {}) => {
+  const {
+    camelCaseHeaders = false,
+    lowerCaseHeaders = false,
+    ...rest
+  } = parserOptions;
+
+  const papaParseOpts = { ...rest };
+  if (camelCaseHeaders) {
+    logDebug("[CSV-PARSER] camelCasing headers");
+    papaParseOpts.transformHeader = (header) => {
+      let transHeader = header;
+      if (!startsWith(header.trim(), "ext-")) {
+        transHeader = camelCase(header);
+      }
+
+      if (transHeader) {
+        logDebug(
+          `[CSV-PARSER] Transformed header from: ${header} to: ${transHeader}`
+        );
+        transHeader = transHeader.trim();
+      } else {
+        logDebug(`[CSV-PARSER] Not transforming header: ${header}`);
+      }
+
+      return transHeader;
+    };
+  } else if (lowerCaseHeaders) {
+    papaParseOpts.transformHeader = (header) => {
+      let transHeader = header;
+      if (!startsWith(header, "ext-")) {
+        transHeader = header.toLowerCase();
+      }
+
+      if (transHeader) {
+        logDebug(
+          `[CSV-PARSER] Transformed header from: ${header} to: ${transHeader}`
+        );
+        transHeader = transHeader.trim();
+      } else {
+        logDebug(`[CSV-PARSER] Not transforming header: ${header}`);
+      }
+
+      return transHeader;
+    };
+  }
+
+  return papaParseOpts;
+};
+
+const normalizeCsvHeaderHelper = (h) =>
+  snakeCase(h.toUpperCase()).toUpperCase();
+
+export function normalizeCsvHeader(header) {
+  if (header.startsWith("ext-") || header.startsWith("EXT-")) {
+    return header;
+  }
+  return normalizeCsvHeaderHelper(header);
+}
+
+export const parseCsvFile = (csvFile, parserOptions = {}) => {
+  return new Promise((resolve, reject) => {
+    const opts = {
+      ...defaultCsvParserOptions,
+      ...setupCsvParserOptions(parserOptions),
+      complete: (results) => {
+        if (
+          results &&
+          results.data?.length &&
+          results.errors &&
+          results.errors.length === 1 &&
+          results.errors[0].code === `UndetectableDelimiter`
+        ) {
+          return resolve(results);
+        } else if (results && results.errors && results.errors.length) {
+          return reject("Error in csv: " + JSON.stringify(results.errors));
+        }
+        return resolve(results);
+      },
+      error: (error) => {
+        reject(error);
+      }
+    };
+    logDebug(`[CSV-PARSER] parseCsvFile opts:`, opts);
+    parse(csvFile.originFileObj, opts);
+  });
+};
+
+/**
+ * Gets a properly formatted json object into s csv string
+ * https://www.papaparse.com/docs#json-to-csv
+ * const options = {
+      quotes: false, //or array of booleans
+      quoteChar: '"',
+      escapeChar: '"',
+      delimiter: ",",
+      header: true,
+      newline: "\r\n",
+      skipEmptyLines: false, //other option is 'greedy', meaning skip delimiters, quotes, and whitespace.
+      columns: null //or array of strings
+    }
+ * @returns csv as string
+ */
+export const jsonToCsv = (jsonData, options = {}) => {
+  const csv = unparse(jsonData, options);
+  return csv;
+};
+
+export const parseCsvString = (csvString, parserOptions = {}) => {
+  const opts = {
+    ...defaultCsvParserOptions,
+    ...setupCsvParserOptions(parserOptions)
+  };
+  logDebug(`[CSV-PARSER] parseCsvString opts:`, opts);
+  return parse(csvString, opts);
+};
+
+export async function parseCsvOrExcelFile(
+  fileOrFiles,
+  { csvParserOptions } = {}
+) {
+  let csvFile, excelFile, txtFile;
+  if (Array.isArray(fileOrFiles)) {
+    csvFile = fileOrFiles.find(isCsvFile);
+    excelFile = fileOrFiles.find(isExcelFile);
+    txtFile = fileOrFiles.find(isTextFile);
+  } else {
+    if (isExcelFile(fileOrFiles)) excelFile = fileOrFiles;
+    else if (isCsvFile(fileOrFiles)) csvFile = fileOrFiles;
+    else if (isTextFile(fileOrFiles)) txtFile = fileOrFiles;
+  }
+  if (!csvFile && !excelFile && !txtFile) {
+    throw new Error("No csv or excel files found");
+  }
+
+  if (!csvFile && !excelFile) csvFile = txtFile;
+
+  if (!csvFile && excelFile && window.parseExcelToCsv) {
+    csvFile = await window.parseExcelToCsv(
+      excelFile.originFileObj || excelFile
+    );
+    if (csvFile.error) {
+      throw new Error(csvFile.error);
+    }
+  } else if (excelFile) {
+    throw new Error("Excel Parser not initialized on the window");
+  }
+  const parsedCsv = await parseCsvFile(csvFile, csvParserOptions);
+  parsedCsv.originalFile = csvFile;
+  return parsedCsv;
+}
+
+export const validateCSVRequiredHeaders = (
+  fields,
+  requiredHeaders,
+  filename
+) => {
+  const missingRequiredHeaders = requiredHeaders.filter((field) => {
+    return !fields.includes(field);
+  });
+  if (missingRequiredHeaders.length) {
+    const name = filename ? `The file ${filename}` : "CSV file";
+    return `${name} is missing required headers. (${missingRequiredHeaders.join(
+      ", "
+    )})`;
+  }
+};
+
+export const validateCSVRow = (row, requiredHeaders, index) => {
+  const missingRequiredFields = requiredHeaders.filter((field) => !row[field]);
+  if (missingRequiredFields.length) {
+    if (missingRequiredFields.length === 1) {
+      return `Row ${index + 1} is missing the required field "${
+        missingRequiredFields[0]
+      }"`;
+    } else {
+      return `Row ${
+        index + 1
+      } is missing these required fields: ${missingRequiredFields.join(", ")}`;
+    }
+  }
+};
+
+export const cleanCommaSeparatedCell = (cellData) =>
+  (cellData || "")
+    .split(",")
+    .map((n) => n.trim())
+    .filter((n) => n);
+
+/**
+ * Because the csv rows might not have the same header keys in some cases (extended properties)
+ * this function will make sure that each row will have all headers so that the export
+ * does not drop fields
+ * @param {*} rows
+ */
+export const cleanCsvExport = (rows) => {
+  const allHeaders = [];
+  rows.forEach((row) => {
+    Object.keys(row).forEach((header) => {
+      if (!allHeaders.includes(header)) {
+        allHeaders.push(header);
+      }
+    });
+  });
+  rows.forEach((row) => {
+    allHeaders.forEach((header) => {
+      row[header] = row[header] || "";
+    });
+  });
+  return rows;
+};
+
+export const filterFilesInZip = async (file, accepted) => {
+  const zipExtracted = await extractZipFiles(file);
+  const acceptedFiles = [];
+  for (const extFile of zipExtracted) {
+    if (accepted.some((ext) => ext?.replace(".", "") === getExt(extFile))) {
+      acceptedFiles.push(extFile);
+    }
+  }
+
+  if (acceptedFiles.length && acceptedFiles.length < zipExtracted.length)
+    window.toastr.warning("Some files don't have the proper file extension.");
+
+  if (!acceptedFiles.length)
+    window.toastr.warning("No files with the proper extension were found.");
+
+  return acceptedFiles;
+};
+
+export function removeExt(filename) {
+  if (filename && filename.includes(".")) {
+    return filename.split(".").slice(0, -1).join(".");
+  } else {
+    return filename;
+  }
+}
+
+export async function uploadAndProcessFiles(files = []) {
+  if (!files.length) return null;
+
+  const formData = new FormData();
+  files.forEach(({ originFileObj }) => formData.append("file", originFileObj));
+
+  const response = await window.api.post("/user_uploads/", formData);
+
+  return response.data.map((d) => ({
+    encoding: d.encoding,
+    mimetype: d.mimetype,
+    originalname: d.originalname,
+    path: d.path,
+    size: d.size
+  }));
+}
+
+export async function encodeFilesForRequest(files) {
+  const encodedFiles = [];
+  for (const file of files) {
+    const encoded = await fileToBase64(file.originalFileObj);
+    const data = encoded.split(",");
+    encodedFiles.push({
+      type: file.type,
+      base64Data: data[1],
+      name: file.name
+    });
+  }
+  return encodedFiles;
+}
+
+const fileToBase64 = (file) => {
+  return new Promise((resolve) => {
+    const reader = new FileReader();
+    // Read file content on file loaded event
+    reader.onload = function (event) {
+      resolve(event.target.result);
+    };
+
+    // Convert data to base64
+    reader.readAsDataURL(file);
+  });
+};

package/src/file-utils.spec.js

@@ -0,0 +1,186 @@
+import {
+  isZipFile,
+  isExcelFile,
+  isCsvFile,
+  isTextFile,
+  normalizeCsvHeader,
+  validateCSVRequiredHeaders,
+  validateCSVRow,
+  removeExt,
+  filterFilesInZip,
+  parseCsvFile
+} from "./file-utils"; // replace 'yourFile' with the path of your actual file
+
+import * as JSZip from "jszip";
+import * as mock from "mock-fs";
+
+describe("parseCsvFile", () => {
+  it("resolves with results when parsing is successful", async () => {
+    const results = await parseCsvFile({
+      originFileObj: new File(
+        [
+          `Material name
+mat 1
+mat 2`
+        ],
+        "dummyFile"
+      )
+    });
+    expect(results.data).toEqual([
+      { "Material name": "mat 1" },
+      { "Material name": "mat 2" }
+    ]);
+  });
+});
+
+describe.skip("filterFilesInZip", () => {
+  beforeAll(async () => {
+    const zip = new JSZip();
+    zip.file("test1.txt", "Hello World");
+    zip.file("test2.csv", "id,name\n1,John");
+
+    const data = await zip.generateAsync({ type: "nodebuffer" });
+
+    mock({
+      "/path/to": {
+        "myzipfile.zip": data
+      }
+    });
+  });
+
+  afterAll(() => {
+    mock.restore();
+  });
+
+  it("should filter and return only .csv files", async () => {
+    const file = {
+      path: "/path/to/myzipfile.zip",
+      originalname: "myzipfile.zip",
+      mimetype: "application/zip"
+    };
+    const accepted = [".csv"];
+
+    const files = await filterFilesInZip(file, accepted);
+
+    expect(files.length).toBe(1);
+    expect(files[0].name).toBe("test2.csv");
+    const accepted2 = ["csv"];
+
+    const files2 = await filterFilesInZip(file, accepted2);
+
+    expect(files2.length).toBe(1);
+    expect(files2[0].name).toBe("test2.csv");
+  });
+});
+
+describe("CSV and Excel file tests", () => {
+  describe("isZipFile", () => {
+    it("should return true if file type is zip", () => {
+      const file = { mimetype: "application/zip" };
+      expect(isZipFile(file)).toBeTruthy();
+    });
+
+    it("should return false if file type is not zip", () => {
+      const file = { mimetype: "application/pdf" };
+      expect(isZipFile(file)).toBeFalsy();
+    });
+  });
+
+  describe("isExcelFile", () => {
+    it("should return true if file type is excel", () => {
+      const file = { name: "test.xlsx" };
+      expect(isExcelFile(file)).toBeTruthy();
+    });
+
+    it("should return false if file type is not excel", () => {
+      const file = { name: "test.pdf" };
+      expect(isExcelFile(file)).toBeFalsy();
+    });
+  });
+
+  describe("isCsvFile", () => {
+    it("should return true if file type is csv", () => {
+      const file = { name: "test.csv" };
+      expect(isCsvFile(file)).toBeTruthy();
+    });
+
+    it("should return false if file type is not csv", () => {
+      const file = { name: "test.pdf" };
+      expect(isCsvFile(file)).toBeFalsy();
+    });
+  });
+
+  describe("isTextFile", () => {
+    it("should return true if file type is txt", () => {
+      const file = { name: "test.txt" };
+      expect(isTextFile(file)).toBeTruthy();
+    });
+
+    it("should return false if file type is not txt", () => {
+      const file = { name: "test.pdf" };
+      expect(isTextFile(file)).toBeFalsy();
+    });
+  });
+
+  describe("normalizeCsvHeader", () => {
+    it('should return the same header if it starts with "ext-"', () => {
+      const header = "ext-name";
+      expect(normalizeCsvHeader(header)).toBe(header);
+    });
+
+    it('should return a normalized header if it does not start with "ext-"', () => {
+      const header = "name";
+      expect(normalizeCsvHeader(header)).toBe("NAME");
+    });
+  });
+
+  describe("validateCSVRequiredHeaders", () => {
+    it("should return error message if required headers are missing", () => {
+      const fields = ["name", "address"];
+      const requiredHeaders = ["name", "email"];
+      const filename = "test.csv";
+      expect(
+        validateCSVRequiredHeaders(fields, requiredHeaders, filename)
+      ).toBe("The file test.csv is missing required headers. (email)");
+    });
+
+    it("should return undefined if no required headers are missing", () => {
+      const fields = ["name", "email"];
+      const requiredHeaders = ["name", "email"];
+      const filename = "test.csv";
+      expect(
+        validateCSVRequiredHeaders(fields, requiredHeaders, filename)
+      ).toBeUndefined();
+    });
+  });
+
+  describe("validateCSVRow", () => {
+    it("should return error message if required fields are missing", () => {
+      const row = { name: "John", email: "" };
+      const requiredHeaders = ["name", "email"];
+      const index = 0;
+      expect(validateCSVRow(row, requiredHeaders, index)).toBe(
+        'Row 1 is missing the required field "email"'
+      );
+    });
+
+    it("should return undefined if no required fields are missing", () => {
+      const row = { name: "John", email: "john@example.com" };
+      const requiredHeaders = ["name", "email"];
+      const index = 0;
+      expect(validateCSVRow(row, requiredHeaders, index)).toBeUndefined();
+    });
+  });
+
+  describe("removeExt", () => {
+    it("should remove extension from filename", () => {
+      const filename = "test.csv";
+      expect(removeExt(filename)).toBe("test");
+    });
+
+    it("should return the same filename if there is no extension", () => {
+      const filename = "test";
+      expect(removeExt(filename)).toBe("test");
+    });
+  });
+});

package/src/index.js

@@ -0,0 +1 @@
+export * from './file-utils';

package/CHANGELOG.md

@@ -1,12 +0,0 @@
-# Changelog
-
-This file was generated using [@jscutlery/semver](https://github.com/jscutlery/semver).
-
-## 0.1.0 (2023-05-30)
-
-
-### Features
-
-* more work adding more repos ([f320d76](https://github.com/TeselaGen/tg-oss/commit/f320d76a7a2e0db34d68d1a51803efa24f6831df))
-* wip adding a bunch of repos to tg-oss ([6040094](https://github.com/TeselaGen/tg-oss/commit/60400941f0d7f72c02bf19b90896d9a35d32634d))
-* wip setting up nx ([24a1952](https://github.com/TeselaGen/tg-oss/commit/24a1952fdbadc2c733223109d620cbc508c94a28))

package/README.md

@@ -1,11 +0,0 @@
-# file-utils
-
-This library was generated with [Nx](https://nx.dev).
-
-## Building
-
-Run `nx build file-utils` to build the library.
-
-## Running unit tests
-
-Run `nx test file-utils` to execute the unit tests via [Jest](https://jestjs.io).