@teselagen/file-utils 0.0.3 → 0.2.0

package/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

package/CHANGELOG.md

@@ -0,0 +1,26 @@
+# Changelog
+
+This file was generated using [@jscutlery/semver](https://github.com/jscutlery/semver).
+
+## [0.2.0](https://github.com/TeselaGen/tg-oss/compare/file-utils-0.1.0...file-utils-0.2.0) (2023-05-30)
+
+
+### Features
+
+* more work on packaging up files ([67c8a69](https://github.com/TeselaGen/tg-oss/commit/67c8a6983b2ca772cc72700b4bf344b4adccf26d))
+
+## [0.2.0](https://github.com/TeselaGen/tg-oss/compare/file-utils-0.1.0...file-utils-0.2.0) (2023-05-30)
+
+
+### Features
+
+* more work on packaging up files ([67c8a69](https://github.com/TeselaGen/tg-oss/commit/67c8a6983b2ca772cc72700b4bf344b4adccf26d))
+
+## 0.1.0 (2023-05-30)
+
+
+### Features
+
+* more work adding more repos ([f320d76](https://github.com/TeselaGen/tg-oss/commit/f320d76a7a2e0db34d68d1a51803efa24f6831df))
+* wip adding a bunch of repos to tg-oss ([6040094](https://github.com/TeselaGen/tg-oss/commit/60400941f0d7f72c02bf19b90896d9a35d32634d))
+* wip setting up nx ([24a1952](https://github.com/TeselaGen/tg-oss/commit/24a1952fdbadc2c733223109d620cbc508c94a28))

package/jest.config.js

@@ -0,0 +1,10 @@
+/* eslint-disable */
+module.exports = {
+  displayName: 'file-utils',
+  preset: '../../jest.preset.js',
+  transform: {
+    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: '../../coverage/packages/file-utils',
+};

package/package.json

@@ -1,13 +1,11 @@
 {
   "name": "@teselagen/file-utils",
-  "version": "0.0.3",
-  "type": "module",
+  "version": "0.2.0",
+  "type": "commonjs",
   "dependencies": {
     "bluebird": "^3.7.2",
     "jszip": "^3.10.1",
     "lodash": "^4.17.21",
     "papaparse": "^5.4.1"
-  },
-  "module": "index.js",
-  "main": "index.js"
+  }
 }

package/project.json

@@ -0,0 +1,51 @@
+{
+  "name": "file-utils",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "packages/file-utils/src",
+  "projectType": "library",
+  "targets": {
+ "version": {
+      "executor": "@jscutlery/semver:version",
+      "options": {
+        "preset": "conventional"
+      }
+    },
+    "build": {
+      "executor": "@nx/esbuild:esbuild",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "outputPath": "dist/packages/file-utils",
+        "main": "packages/file-utils/src/index.js",
+        "tsConfig": "packages/file-utils/tsconfig.lib.json",
+        "assets": ["packages/file-utils/*.md"],
+        "generatePackageJson": true
+      }
+    },
+    "publish": {
+      "command": "node tools/scripts/publish.mjs file-utils {args.ver} {args.tag}",
+      "dependsOn": ["build"]
+    },
+    "lint": {
+      "executor": "@nx/linter:eslint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": ["packages/file-utils/**/*.js"]
+      }
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "options": {
+        "jestConfig": "packages/file-utils/jest.config.js",
+        "passWithNoTests": true
+      },
+      "configurations": {
+        "ci": {
+          "ci": true,
+          "codeCoverage": true
+        }
+      }
+    }
+  },
+  "tags": []
+}

package/{index.js → src/file-utils.js}

@@ -1,40 +1,45 @@
-// packages/file-utils/src/lib/file-utils.js
+/* Copyright (C) 2018 TeselaGen Biotechnology, Inc. */
 import { camelCase, flatMap, remove, startsWith, snakeCase } from "lodash";
 import { loadAsync } from "jszip";
-import Promise2 from "bluebird";
+import Promise from "bluebird";
 import { parse, unparse } from "papaparse";
-var logDebug = (...args) => {
+
+const logDebug = (...args) => {
   if (process.env.DEBUG_CSV_PARSING) {
+    // eslint-disable-next-line no-console
     console.log(...args);
   }
 };
-var allowedCsvFileTypes = [".csv", ".txt", ".xlsx"];
-var isZipFile = (file) => {
+
+export const allowedCsvFileTypes = [".csv", ".txt", ".xlsx"];
+
+export const isZipFile = file => {
   const type = file.mimetype || file.type;
   return type === "application/zip" || type === "application/x-zip-compressed";
 };
-var getExt = (file) => file.name.split(".").pop();
-var isExcelFile = (file) => getExt(file) === "xlsx";
-var isCsvFile = (file) => getExt(file) === "csv";
-var isTextFile = (file) => ["text", "txt"].includes(getExt(file));
-var isCsvOrExcelFile = (file) => isCsvFile(file) || isExcelFile(file);
-var extractZipFiles = async (allFiles) => {
-  if (!Array.isArray(allFiles))
-    allFiles = [allFiles];
+
+export const getExt = file => file.name.split(".").pop();
+export const isExcelFile = file => getExt(file) === "xlsx";
+export const isCsvFile = file => getExt(file) === "csv";
+export const isTextFile = file => ["text", "txt"].includes(getExt(file));
+
+export const isCsvOrExcelFile = file => isCsvFile(file) || isExcelFile(file);
+
+export const extractZipFiles = async allFiles => {
+  if (!Array.isArray(allFiles)) allFiles = [allFiles];
+  // make a copy so we don't mutate the form value
   allFiles = [...allFiles];
   const zipFiles = remove(allFiles, isZipFile);
-  if (!zipFiles.length)
-    return allFiles;
+  if (!zipFiles.length) return allFiles;
   const zipFilesArray = Array.isArray(zipFiles) ? zipFiles : [zipFiles];
-  const parsedZips = await Promise2.map(
-    zipFilesArray,
-    (file) => loadAsync(file instanceof Blob ? file : file.originFileObj)
+  const parsedZips = await Promise.map(zipFilesArray, file =>
+    loadAsync(file instanceof Blob ? file : file.originFileObj)
   );
-  const zippedFiles = flatMap(
-    parsedZips,
-    (zip) => Object.keys(zip.files).map((key) => zip.files[key])
+  const zippedFiles = flatMap(parsedZips, zip =>
+    Object.keys(zip.files).map(key => zip.files[key])
   );
-  const unzippedFiles = await Promise2.map(zippedFiles, (file) => {
+  const unzippedFiles = await Promise.map(zippedFiles, file => {
+    // converts the compressed file to a string of its contents
     return file.async("blob").then(function(fileData) {
       const newFileObj = new File([fileData], file.name);
       return {
@@ -47,32 +52,38 @@ var extractZipFiles = async (allFiles) => {
   if (unzippedFiles.length) {
     return allFiles.concat(
       unzippedFiles.filter(
-        ({ name, originFileObj }) => !name.includes("__MACOSX") && !name.includes(".DS_Store") && originFileObj.size !== 0
+        ({ name, originFileObj }) =>
+          !name.includes("__MACOSX") &&
+          !name.includes(".DS_Store") &&
+          originFileObj.size !== 0
       )
     );
   } else {
     return allFiles;
   }
 };
-var defaultCsvParserOptions = {
+
+const defaultCsvParserOptions = {
   header: true,
   skipEmptyLines: "greedy",
   trimHeaders: true
 };
-var setupCsvParserOptions = (parserOptions = {}) => {
+export const setupCsvParserOptions = (parserOptions = {}) => {
   const {
     camelCaseHeaders = false,
     lowerCaseHeaders = false,
     ...rest
   } = parserOptions;
+
   const papaParseOpts = { ...rest };
   if (camelCaseHeaders) {
     logDebug("[CSV-PARSER] camelCasing headers");
-    papaParseOpts.transformHeader = (header) => {
+    papaParseOpts.transformHeader = header => {
       let transHeader = header;
       if (!startsWith(header.trim(), "ext-")) {
         transHeader = camelCase(header);
       }
+
       if (transHeader) {
         logDebug(
           `[CSV-PARSER] Transformed header from: ${header} to: ${transHeader}`
@@ -81,14 +92,16 @@ var setupCsvParserOptions = (parserOptions = {}) => {
       } else {
         logDebug(`[CSV-PARSER] Not transforming header: ${header}`);
       }
+
       return transHeader;
     };
   } else if (lowerCaseHeaders) {
-    papaParseOpts.transformHeader = (header) => {
+    papaParseOpts.transformHeader = header => {
       let transHeader = header;
       if (!startsWith(header, "ext-")) {
         transHeader = header.toLowerCase();
       }
+
       if (transHeader) {
         logDebug(
           `[CSV-PARSER] Transformed header from: ${header} to: ${transHeader}`
@@ -97,30 +110,35 @@ var setupCsvParserOptions = (parserOptions = {}) => {
       } else {
         logDebug(`[CSV-PARSER] Not transforming header: ${header}`);
       }
+
       return transHeader;
     };
   }
+
   return papaParseOpts;
 };
-var normalizeCsvHeaderHelper = (h) => snakeCase(h.toUpperCase()).toUpperCase();
-function normalizeCsvHeader(header) {
+
+const normalizeCsvHeaderHelper = h => snakeCase(h.toUpperCase()).toUpperCase();
+
+export function normalizeCsvHeader(header) {
   if (header.startsWith("ext-") || header.startsWith("EXT-")) {
     return header;
   }
   return normalizeCsvHeaderHelper(header);
 }
-var parseCsvFile = (csvFile, parserOptions = {}) => {
-  return new Promise2((resolve, reject) => {
+
+export const parseCsvFile = (csvFile, parserOptions = {}) => {
+  return new Promise((resolve, reject) => {
     const opts = {
       ...defaultCsvParserOptions,
       ...setupCsvParserOptions(parserOptions),
-      complete: (results) => {
+      complete: results => {
         if (results && results.errors && results.errors.length) {
           return reject("Error in csv: " + JSON.stringify(results.errors));
         }
         resolve(results);
       },
-      error: (error) => {
+      error: error => {
         reject(error);
       }
     };
@@ -128,11 +146,28 @@ var parseCsvFile = (csvFile, parserOptions = {}) => {
     parse(csvFile.originFileObj, opts);
   });
 };
-var jsonToCsv = (jsonData, options = {}) => {
+
+/**
+ * Gets a properly formatted json object into s csv string
+ * https://www.papaparse.com/docs#json-to-csv
+ * const options = {
+      quotes: false, //or array of booleans
+      quoteChar: '"',
+      escapeChar: '"',
+      delimiter: ",",
+      header: true,
+      newline: "\r\n",
+      skipEmptyLines: false, //other option is 'greedy', meaning skip delimiters, quotes, and whitespace.
+      columns: null //or array of strings
+    }
+ * @returns csv as string
+ */
+export const jsonToCsv = (jsonData, options = {}) => {
   const csv = unparse(jsonData, options);
   return csv;
 };
-var parseCsvString = (csvString, parserOptions = {}) => {
+
+export const parseCsvString = (csvString, parserOptions = {}) => {
   const opts = {
     ...defaultCsvParserOptions,
     ...setupCsvParserOptions(parserOptions)
@@ -140,25 +175,27 @@ var parseCsvString = (csvString, parserOptions = {}) => {
   logDebug(`[CSV-PARSER] parseCsvString opts:`, opts);
   return parse(csvString, opts);
 };
-async function parseCsvOrExcelFile(fileOrFiles, { csvParserOptions } = {}) {
+
+export async function parseCsvOrExcelFile(
+  fileOrFiles,
+  { csvParserOptions } = {}
+) {
   let csvFile, excelFile, txtFile;
   if (Array.isArray(fileOrFiles)) {
     csvFile = fileOrFiles.find(isCsvFile);
     excelFile = fileOrFiles.find(isExcelFile);
     txtFile = fileOrFiles.find(isTextFile);
   } else {
-    if (isExcelFile(fileOrFiles))
-      excelFile = fileOrFiles;
-    else if (isCsvFile(fileOrFiles))
-      csvFile = fileOrFiles;
-    else if (isTextFile(fileOrFiles))
-      txtFile = fileOrFiles;
+    if (isExcelFile(fileOrFiles)) excelFile = fileOrFiles;
+    else if (isCsvFile(fileOrFiles)) csvFile = fileOrFiles;
+    else if (isTextFile(fileOrFiles)) txtFile = fileOrFiles;
   }
   if (!csvFile && !excelFile && !txtFile) {
     throw new Error("No csv or excel files found");
   }
-  if (!csvFile && !excelFile)
-    csvFile = txtFile;
+
+  if (!csvFile && !excelFile) csvFile = txtFile;
+
   if (!csvFile && excelFile && window.parseExcelToCsv) {
     csvFile = await window.parseExcelToCsv(
       excelFile.originFileObj || excelFile
@@ -173,8 +210,13 @@ async function parseCsvOrExcelFile(fileOrFiles, { csvParserOptions } = {}) {
   parsedCsv.originalFile = csvFile;
   return parsedCsv;
 }
-var validateCSVRequiredHeaders = (fields, requiredHeaders, filename) => {
-  const missingRequiredHeaders = requiredHeaders.filter((field) => {
+
+export const validateCSVRequiredHeaders = (
+  fields,
+  requiredHeaders,
+  filename
+) => {
+  const missingRequiredHeaders = requiredHeaders.filter(field => {
     return !fields.includes(field);
   });
   if (missingRequiredHeaders.length) {
@@ -184,64 +226,91 @@ var validateCSVRequiredHeaders = (fields, requiredHeaders, filename) => {
     )})`;
   }
 };
-var validateCSVRow = (row, requiredHeaders, index) => {
-  const missingRequiredFields = requiredHeaders.filter((field) => !row[field]);
+
+export const validateCSVRow = (row, requiredHeaders, index) => {
+  const missingRequiredFields = requiredHeaders.filter(field => !row[field]);
   if (missingRequiredFields.length) {
     if (missingRequiredFields.length === 1) {
-      return `Row ${index + 1} is missing the required field "${missingRequiredFields[0]}"`;
+      return `Row ${index + 1} is missing the required field "${
+        missingRequiredFields[0]
+      }"`;
     } else {
-      return `Row ${index + 1} is missing these required fields: ${missingRequiredFields.join(
+      return `Row ${index +
+        1} is missing these required fields: ${missingRequiredFields.join(
         ", "
       )}`;
     }
   }
 };
-var cleanCommaSeparatedCell = (cellData) => (cellData || "").split(",").map((n) => n.trim()).filter((n) => n);
-var cleanCsvExport = (rows) => {
+
+export const cleanCommaSeparatedCell = cellData =>
+  (cellData || "")
+    .split(",")
+    .map(n => n.trim())
+    .filter(n => n);
+
+/**
+ * Because the csv rows might not have the same header keys in some cases (extended properties)
+ * this function will make sure that each row will have all headers so that the export
+ * does not drop fields
+ * @param {*} rows
+ */
+export const cleanCsvExport = rows => {
   const allHeaders = [];
-  rows.forEach((row) => {
-    Object.keys(row).forEach((header) => {
+  rows.forEach(row => {
+    Object.keys(row).forEach(header => {
       if (!allHeaders.includes(header)) {
         allHeaders.push(header);
       }
     });
   });
-  rows.forEach((row) => {
-    allHeaders.forEach((header) => {
+  rows.forEach(row => {
+    allHeaders.forEach(header => {
       row[header] = row[header] || "";
     });
   });
   return rows;
 };
-var filterFilesInZip = async (file, accepted) => {
+
+export const filterFilesInZip = async (file, accepted) => {
   const zipExtracted = await extractZipFiles(file);
   const acceptedFiles = [];
   for (const extFile of zipExtracted) {
     const extension = "." + getExt(extFile);
-    if (accepted.some((ext) => ext === extension)) {
+    if (accepted.some(ext => ext === extension)) {
       acceptedFiles.push(extFile);
     }
   }
+
   if (acceptedFiles.length && acceptedFiles.length < zipExtracted.length)
     window.toastr.warning("Some files don't have the proper file extension.");
+
   if (!acceptedFiles.length)
     window.toastr.warning("No files with the proper extension were found.");
+
   return acceptedFiles;
 };
-function removeExt(filename) {
+
+export function removeExt(filename) {
   if (filename && filename.includes(".")) {
-    return filename.split(".").slice(0, -1).join(".");
+    return filename
+      .split(".")
+      .slice(0, -1)
+      .join(".");
   } else {
     return filename;
   }
 }
-async function uploadAndProcessFiles(files = []) {
-  if (!files.length)
-    return null;
+
+export async function uploadAndProcessFiles(files = []) {
+  if (!files.length) return null;
+
   const formData = new FormData();
   files.forEach(({ originFileObj }) => formData.append("file", originFileObj));
+
   const response = await window.api.post("/user_uploads/", formData);
-  return response.data.map((d) => ({
+
+  return response.data.map(d => ({
     encoding: d.encoding,
     mimetype: d.mimetype,
     originalname: d.originalname,
@@ -249,7 +318,8 @@ async function uploadAndProcessFiles(files = []) {
     size: d.size
   }));
 }
-async function encodeFilesForRequest(files) {
+
+export async function encodeFilesForRequest(files) {
   const encodedFiles = [];
   for (const file of files) {
     const encoded = await fileToBase64(file.originalFileObj);
@@ -262,36 +332,16 @@ async function encodeFilesForRequest(files) {
   }
   return encodedFiles;
 }
-var fileToBase64 = (file) => {
-  return new Promise2((resolve) => {
+
+const fileToBase64 = file => {
+  return new Promise(resolve => {
     const reader = new FileReader();
+    // Read file content on file loaded event
     reader.onload = function(event) {
       resolve(event.target.result);
     };
+
+    // Convert data to base64
     reader.readAsDataURL(file);
   });
 };
-export {
-  allowedCsvFileTypes,
-  cleanCommaSeparatedCell,
-  cleanCsvExport,
-  encodeFilesForRequest,
-  extractZipFiles,
-  filterFilesInZip,
-  getExt,
-  isCsvFile,
-  isCsvOrExcelFile,
-  isExcelFile,
-  isTextFile,
-  isZipFile,
-  jsonToCsv,
-  normalizeCsvHeader,
-  parseCsvFile,
-  parseCsvOrExcelFile,
-  parseCsvString,
-  removeExt,
-  setupCsvParserOptions,
-  uploadAndProcessFiles,
-  validateCSVRequiredHeaders,
-  validateCSVRow
-};

package/src/file-utils.spec.js

@@ -0,0 +1,6 @@
+import { fileUtils } from './file-utils';
+describe('fileUtils', () => {
+  it('should work', () => {
+    expect(fileUtils()).toEqual('file-utils');
+  });
+});

package/src/index.js

@@ -0,0 +1 @@
+export * from './file-utils';

package/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "allowJs": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "noImplicitOverride": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

package/tsconfig.lib.json

@@ -0,0 +1,16 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts", "src/**/*.js"],
+  "exclude": [
+    "jest.config.ts",
+    "src/**/*.spec.ts",
+    "src/**/*.test.ts",
+    "src/**/*.spec.js",
+    "src/**/*.test.js"
+  ]
+}

package/tsconfig.spec.json

@@ -0,0 +1,14 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": [
+    "jest.config.ts",
+    "src/**/*.test.ts",
+    "src/**/*.spec.ts",
+    "src/**/*.d.ts"
+  ]
+}