@teselagen/file-utils 0.0.2

package/README.md

@@ -0,0 +1,11 @@
+# file-utils
+
+This library was generated with [Nx](https://nx.dev).
+
+## Building
+
+Run `nx build file-utils` to build the library.
+
+## Running unit tests
+
+Run `nx test file-utils` to execute the unit tests via [Jest](https://jestjs.io).

package/index.js

@@ -0,0 +1,297 @@
+// packages/file-utils/src/lib/file-utils.js
+import { camelCase, flatMap, remove, startsWith, snakeCase } from "lodash";
+import { loadAsync } from "jszip";
+import Promise2 from "bluebird";
+import { parse, unparse } from "papaparse";
+var logDebug = (...args) => {
+  if (process.env.DEBUG_CSV_PARSING) {
+    console.log(...args);
+  }
+};
+var allowedCsvFileTypes = [".csv", ".txt", ".xlsx"];
+var isZipFile = (file) => {
+  const type = file.mimetype || file.type;
+  return type === "application/zip" || type === "application/x-zip-compressed";
+};
+var getExt = (file) => file.name.split(".").pop();
+var isExcelFile = (file) => getExt(file) === "xlsx";
+var isCsvFile = (file) => getExt(file) === "csv";
+var isTextFile = (file) => ["text", "txt"].includes(getExt(file));
+var isCsvOrExcelFile = (file) => isCsvFile(file) || isExcelFile(file);
+var extractZipFiles = async (allFiles) => {
+  if (!Array.isArray(allFiles))
+    allFiles = [allFiles];
+  allFiles = [...allFiles];
+  const zipFiles = remove(allFiles, isZipFile);
+  if (!zipFiles.length)
+    return allFiles;
+  const zipFilesArray = Array.isArray(zipFiles) ? zipFiles : [zipFiles];
+  const parsedZips = await Promise2.map(
+    zipFilesArray,
+    (file) => loadAsync(file instanceof Blob ? file : file.originFileObj)
+  );
+  const zippedFiles = flatMap(
+    parsedZips,
+    (zip) => Object.keys(zip.files).map((key) => zip.files[key])
+  );
+  const unzippedFiles = await Promise2.map(zippedFiles, (file) => {
+    return file.async("blob").then(function(fileData) {
+      const newFileObj = new File([fileData], file.name);
+      return {
+        name: file.name,
+        originFileObj: newFileObj,
+        originalFileObj: newFileObj
+      };
+    });
+  });
+  if (unzippedFiles.length) {
+    return allFiles.concat(
+      unzippedFiles.filter(
+        ({ name, originFileObj }) => !name.includes("__MACOSX") && !name.includes(".DS_Store") && originFileObj.size !== 0
+      )
+    );
+  } else {
+    return allFiles;
+  }
+};
+var defaultCsvParserOptions = {
+  header: true,
+  skipEmptyLines: "greedy",
+  trimHeaders: true
+};
+var setupCsvParserOptions = (parserOptions = {}) => {
+  const {
+    camelCaseHeaders = false,
+    lowerCaseHeaders = false,
+    ...rest
+  } = parserOptions;
+  const papaParseOpts = { ...rest };
+  if (camelCaseHeaders) {
+    logDebug("[CSV-PARSER] camelCasing headers");
+    papaParseOpts.transformHeader = (header) => {
+      let transHeader = header;
+      if (!startsWith(header.trim(), "ext-")) {
+        transHeader = camelCase(header);
+      }
+      if (transHeader) {
+        logDebug(
+          `[CSV-PARSER] Transformed header from: ${header} to: ${transHeader}`
+        );
+        transHeader = transHeader.trim();
+      } else {
+        logDebug(`[CSV-PARSER] Not transforming header: ${header}`);
+      }
+      return transHeader;
+    };
+  } else if (lowerCaseHeaders) {
+    papaParseOpts.transformHeader = (header) => {
+      let transHeader = header;
+      if (!startsWith(header, "ext-")) {
+        transHeader = header.toLowerCase();
+      }
+      if (transHeader) {
+        logDebug(
+          `[CSV-PARSER] Transformed header from: ${header} to: ${transHeader}`
+        );
+        transHeader = transHeader.trim();
+      } else {
+        logDebug(`[CSV-PARSER] Not transforming header: ${header}`);
+      }
+      return transHeader;
+    };
+  }
+  return papaParseOpts;
+};
+var normalizeCsvHeaderHelper = (h) => snakeCase(h.toUpperCase()).toUpperCase();
+function normalizeCsvHeader(header) {
+  if (header.startsWith("ext-") || header.startsWith("EXT-")) {
+    return header;
+  }
+  return normalizeCsvHeaderHelper(header);
+}
+var parseCsvFile = (csvFile, parserOptions = {}) => {
+  return new Promise2((resolve, reject) => {
+    const opts = {
+      ...defaultCsvParserOptions,
+      ...setupCsvParserOptions(parserOptions),
+      complete: (results) => {
+        if (results && results.errors && results.errors.length) {
+          return reject("Error in csv: " + JSON.stringify(results.errors));
+        }
+        resolve(results);
+      },
+      error: (error) => {
+        reject(error);
+      }
+    };
+    logDebug(`[CSV-PARSER] parseCsvFile opts:`, opts);
+    parse(csvFile.originFileObj, opts);
+  });
+};
+var jsonToCsv = (jsonData, options = {}) => {
+  const csv = unparse(jsonData, options);
+  return csv;
+};
+var parseCsvString = (csvString, parserOptions = {}) => {
+  const opts = {
+    ...defaultCsvParserOptions,
+    ...setupCsvParserOptions(parserOptions)
+  };
+  logDebug(`[CSV-PARSER] parseCsvString opts:`, opts);
+  return parse(csvString, opts);
+};
+async function parseCsvOrExcelFile(fileOrFiles, { csvParserOptions } = {}) {
+  let csvFile, excelFile, txtFile;
+  if (Array.isArray(fileOrFiles)) {
+    csvFile = fileOrFiles.find(isCsvFile);
+    excelFile = fileOrFiles.find(isExcelFile);
+    txtFile = fileOrFiles.find(isTextFile);
+  } else {
+    if (isExcelFile(fileOrFiles))
+      excelFile = fileOrFiles;
+    else if (isCsvFile(fileOrFiles))
+      csvFile = fileOrFiles;
+    else if (isTextFile(fileOrFiles))
+      txtFile = fileOrFiles;
+  }
+  if (!csvFile && !excelFile && !txtFile) {
+    throw new Error("No csv or excel files found");
+  }
+  if (!csvFile && !excelFile)
+    csvFile = txtFile;
+  if (!csvFile && excelFile && window.parseExcelToCsv) {
+    csvFile = await window.parseExcelToCsv(
+      excelFile.originFileObj || excelFile
+    );
+    if (csvFile.error) {
+      throw new Error(csvFile.error);
+    }
+  } else if (excelFile) {
+    throw new Error("Excel Parser not initialized on the window");
+  }
+  const parsedCsv = await parseCsvFile(csvFile, csvParserOptions);
+  parsedCsv.originalFile = csvFile;
+  return parsedCsv;
+}
+var validateCSVRequiredHeaders = (fields, requiredHeaders, filename) => {
+  const missingRequiredHeaders = requiredHeaders.filter((field) => {
+    return !fields.includes(field);
+  });
+  if (missingRequiredHeaders.length) {
+    const name = filename ? `The file ${filename}` : "CSV file";
+    return `${name} is missing required headers. (${missingRequiredHeaders.join(
+      ", "
+    )})`;
+  }
+};
+var validateCSVRow = (row, requiredHeaders, index) => {
+  const missingRequiredFields = requiredHeaders.filter((field) => !row[field]);
+  if (missingRequiredFields.length) {
+    if (missingRequiredFields.length === 1) {
+      return `Row ${index + 1} is missing the required field "${missingRequiredFields[0]}"`;
+    } else {
+      return `Row ${index + 1} is missing these required fields: ${missingRequiredFields.join(
+        ", "
+      )}`;
+    }
+  }
+};
+var cleanCommaSeparatedCell = (cellData) => (cellData || "").split(",").map((n) => n.trim()).filter((n) => n);
+var cleanCsvExport = (rows) => {
+  const allHeaders = [];
+  rows.forEach((row) => {
+    Object.keys(row).forEach((header) => {
+      if (!allHeaders.includes(header)) {
+        allHeaders.push(header);
+      }
+    });
+  });
+  rows.forEach((row) => {
+    allHeaders.forEach((header) => {
+      row[header] = row[header] || "";
+    });
+  });
+  return rows;
+};
+var filterFilesInZip = async (file, accepted) => {
+  const zipExtracted = await extractZipFiles(file);
+  const acceptedFiles = [];
+  for (const extFile of zipExtracted) {
+    const extension = "." + getExt(extFile);
+    if (accepted.some((ext) => ext === extension)) {
+      acceptedFiles.push(extFile);
+    }
+  }
+  if (acceptedFiles.length && acceptedFiles.length < zipExtracted.length)
+    window.toastr.warning("Some files don't have the proper file extension.");
+  if (!acceptedFiles.length)
+    window.toastr.warning("No files with the proper extension were found.");
+  return acceptedFiles;
+};
+function removeExt(filename) {
+  if (filename && filename.includes(".")) {
+    return filename.split(".").slice(0, -1).join(".");
+  } else {
+    return filename;
+  }
+}
+async function uploadAndProcessFiles(files = []) {
+  if (!files.length)
+    return null;
+  const formData = new FormData();
+  files.forEach(({ originFileObj }) => formData.append("file", originFileObj));
+  const response = await window.api.post("/user_uploads/", formData);
+  return response.data.map((d) => ({
+    encoding: d.encoding,
+    mimetype: d.mimetype,
+    originalname: d.originalname,
+    path: d.path,
+    size: d.size
+  }));
+}
+async function encodeFilesForRequest(files) {
+  const encodedFiles = [];
+  for (const file of files) {
+    const encoded = await fileToBase64(file.originalFileObj);
+    const data = encoded.split(",");
+    encodedFiles.push({
+      type: file.type,
+      base64Data: data[1],
+      name: file.name
+    });
+  }
+  return encodedFiles;
+}
+var fileToBase64 = (file) => {
+  return new Promise2((resolve) => {
+    const reader = new FileReader();
+    reader.onload = function(event) {
+      resolve(event.target.result);
+    };
+    reader.readAsDataURL(file);
+  });
+};
+export {
+  allowedCsvFileTypes,
+  cleanCommaSeparatedCell,
+  cleanCsvExport,
+  encodeFilesForRequest,
+  extractZipFiles,
+  filterFilesInZip,
+  getExt,
+  isCsvFile,
+  isCsvOrExcelFile,
+  isExcelFile,
+  isTextFile,
+  isZipFile,
+  jsonToCsv,
+  normalizeCsvHeader,
+  parseCsvFile,
+  parseCsvOrExcelFile,
+  parseCsvString,
+  removeExt,
+  setupCsvParserOptions,
+  uploadAndProcessFiles,
+  validateCSVRequiredHeaders,
+  validateCSVRow
+};

package/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@teselagen/file-utils",
+  "version": "0.0.2",
+  "type": "module",
+  "dependencies": {
+    "bluebird": "^3.7.2",
+    "jszip": "^3.10.1",
+    "lodash": "^4.17.21",
+    "papaparse": "^5.4.1"
+  },
+  "module": "index.js",
+  "main": "index.js"
+}