@budibase/server 2.2.12-alpha.5 → 2.2.12-alpha.51

package/dist/utilities/csvParser.js

@@ -1,152 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.transform = exports.updateSchema = exports.parse = void 0;
-const csvtojson_1 = __importDefault(require("csvtojson"));
-const constants_1 = require("../constants");
-const VALIDATORS = {
-    [constants_1.FieldTypes.STRING]: () => true,
-    [constants_1.FieldTypes.OPTIONS]: () => true,
-    [constants_1.FieldTypes.BARCODEQR]: () => true,
-    [constants_1.FieldTypes.NUMBER]: (attribute) => {
-        // allow not to be present
-        if (!attribute) {
-            return true;
-        }
-        return !isNaN(Number(attribute));
-    },
-    [constants_1.FieldTypes.DATETIME]: (attribute) => {
-        // allow not to be present
-        if (!attribute) {
-            return true;
-        }
-        return !isNaN(new Date(attribute).getTime());
-    },
-};
-const PARSERS = {
-    [constants_1.FieldTypes.NUMBER]: (attribute) => {
-        if (!attribute) {
-            return attribute;
-        }
-        return Number(attribute);
-    },
-    [constants_1.FieldTypes.DATETIME]: (attribute) => {
-        if (!attribute) {
-            return attribute;
-        }
-        return new Date(attribute).toISOString();
-    },
-};
-function parse(csvString, parsers) {
-    const result = (0, csvtojson_1.default)().fromString(csvString);
-    const schema = {};
-    return new Promise((resolve, reject) => {
-        result.on("header", headers => {
-            for (let header of headers) {
-                schema[header] = {
-                    type: parsers[header] ? parsers[header].type : "string",
-                    success: true,
-                };
-            }
-        });
-        result.subscribe(row => {
-            // For each CSV row parse all the columns that need parsed
-            for (let key of Object.keys(parsers)) {
-                if (!schema[key] || schema[key].success) {
-                    // get the validator for the column type
-                    const validator = VALIDATORS[parsers[key].type];
-                    try {
-                        // allow null/undefined values
-                        schema[key].success = !row[key] || validator(row[key]);
-                    }
-                    catch (err) {
-                        schema[key].success = false;
-                    }
-                }
-            }
-        });
-        result.on("done", error => {
-            if (error) {
-                console.error(error);
-                reject(error);
-            }
-            resolve(schema);
-        });
-    });
-}
-exports.parse = parse;
-function updateSchema({ schema, existingTable, }) {
-    if (!schema) {
-        return schema;
-    }
-    const finalSchema = {};
-    const schemaKeyMap = {};
-    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
-    for (let [key, field] of Object.entries((existingTable === null || existingTable === void 0 ? void 0 : existingTable.schema) || {})) {
-        const lcKey = key.toLowerCase();
-        const foundKey = schemaKeyMap[lcKey];
-        if (foundKey) {
-            finalSchema[key] = schema[foundKey];
-            finalSchema[key].type = field.type;
-        }
-    }
-    return finalSchema;
-}
-exports.updateSchema = updateSchema;
-function transform({ schema, csvString, existingTable, }) {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (!schema || !csvString) {
-            throw new Error("Unable to transform CSV without schema");
-        }
-        const colParser = {};
-        // make sure the table has all the columns required for import
-        if (existingTable) {
-            schema = updateSchema({ schema, existingTable });
-        }
-        for (let [key, field] of Object.entries(schema || {})) {
-            // don't import data to auto columns
-            if (!field.autocolumn) {
-                colParser[key] = PARSERS[field.type] || field.type;
-            }
-        }
-        try {
-            const data = yield (0, csvtojson_1.default)({ colParser }).fromString(csvString);
-            const schemaKeyMap = {};
-            Object.keys(schema || {}).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
-            for (let element of data) {
-                if (!data) {
-                    continue;
-                }
-                for (let key of Object.keys(element)) {
-                    const mappedKey = schemaKeyMap[key.toLowerCase()];
-                    // isn't a column in the table, remove it
-                    if (mappedKey == null) {
-                        delete element[key];
-                    }
-                    // casing is different, fix it in row
-                    else if (key !== mappedKey) {
-                        element[mappedKey] = element[key];
-                        delete element[key];
-                    }
-                }
-            }
-            return data;
-        }
-        catch (err) {
-            console.error(`Error transforming CSV to JSON for data import`, err);
-            throw err;
-        }
-    });
-}
-exports.transform = transform;

package/src/utilities/csvParser.ts

@@ -1,161 +0,0 @@
-import { FieldSchema, Table } from "@budibase/types"
-import csv from "csvtojson"
-import { FieldTypes } from "../constants"
-
-type CsvParseOpts = {
-  schema?: { [key: string]: any }
-  existingTable: Table
-  csvString?: string
-}
-
-const VALIDATORS: any = {
-  [FieldTypes.STRING]: () => true,
-  [FieldTypes.OPTIONS]: () => true,
-  [FieldTypes.BARCODEQR]: () => true,
-  [FieldTypes.NUMBER]: (attribute?: string) => {
-    // allow not to be present
-    if (!attribute) {
-      return true
-    }
-    return !isNaN(Number(attribute))
-  },
-  [FieldTypes.DATETIME]: (attribute?: string) => {
-    // allow not to be present
-    if (!attribute) {
-      return true
-    }
-    return !isNaN(new Date(attribute).getTime())
-  },
-}
-
-const PARSERS: any = {
-  [FieldTypes.NUMBER]: (attribute?: string) => {
-    if (!attribute) {
-      return attribute
-    }
-    return Number(attribute)
-  },
-  [FieldTypes.DATETIME]: (attribute?: string) => {
-    if (!attribute) {
-      return attribute
-    }
-    return new Date(attribute).toISOString()
-  },
-}
-
-export function parse(csvString: string, parsers: any): Record<string, any> {
-  const result = csv().fromString(csvString)
-
-  const schema: Record<string, any> = {}
-
-  return new Promise((resolve, reject) => {
-    result.on("header", headers => {
-      for (let header of headers) {
-        schema[header] = {
-          type: parsers[header] ? parsers[header].type : "string",
-          success: true,
-        }
-      }
-    })
-    result.subscribe(row => {
-      // For each CSV row parse all the columns that need parsed
-      for (let key of Object.keys(parsers)) {
-        if (!schema[key] || schema[key].success) {
-          // get the validator for the column type
-          const validator = VALIDATORS[parsers[key].type]
-
-          try {
-            // allow null/undefined values
-            schema[key].success = !row[key] || validator(row[key])
-          } catch (err) {
-            schema[key].success = false
-          }
-        }
-      }
-    })
-    result.on("done", error => {
-      if (error) {
-        console.error(error)
-        reject(error)
-      }
-
-      resolve(schema)
-    })
-  })
-}
-
-export function updateSchema({
-  schema,
-  existingTable,
-}: {
-  schema?: Record<string, any>
-  existingTable?: Table
-}) {
-  if (!schema) {
-    return schema
-  }
-  const finalSchema: Record<string, FieldSchema> = {}
-  const schemaKeyMap: Record<string, any> = {}
-  Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
-  for (let [key, field] of Object.entries(existingTable?.schema || {})) {
-    const lcKey = key.toLowerCase()
-    const foundKey: string = schemaKeyMap[lcKey]
-    if (foundKey) {
-      finalSchema[key] = schema[foundKey]
-      finalSchema[key].type = field.type
-    }
-  }
-  return finalSchema
-}
-
-export async function transform({
-  schema,
-  csvString,
-  existingTable,
-}: CsvParseOpts) {
-  if (!schema || !csvString) {
-    throw new Error("Unable to transform CSV without schema")
-  }
-  const colParser: any = {}
-
-  // make sure the table has all the columns required for import
-  if (existingTable) {
-    schema = updateSchema({ schema, existingTable })
-  }
-
-  for (let [key, field] of Object.entries(schema || {})) {
-    // don't import data to auto columns
-    if (!field.autocolumn) {
-      colParser[key] = PARSERS[field.type] || field.type
-    }
-  }
-
-  try {
-    const data = await csv({ colParser }).fromString(csvString)
-    const schemaKeyMap: any = {}
-    Object.keys(schema || {}).forEach(
-      key => (schemaKeyMap[key.toLowerCase()] = key)
-    )
-    for (let element of data) {
-      if (!data) {
-        continue
-      }
-      for (let key of Object.keys(element)) {
-        const mappedKey = schemaKeyMap[key.toLowerCase()]
-        // isn't a column in the table, remove it
-        if (mappedKey == null) {
-          delete element[key]
-        }
-        // casing is different, fix it in row
-        else if (key !== mappedKey) {
-          element[mappedKey] = element[key]
-          delete element[key]
-        }
-      }
-    }
-    return data
-  } catch (err) {
-    console.error(`Error transforming CSV to JSON for data import`, err)
-    throw err
-  }
-}

package/src/utilities/tests/__snapshots__/csvParser.spec.js.snap

@@ -1,15 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = `
-Array [
-  Object {
-    "Age": 4324,
-  },
-  Object {
-    "Age": 34,
-  },
-  Object {
-    "Age": 23423,
-  },
-]
-`;

package/src/utilities/tests/csvParser.spec.js

@@ -1,112 +0,0 @@
-const { readFileSync } = require("../fileSystem")
-const csvParser = require("../csvParser")
-
-const CSV_PATH = __dirname + "/test.csv"
-
-const SCHEMAS = {
-  VALID: {
-    Age: {
-      type: "number",
-    },
-  },
-  INVALID: {
-    Address: {
-      type: "number",
-    },
-    Age: {
-      type: "number",
-    },
-  },
-  IGNORE: {
-    Address: {
-      type: "omit",
-    },
-    Age: {
-      type: "omit",
-    },
-    Name: {
-      type: "string",
-    },
-  },
-  BROKEN: {
-    Address: {
-      type: "datetime",
-    },
-  },
-}
-
-describe("CSV Parser", () => {
-  const csvString = readFileSync(CSV_PATH, "utf8")
-
-  describe("parsing", () => {
-    it("returns status and types for a valid CSV transformation", async () => {
-      expect(await csvParser.parse(csvString, SCHEMAS.VALID)).toEqual({
-        Address: {
-          success: true,
-          type: "string",
-        },
-        Age: {
-          success: true,
-          type: "number",
-        },
-        Name: {
-          success: true,
-          type: "string",
-        },
-      })
-    })
-
-    it("returns status and types for an invalid CSV transformation", async () => {
-      expect(await csvParser.parse(csvString, SCHEMAS.INVALID)).toEqual({
-        Address: {
-          success: false,
-          type: "number",
-        },
-        Age: {
-          success: true,
-          type: "number",
-        },
-        Name: {
-          success: true,
-          type: "string",
-        },
-      })
-    })
-  })
-
-  describe("transformation", () => {
-    it("transforms a CSV file into JSON", async () => {
-      expect(
-        await csvParser.transform({
-          schema: SCHEMAS.VALID,
-          csvString,
-        })
-      ).toMatchSnapshot()
-    })
-
-    it("transforms a CSV file into JSON ignoring certain fields", async () => {
-      expect(
-        await csvParser.transform({
-          schema: SCHEMAS.IGNORE,
-          csvString,
-        })
-      ).toEqual([
-        {
-          Name: "Bertå",
-        },
-        {
-          Name: "Ernie",
-        },
-        {
-          Name: "Big Bird",
-        },
-      ])
-    })
-
-    it("throws an error on invalid schema", async () => {
-      await expect(
-        csvParser.transform({ schema: SCHEMAS.BROKEN, csvString })
-      ).rejects.toThrow()
-    })
-  })
-})