@budibase/server 2.2.12-alpha.20 → 2.2.12-alpha.21

package/src/api/routes/tests/table.spec.js

@@ -43,21 +43,18 @@ describe("/tables", () => {
       expect(events.table.created).toBeCalledWith(res.body)
     })
 
-    it("creates a table via data import CSV", async () => {
+    it("creates a table via data import", async () => {
       const table = basicTable()
-      table.dataImport = {
-        csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"",
-      }
-      table.dataImport.schema = table.schema
+      table.rows = [{ name: 'test-name', description: 'test-desc' }]
 
       const res = await createTable(table)
 
       expect(events.table.created).toBeCalledTimes(1)
       expect(events.table.created).toBeCalledWith(res.body)
       expect(events.table.imported).toBeCalledTimes(1)
-      expect(events.table.imported).toBeCalledWith(res.body, "csv")
+      expect(events.table.imported).toBeCalledWith(res.body)
       expect(events.rows.imported).toBeCalledTimes(1)
-      expect(events.rows.imported).toBeCalledWith(res.body, "csv", 1)
+      expect(events.rows.imported).toBeCalledWith(res.body, 1)
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -155,11 +152,10 @@ describe("/tables", () => {
     it("imports rows successfully", async () => {
       const table = await config.createTable()
       const importRequest = {
-        dataImport: {
-          csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"",
-          schema: table.schema
-        }
+        schema: table.schema,
+        rows: [{ name: 'test-name', description: 'test-desc' }]
       }
+
       jest.clearAllMocks()
 
       await request
@@ -171,7 +167,7 @@ describe("/tables", () => {
 
       expect(events.table.created).not.toHaveBeenCalled()
       expect(events.rows.imported).toBeCalledTimes(1)
-      expect(events.rows.imported).toBeCalledWith(table, "csv", 1)
+      expect(events.rows.imported).toBeCalledWith(table, 1)
     })
   })
 
@@ -206,24 +202,6 @@ describe("/tables", () => {
     })
   })
 
-  describe("validate csv", () => {
-    it("should be able to validate a CSV layout", async () => {
-      const res = await request
-        .post(`/api/tables/csv/validate`)
-        .send({
-          csvString: "a,b,c,d\n1,2,3,4"
-        })
-        .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
-        .expect(200)
-      expect(res.body.schema).toBeDefined()
-      expect(res.body.schema.a).toEqual({
-        type: "string",
-        success: true,
-      })
-    })
-  })
-
   describe("indexing", () => {
     it("should be able to create a table with indexes", async () => {
       await context.doInAppContext(appId, async () => {

package/src/api/routes/utils/validators.ts

@@ -18,7 +18,7 @@ export function tableValidator() {
     schema: Joi.object().required(),
     name: Joi.string().required(),
     views: Joi.object(),
-    dataImport: Joi.object(),
+    rows: Joi.array(),
   }).unknown(true))
 }
 

package/src/utilities/schema.ts

@@ -0,0 +1,141 @@
+import { FieldTypes } from "../constants"
+
+interface SchemaColumn {
+  readonly name: string
+  readonly type: FieldTypes
+  readonly autocolumn?: boolean
+}
+
+interface Schema {
+  readonly [index: string]: SchemaColumn
+}
+
+interface Row {
+  [index: string]: any
+}
+
+type Rows = Array<Row>
+
+interface SchemaValidation {
+  [index: string]: boolean
+}
+
+interface ValidationResults {
+  schemaValidation: SchemaValidation
+  allValid: boolean
+  invalidColumns: Array<string>
+}
+
+const PARSERS: any = {
+  [FieldTypes.NUMBER]: (attribute?: string) => {
+    if (!attribute) {
+      return attribute
+    }
+    return Number(attribute)
+  },
+  [FieldTypes.DATETIME]: (attribute?: string) => {
+    if (!attribute) {
+      return attribute
+    }
+    return new Date(attribute).toISOString()
+  },
+}
+
+export function isSchema(schema: any): schema is Schema {
+  return (
+    typeof schema === "object" &&
+    Object.values(schema).every(rawColumn => {
+      const column = rawColumn as SchemaColumn
+
+      return (
+        column !== null &&
+        typeof column === "object" &&
+        typeof column.type === "string" &&
+        Object.values(FieldTypes).includes(column.type as FieldTypes)
+      )
+    })
+  )
+}
+
+export function isRows(rows: any): rows is Rows {
+  return Array.isArray(rows) && rows.every(row => typeof row === "object")
+}
+
+export function validate(rows: Rows, schema: Schema): ValidationResults {
+  const results: ValidationResults = {
+    schemaValidation: {},
+    allValid: false,
+    invalidColumns: [],
+  }
+
+  rows.forEach(row => {
+    Object.entries(row).forEach(([columnName, columnData]) => {
+      const columnType = schema[columnName]?.type
+      const isAutoColumn = schema[columnName]?.autocolumn
+
+      // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
+      if (typeof columnType !== "string") {
+        results.invalidColumns.push(columnName)
+      } else if (
+        // If there's no data for this field don't bother with further checks
+        // If the field is already marked as invalid there's no need for further checks
+        results.schemaValidation[columnName] === false ||
+        columnData == null ||
+        isAutoColumn
+      ) {
+        return
+      } else if (
+        columnType === FieldTypes.NUMBER &&
+        isNaN(Number(columnData))
+      ) {
+        // If provided must be a valid number
+        results.schemaValidation[columnName] = false
+      } else if (
+        // If provided must be a valid date
+        columnType === FieldTypes.DATETIME &&
+        isNaN(new Date(columnData).getTime())
+      ) {
+        results.schemaValidation[columnName] = false
+      } else {
+        results.schemaValidation[columnName] = true
+      }
+    })
+  })
+
+  results.allValid =
+    Object.values(results.schemaValidation).length > 0 &&
+    Object.values(results.schemaValidation).every(column => column)
+
+  // Select unique values
+  results.invalidColumns = [...new Set(results.invalidColumns)]
+  return results
+}
+
+export function parse(rows: Rows, schema: Schema): Rows {
+  return rows.map(row => {
+    const parsedRow: Row = {}
+
+    Object.entries(row).forEach(([columnName, columnData]) => {
+      if (!(columnName in schema) || schema[columnName]?.autocolumn) {
+        // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case
+        return
+      }
+
+      const columnType = schema[columnName].type
+
+      if (columnType === FieldTypes.NUMBER) {
+        // If provided must be a valid number
+        parsedRow[columnName] = columnData ? Number(columnData) : columnData
+      } else if (columnType === FieldTypes.DATETIME) {
+        // If provided must be a valid date
+        parsedRow[columnName] = columnData
+          ? new Date(columnData).toISOString()
+          : columnData
+      } else {
+        parsedRow[columnName] = columnData
+      }
+    })
+
+    return parsedRow
+  })
+}

package/dist/utilities/csvParser.js

@@ -1,152 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.transform = exports.updateSchema = exports.parse = void 0;
-const csvtojson_1 = __importDefault(require("csvtojson"));
-const constants_1 = require("../constants");
-const VALIDATORS = {
-    [constants_1.FieldTypes.STRING]: () => true,
-    [constants_1.FieldTypes.OPTIONS]: () => true,
-    [constants_1.FieldTypes.BARCODEQR]: () => true,
-    [constants_1.FieldTypes.NUMBER]: (attribute) => {
-        // allow not to be present
-        if (!attribute) {
-            return true;
-        }
-        return !isNaN(Number(attribute));
-    },
-    [constants_1.FieldTypes.DATETIME]: (attribute) => {
-        // allow not to be present
-        if (!attribute) {
-            return true;
-        }
-        return !isNaN(new Date(attribute).getTime());
-    },
-};
-const PARSERS = {
-    [constants_1.FieldTypes.NUMBER]: (attribute) => {
-        if (!attribute) {
-            return attribute;
-        }
-        return Number(attribute);
-    },
-    [constants_1.FieldTypes.DATETIME]: (attribute) => {
-        if (!attribute) {
-            return attribute;
-        }
-        return new Date(attribute).toISOString();
-    },
-};
-function parse(csvString, parsers) {
-    const result = (0, csvtojson_1.default)().fromString(csvString);
-    const schema = {};
-    return new Promise((resolve, reject) => {
-        result.on("header", headers => {
-            for (let header of headers) {
-                schema[header] = {
-                    type: parsers[header] ? parsers[header].type : "string",
-                    success: true,
-                };
-            }
-        });
-        result.subscribe(row => {
-            // For each CSV row parse all the columns that need parsed
-            for (let key of Object.keys(parsers)) {
-                if (!schema[key] || schema[key].success) {
-                    // get the validator for the column type
-                    const validator = VALIDATORS[parsers[key].type];
-                    try {
-                        // allow null/undefined values
-                        schema[key].success = !row[key] || validator(row[key]);
-                    }
-                    catch (err) {
-                        schema[key].success = false;
-                    }
-                }
-            }
-        });
-        result.on("done", error => {
-            if (error) {
-                console.error(error);
-                reject(error);
-            }
-            resolve(schema);
-        });
-    });
-}
-exports.parse = parse;
-function updateSchema({ schema, existingTable, }) {
-    if (!schema) {
-        return schema;
-    }
-    const finalSchema = {};
-    const schemaKeyMap = {};
-    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
-    for (let [key, field] of Object.entries((existingTable === null || existingTable === void 0 ? void 0 : existingTable.schema) || {})) {
-        const lcKey = key.toLowerCase();
-        const foundKey = schemaKeyMap[lcKey];
-        if (foundKey) {
-            finalSchema[key] = schema[foundKey];
-            finalSchema[key].type = field.type;
-        }
-    }
-    return finalSchema;
-}
-exports.updateSchema = updateSchema;
-function transform({ schema, csvString, existingTable, }) {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (!schema || !csvString) {
-            throw new Error("Unable to transform CSV without schema");
-        }
-        const colParser = {};
-        // make sure the table has all the columns required for import
-        if (existingTable) {
-            schema = updateSchema({ schema, existingTable });
-        }
-        for (let [key, field] of Object.entries(schema || {})) {
-            // don't import data to auto columns
-            if (!field.autocolumn) {
-                colParser[key] = PARSERS[field.type] || field.type;
-            }
-        }
-        try {
-            const data = yield (0, csvtojson_1.default)({ colParser }).fromString(csvString);
-            const schemaKeyMap = {};
-            Object.keys(schema || {}).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
-            for (let element of data) {
-                if (!data) {
-                    continue;
-                }
-                for (let key of Object.keys(element)) {
-                    const mappedKey = schemaKeyMap[key.toLowerCase()];
-                    // isn't a column in the table, remove it
-                    if (mappedKey == null) {
-                        delete element[key];
-                    }
-                    // casing is different, fix it in row
-                    else if (key !== mappedKey) {
-                        element[mappedKey] = element[key];
-                        delete element[key];
-                    }
-                }
-            }
-            return data;
-        }
-        catch (err) {
-            console.error(`Error transforming CSV to JSON for data import`, err);
-            throw err;
-        }
-    });
-}
-exports.transform = transform;

package/src/utilities/csvParser.ts

@@ -1,161 +0,0 @@
-import { FieldSchema, Table } from "@budibase/types"
-import csv from "csvtojson"
-import { FieldTypes } from "../constants"
-
-type CsvParseOpts = {
-  schema?: { [key: string]: any }
-  existingTable: Table
-  csvString?: string
-}
-
-const VALIDATORS: any = {
-  [FieldTypes.STRING]: () => true,
-  [FieldTypes.OPTIONS]: () => true,
-  [FieldTypes.BARCODEQR]: () => true,
-  [FieldTypes.NUMBER]: (attribute?: string) => {
-    // allow not to be present
-    if (!attribute) {
-      return true
-    }
-    return !isNaN(Number(attribute))
-  },
-  [FieldTypes.DATETIME]: (attribute?: string) => {
-    // allow not to be present
-    if (!attribute) {
-      return true
-    }
-    return !isNaN(new Date(attribute).getTime())
-  },
-}
-
-const PARSERS: any = {
-  [FieldTypes.NUMBER]: (attribute?: string) => {
-    if (!attribute) {
-      return attribute
-    }
-    return Number(attribute)
-  },
-  [FieldTypes.DATETIME]: (attribute?: string) => {
-    if (!attribute) {
-      return attribute
-    }
-    return new Date(attribute).toISOString()
-  },
-}
-
-export function parse(csvString: string, parsers: any): Record<string, any> {
-  const result = csv().fromString(csvString)
-
-  const schema: Record<string, any> = {}
-
-  return new Promise((resolve, reject) => {
-    result.on("header", headers => {
-      for (let header of headers) {
-        schema[header] = {
-          type: parsers[header] ? parsers[header].type : "string",
-          success: true,
-        }
-      }
-    })
-    result.subscribe(row => {
-      // For each CSV row parse all the columns that need parsed
-      for (let key of Object.keys(parsers)) {
-        if (!schema[key] || schema[key].success) {
-          // get the validator for the column type
-          const validator = VALIDATORS[parsers[key].type]
-
-          try {
-            // allow null/undefined values
-            schema[key].success = !row[key] || validator(row[key])
-          } catch (err) {
-            schema[key].success = false
-          }
-        }
-      }
-    })
-    result.on("done", error => {
-      if (error) {
-        console.error(error)
-        reject(error)
-      }
-
-      resolve(schema)
-    })
-  })
-}
-
-export function updateSchema({
-  schema,
-  existingTable,
-}: {
-  schema?: Record<string, any>
-  existingTable?: Table
-}) {
-  if (!schema) {
-    return schema
-  }
-  const finalSchema: Record<string, FieldSchema> = {}
-  const schemaKeyMap: Record<string, any> = {}
-  Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
-  for (let [key, field] of Object.entries(existingTable?.schema || {})) {
-    const lcKey = key.toLowerCase()
-    const foundKey: string = schemaKeyMap[lcKey]
-    if (foundKey) {
-      finalSchema[key] = schema[foundKey]
-      finalSchema[key].type = field.type
-    }
-  }
-  return finalSchema
-}
-
-export async function transform({
-  schema,
-  csvString,
-  existingTable,
-}: CsvParseOpts) {
-  if (!schema || !csvString) {
-    throw new Error("Unable to transform CSV without schema")
-  }
-  const colParser: any = {}
-
-  // make sure the table has all the columns required for import
-  if (existingTable) {
-    schema = updateSchema({ schema, existingTable })
-  }
-
-  for (let [key, field] of Object.entries(schema || {})) {
-    // don't import data to auto columns
-    if (!field.autocolumn) {
-      colParser[key] = PARSERS[field.type] || field.type
-    }
-  }
-
-  try {
-    const data = await csv({ colParser }).fromString(csvString)
-    const schemaKeyMap: any = {}
-    Object.keys(schema || {}).forEach(
-      key => (schemaKeyMap[key.toLowerCase()] = key)
-    )
-    for (let element of data) {
-      if (!data) {
-        continue
-      }
-      for (let key of Object.keys(element)) {
-        const mappedKey = schemaKeyMap[key.toLowerCase()]
-        // isn't a column in the table, remove it
-        if (mappedKey == null) {
-          delete element[key]
-        }
-        // casing is different, fix it in row
-        else if (key !== mappedKey) {
-          element[mappedKey] = element[key]
-          delete element[key]
-        }
-      }
-    }
-    return data
-  } catch (err) {
-    console.error(`Error transforming CSV to JSON for data import`, err)
-    throw err
-  }
-}

package/src/utilities/tests/__snapshots__/csvParser.spec.js.snap

@@ -1,15 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = `
-Array [
-  Object {
-    "Age": 4324,
-  },
-  Object {
-    "Age": 34,
-  },
-  Object {
-    "Age": 23423,
-  },
-]
-`;

package/src/utilities/tests/csvParser.spec.js

@@ -1,112 +0,0 @@
-const { readFileSync } = require("../fileSystem")
-const csvParser = require("../csvParser")
-
-const CSV_PATH = __dirname + "/test.csv"
-
-const SCHEMAS = {
-  VALID: {
-    Age: {
-      type: "number",
-    },
-  },
-  INVALID: {
-    Address: {
-      type: "number",
-    },
-    Age: {
-      type: "number",
-    },
-  },
-  IGNORE: {
-    Address: {
-      type: "omit",
-    },
-    Age: {
-      type: "omit",
-    },
-    Name: {
-      type: "string",
-    },
-  },
-  BROKEN: {
-    Address: {
-      type: "datetime",
-    },
-  },
-}
-
-describe("CSV Parser", () => {
-  const csvString = readFileSync(CSV_PATH, "utf8")
-
-  describe("parsing", () => {
-    it("returns status and types for a valid CSV transformation", async () => {
-      expect(await csvParser.parse(csvString, SCHEMAS.VALID)).toEqual({
-        Address: {
-          success: true,
-          type: "string",
-        },
-        Age: {
-          success: true,
-          type: "number",
-        },
-        Name: {
-          success: true,
-          type: "string",
-        },
-      })
-    })
-
-    it("returns status and types for an invalid CSV transformation", async () => {
-      expect(await csvParser.parse(csvString, SCHEMAS.INVALID)).toEqual({
-        Address: {
-          success: false,
-          type: "number",
-        },
-        Age: {
-          success: true,
-          type: "number",
-        },
-        Name: {
-          success: true,
-          type: "string",
-        },
-      })
-    })
-  })
-
-  describe("transformation", () => {
-    it("transforms a CSV file into JSON", async () => {
-      expect(
-        await csvParser.transform({
-          schema: SCHEMAS.VALID,
-          csvString,
-        })
-      ).toMatchSnapshot()
-    })
-
-    it("transforms a CSV file into JSON ignoring certain fields", async () => {
-      expect(
-        await csvParser.transform({
-          schema: SCHEMAS.IGNORE,
-          csvString,
-        })
-      ).toEqual([
-        {
-          Name: "Bertå",
-        },
-        {
-          Name: "Ernie",
-        },
-        {
-          Name: "Big Bird",
-        },
-      ])
-    })
-
-    it("throws an error on invalid schema", async () => {
-      await expect(
-        csvParser.transform({ schema: SCHEMAS.BROKEN, csvString })
-      ).rejects.toThrow()
-    })
-  })
-})