@budibase/server 2.5.10-alpha.0 → 2.5.10-alpha.1

package/dist/utilities/csv.js

@@ -0,0 +1,33 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.jsonFromCsvString = void 0;
+const csvtojson_1 = __importDefault(require("csvtojson"));
+function jsonFromCsvString(csvString) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const castedWithEmptyValues = yield (0, csvtojson_1.default)({ ignoreEmpty: true }).fromString(csvString);
+        // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion.
+        // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values
+        const result = yield (0, csvtojson_1.default)({ ignoreEmpty: false }).fromString(csvString);
+        result.forEach((r, i) => {
+            for (const [key] of Object.entries(r).filter(([key, value]) => value === "")) {
+                if (castedWithEmptyValues[i][key] === undefined) {
+                    r[key] = null;
+                }
+            }
+        });
+        return result;
+    });
+}
+exports.jsonFromCsvString = jsonFromCsvString;

package/dist/utilities/schema.js

@@ -39,13 +39,17 @@ function validate(rows, schema) {
     };
     rows.forEach(row => {
         Object.entries(row).forEach(([columnName, columnData]) => {
-            var _a, _b;
+            var _a, _b, _c;
             const columnType = (_a = schema[columnName]) === null || _a === void 0 ? void 0 : _a.type;
             const isAutoColumn = (_b = schema[columnName]) === null || _b === void 0 ? void 0 : _b.autocolumn;
             // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
             if (typeof columnType !== "string") {
                 results.invalidColumns.push(columnName);
             }
+            else if (columnData == null &&
+                !((_c = schema[columnName].constraints) === null || _c === void 0 ? void 0 : _c.presence)) {
+                results.schemaValidation[columnName] = true;
+            }
             else if (
             // If there's no data for this field don't bother with further checks
             // If the field is already marked as invalid there's no need for further checks

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.5.10-alpha.0",
+  "version": "2.5.10-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -45,12 +45,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.5.10-alpha.0",
-    "@budibase/client": "2.5.10-alpha.0",
-    "@budibase/pro": "2.5.9",
-    "@budibase/shared-core": "2.5.10-alpha.0",
-    "@budibase/string-templates": "2.5.10-alpha.0",
-    "@budibase/types": "2.5.10-alpha.0",
+    "@budibase/backend-core": "2.5.10-alpha.1",
+    "@budibase/client": "2.5.10-alpha.1",
+    "@budibase/pro": "2.5.10-alpha.0",
+    "@budibase/shared-core": "2.5.10-alpha.1",
+    "@budibase/string-templates": "2.5.10-alpha.1",
+    "@budibase/types": "2.5.10-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -176,5 +176,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "1d50b1f07c1a77b21d9757534c353631ce3314f8"
+  "gitHead": "9fcfb0a6cd4e4003f0e420b4b180b3b069e93191"
 }

package/src/api/controllers/row/internal.ts

@@ -37,7 +37,7 @@ import {
   Table,
 } from "@budibase/types"
 
-const { cleanExportRows } = require("./utils")
+import { cleanExportRows } from "./utils"
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -391,6 +391,9 @@ export async function exportRows(ctx: UserCtx) {
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
+  if (typeof format !== "string") {
+    ctx.throw(400, "Format parameter is not valid")
+  }
   const { columns, query } = ctx.request.body
 
   let result

package/src/api/controllers/row/utils.ts

@@ -137,8 +137,8 @@ export function cleanExportRows(
     delete schema[column]
   })
 
-  // Intended to avoid 'undefined' in export
   if (format === Format.CSV) {
+    // Intended to append empty values in export
     const schemaKeys = Object.keys(schema)
     for (let key of schemaKeys) {
       if (columns?.length && columns.indexOf(key) > 0) {
@@ -146,7 +146,7 @@ export function cleanExportRows(
       }
       for (let row of cleanRows) {
         if (row[key] == null) {
-          row[key] = ""
+          row[key] = undefined
         }
       }
     }

package/src/api/controllers/table/index.ts

@@ -10,7 +10,7 @@ import { getDatasourceParams } from "../../../db/utils"
 import { context, events } from "@budibase/backend-core"
 import { Table, UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
-import csv from "csvtojson"
+import { jsonFromCsvString } from "../../../utilities/csv"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -104,7 +104,7 @@ export async function bulkImport(ctx: UserCtx) {
 export async function csvToJson(ctx: UserCtx) {
   const { csvString } = ctx.request.body
 
-  const result = await csv().fromString(csvString)
+  const result = await jsonFromCsvString(csvString)
 
   ctx.status = 200
   ctx.body = result

package/src/api/controllers/view/exporters.ts

@@ -10,7 +10,9 @@ export function csv(headers: string[], rows: Row[]) {
         val =
           typeof val === "object" && !(val instanceof Date)
             ? `"${JSON.stringify(val).replace(/"/g, "'")}"`
-            : `"${val}"`
+            : val !== undefined
+            ? `"${val}"`
+            : ""
         return val.trim()
       })
       .join(",")}`

package/src/utilities/csv.ts

@@ -0,0 +1,22 @@
+import csv from "csvtojson"
+
+export async function jsonFromCsvString(csvString: string) {
+  const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString(
+    csvString
+  )
+
+  // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion.
+  // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values
+  const result = await csv({ ignoreEmpty: false }).fromString(csvString)
+  result.forEach((r, i) => {
+    for (const [key] of Object.entries(r).filter(
+      ([key, value]) => value === ""
+    )) {
+      if (castedWithEmptyValues[i][key] === undefined) {
+        r[key] = null
+      }
+    }
+  })
+
+  return result
+}

package/src/utilities/schema.ts

@@ -4,6 +4,9 @@ interface SchemaColumn {
   readonly name: string
   readonly type: FieldTypes
   readonly autocolumn?: boolean
+  readonly constraints?: {
+    presence: boolean
+  }
 }
 
 interface Schema {
@@ -76,6 +79,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
+      } else if (
+        columnData == null &&
+        !schema[columnName].constraints?.presence
+      ) {
+        results.schemaValidation[columnName] = true
       } else if (
         // If there's no data for this field don't bother with further checks
         // If the field is already marked as invalid there's no need for further checks

package/src/utilities/tests/csv.spec.ts

@@ -0,0 +1,33 @@
+import { jsonFromCsvString } from "../csv"
+
+describe("csv", () => {
+  describe("jsonFromCsvString", () => {
+    test("multiple lines csv can be casted", async () => {
+      const csvString = '"id","title"\n"1","aaa"\n"2","bbb"'
+
+      const result = await jsonFromCsvString(csvString)
+
+      expect(result).toEqual([
+        { id: "1", title: "aaa" },
+        { id: "2", title: "bbb" },
+      ])
+      result.forEach(r => expect(Object.keys(r)).toEqual(["id", "title"]))
+    })
+
+    test("empty values are casted as undefined", async () => {
+      const csvString =
+        '"id","optional","title"\n1,,"aaa"\n2,"value","bbb"\n3,,"ccc"'
+
+      const result = await jsonFromCsvString(csvString)
+
+      expect(result).toEqual([
+        { id: "1", optional: null, title: "aaa" },
+        { id: "2", optional: "value", title: "bbb" },
+        { id: "3", optional: null, title: "ccc" },
+      ])
+      result.forEach(r =>
+        expect(Object.keys(r)).toEqual(["id", "optional", "title"])
+      )
+    })
+  })
+})