@budibase/server 2.2.12-alpha.20 → 2.2.12-alpha.21

package/dist/api/controllers/row/internal.js

@@ -61,7 +61,7 @@ const utils_2 = require("../view/utils");
 const fp_1 = require("lodash/fp");
 const backend_core_1 = require("@budibase/backend-core");
 const staticFormula_1 = require("./staticFormula");
-const exporters = __importStar(require("../view/exporters"));
+const exporters_1 = require("../view/exporters");
 const fileSystem_1 = require("../../../utilities/fileSystem");
 const CALCULATION_TYPES = {
     SUM: "sum",
@@ -418,13 +418,17 @@ function exportRows(ctx) {
         else {
             rows = result;
         }
-        let headers = Object.keys(rows[0]);
-        // @ts-ignore
-        const exporter = exporters[format];
-        const filename = `export.${format}`;
-        // send down the file
-        ctx.attachment(filename);
-        return (0, fileSystem_1.apiFileReturn)(exporter(headers, rows));
+        if (format === exporters_1.Format.CSV) {
+            ctx.attachment("export.csv");
+            return (0, fileSystem_1.apiFileReturn)((0, exporters_1.csv)(Object.keys(rows[0]), rows));
+        }
+        else if (format === exporters_1.Format.JSON) {
+            ctx.attachment("export.json");
+            return (0, fileSystem_1.apiFileReturn)((0, exporters_1.json)(rows));
+        }
+        else {
+            throw "Format not recognised";
+        }
     });
 }
 exports.exportRows = exportRows;

package/dist/api/controllers/table/external.js

@@ -1,27 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -40,9 +17,9 @@ const utils_1 = require("../../../integrations/utils");
 const utils_2 = require("./utils");
 const constants_1 = require("../../../constants");
 const query_1 = require("../../../integrations/base/query");
-const csvParser = __importStar(require("../../../utilities/csvParser"));
 const external_1 = require("../row/external");
 const backend_core_1 = require("@budibase/backend-core");
+const schema_1 = require("../../../utilities/schema");
 const types_1 = require("@budibase/types");
 const sdk_1 = __importDefault(require("../../../sdk"));
 const { cloneDeep } = require("lodash/fp");
@@ -176,7 +153,7 @@ function save(ctx) {
         const table = ctx.request.body;
         const renamed = table === null || table === void 0 ? void 0 : table._rename;
         // can't do this right now
-        delete table.dataImport;
+        delete table.rows;
         const datasourceId = getDatasourceId(ctx.request.body);
         // table doesn't exist already, note that it is created
         if (!table._id) {
@@ -279,15 +256,16 @@ exports.destroy = destroy;
 function bulkImport(ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const table = yield sdk_1.default.tables.getTable(ctx.params.tableId);
-        const { dataImport } = ctx.request.body;
-        if (!dataImport || !dataImport.schema || !dataImport.csvString) {
+        const { rows } = ctx.request.body;
+        const schema = table.schema;
+        if (!rows || !(0, schema_1.isRows)(rows) || !(0, schema_1.isSchema)(schema)) {
             ctx.throw(400, "Provided data import information is invalid.");
         }
-        const rows = yield csvParser.transform(Object.assign(Object.assign({}, dataImport), { existingTable: table }));
+        const parsedRows = yield (0, schema_1.parse)(rows, schema);
         yield (0, external_1.handleRequest)(types_1.Operation.BULK_CREATE, table._id, {
-            rows,
+            rows: parsedRows,
         });
-        yield backend_core_1.events.rows.imported(table, "csv", rows.length);
+        yield backend_core_1.events.rows.imported(table, parsedRows.length);
         return table;
     });
 }

package/dist/api/controllers/table/index.js

@@ -35,14 +35,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateCSVSchema = exports.bulkImport = exports.destroy = exports.save = exports.find = exports.fetch = void 0;
+exports.validateExistingTableImport = exports.validateNewTableImport = exports.csvToJson = exports.bulkImport = exports.destroy = exports.save = exports.find = exports.fetch = void 0;
 const internal = __importStar(require("./internal"));
 const external = __importStar(require("./external"));
-const csvParser = __importStar(require("../../../utilities/csvParser"));
+const schema_1 = require("../../../utilities/schema");
 const utils_1 = require("../../../integrations/utils");
 const utils_2 = require("../../../db/utils");
 const backend_core_1 = require("@budibase/backend-core");
 const sdk_1 = __importDefault(require("../../../sdk"));
+const csvtojson_1 = __importDefault(require("csvtojson"));
 function pickApi({ tableId, table }) {
     if (table && !tableId) {
         tableId = table._id;
@@ -87,7 +88,7 @@ function save(ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const appId = ctx.appId;
         const table = ctx.request.body;
-        const importFormat = table.dataImport && table.dataImport.csvString ? "csv" : undefined;
+        const isImport = table.rows;
         const savedTable = yield pickApi({ table }).save(ctx);
         if (!table._id) {
             yield backend_core_1.events.table.created(savedTable);
@@ -95,8 +96,8 @@ function save(ctx) {
         else {
             yield backend_core_1.events.table.updated(savedTable);
         }
-        if (importFormat) {
-            yield backend_core_1.events.table.imported(savedTable, importFormat);
+        if (isImport) {
+            yield backend_core_1.events.table.imported(savedTable);
         }
         ctx.status = 200;
         ctx.message = `Table ${table.name} saved successfully.`;
@@ -132,19 +133,47 @@ function bulkImport(ctx) {
     });
 }
 exports.bulkImport = bulkImport;
-function validateCSVSchema(ctx) {
+function csvToJson(ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        // tableId being specified means its an import to an existing table
-        const { csvString, schema = {}, tableId } = ctx.request.body;
-        let existingTable;
+        const { csvString } = ctx.request.body;
+        const result = yield (0, csvtojson_1.default)().fromString(csvString);
+        ctx.status = 200;
+        ctx.body = result;
+    });
+}
+exports.csvToJson = csvToJson;
+function validateNewTableImport(ctx) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { rows, schema } = ctx.request.body;
+        if ((0, schema_1.isRows)(rows) && (0, schema_1.isSchema)(schema)) {
+            ctx.status = 200;
+            ctx.body = (0, schema_1.validate)(rows, schema);
+        }
+        else {
+            ctx.status = 422;
+        }
+    });
+}
+exports.validateNewTableImport = validateNewTableImport;
+function validateExistingTableImport(ctx) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { rows, tableId } = ctx.request.body;
+        let schema = null;
         if (tableId) {
-            existingTable = yield sdk_1.default.tables.getTable(tableId);
+            const table = yield sdk_1.default.tables.getTable(tableId);
+            schema = table.schema;
         }
-        let result = yield csvParser.parse(csvString, schema);
-        if (existingTable) {
-            result = csvParser.updateSchema({ schema: result, existingTable });
+        else {
+            ctx.status = 422;
+            return;
+        }
+        if (tableId && (0, schema_1.isRows)(rows) && (0, schema_1.isSchema)(schema)) {
+            ctx.status = 200;
+            ctx.body = (0, schema_1.validate)(rows, schema);
+        }
+        else {
+            ctx.status = 422;
         }
-        ctx.body = { schema: result };
     });
 }
-exports.validateCSVSchema = validateCSVSchema;
+exports.validateExistingTableImport = validateExistingTableImport;

package/dist/api/controllers/table/internal.js

@@ -57,7 +57,7 @@ function checkAutoColumns(table, oldTable) {
 function save(ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = backend_core_1.context.getAppDB();
-        const _a = ctx.request.body, { dataImport } = _a, rest = __rest(_a, ["dataImport"]);
+        const _a = ctx.request.body, { rows } = _a, rest = __rest(_a, ["rows"]);
         let tableToSave = Object.assign({ type: "table", _id: (0, utils_1.generateTableID)(), views: {} }, rest);
         // if the table obj had an _id then it will have been retrieved
         let oldTable;
@@ -75,7 +75,7 @@ function save(ctx) {
         const tableSaveFunctions = new utils_2.TableSaveFunctions({
             user: ctx.user,
             oldTable,
-            dataImport,
+            importRows: rows,
         });
         tableToSave = yield tableSaveFunctions.before(tableToSave);
         // make sure that types don't change of a column, have to remove
@@ -186,8 +186,8 @@ exports.destroy = destroy;
 function bulkImport(ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const table = yield sdk_1.default.tables.getTable(ctx.params.tableId);
-        const { dataImport } = ctx.request.body;
-        yield (0, utils_2.handleDataImport)(ctx.user, table, dataImport);
+        const { rows } = ctx.request.body;
+        yield (0, utils_2.handleDataImport)(ctx.user, table, rows);
         return table;
     });
 }

package/dist/api/controllers/table/utils.js

@@ -13,7 +13,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TableSaveFunctions = exports.hasTypeChanged = exports.areSwitchableTypes = exports.foreignKeyStructure = exports.generateJunctionTableName = exports.generateForeignKey = exports.checkForViewUpdates = exports.checkStaticTables = exports.handleSearchIndexes = exports.handleDataImport = exports.importToRows = exports.makeSureTableUpToDate = exports.checkForColumnUpdates = exports.clearColumns = void 0;
-const csvParser_1 = require("../../../utilities/csvParser");
+const schema_1 = require("../../../utilities/schema");
 const utils_1 = require("../../../db/utils");
 const lodash_1 = require("lodash");
 const constants_1 = require("../../../constants");
@@ -124,19 +124,19 @@ function importToRows(data, table, user = {}) {
     return finalData;
 }
 exports.importToRows = importToRows;
-function handleDataImport(user, table, dataImport) {
+function handleDataImport(user, table, rows) {
     return __awaiter(this, void 0, void 0, function* () {
-        if (!dataImport || !dataImport.csvString) {
+        const schema = table.schema;
+        if (!rows || !(0, schema_1.isRows)(rows) || !(0, schema_1.isSchema)(schema)) {
             return table;
         }
         const db = backend_core_1.context.getAppDB();
-        // Populate the table with rows imported from CSV in a bulk update
-        const data = yield (0, csvParser_1.transform)(Object.assign(Object.assign({}, dataImport), { existingTable: table }));
+        const data = (0, schema_1.parse)(rows, schema);
         let finalData = importToRows(data, table, user);
         yield pro_1.quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
             tableId: table._id,
         });
-        yield backend_core_1.events.rows.imported(table, "csv", finalData.length);
+        yield backend_core_1.events.rows.imported(table, finalData.length);
         return table;
     });
 }
@@ -195,11 +195,11 @@ function checkStaticTables(table) {
 }
 exports.checkStaticTables = checkStaticTables;
 class TableSaveFunctions {
-    constructor({ user, oldTable, dataImport }) {
+    constructor({ user, oldTable, importRows }) {
         this.db = backend_core_1.context.getAppDB();
         this.user = user;
         this.oldTable = oldTable;
-        this.dataImport = dataImport;
+        this.importRows = importRows;
         // any rows that need updated
         this.rows = [];
     }
@@ -225,7 +225,7 @@ class TableSaveFunctions {
     after(table) {
         return __awaiter(this, void 0, void 0, function* () {
             table = yield handleSearchIndexes(table);
-            table = yield handleDataImport(this.user, table, this.dataImport);
+            table = yield handleDataImport(this.user, table, this.importRows);
             return table;
         });
     }

package/dist/api/controllers/view/exporters.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExportFormats = exports.json = exports.csv = void 0;
+exports.isFormat = exports.Format = exports.jsonWithSchema = exports.json = exports.csv = void 0;
 function csv(headers, rows) {
     let csv = headers.map(key => `"${key}"`).join(",");
     for (let row of rows) {
@@ -18,11 +18,27 @@ function csv(headers, rows) {
     return csv;
 }
 exports.csv = csv;
-function json(headers, rows) {
+function json(rows) {
     return JSON.stringify(rows, undefined, 2);
 }
 exports.json = json;
-exports.ExportFormats = {
-    CSV: "csv",
-    JSON: "json",
-};
+function jsonWithSchema(schema, rows) {
+    const newSchema = {};
+    Object.values(schema).forEach(column => {
+        if (!column.autocolumn) {
+            newSchema[column.name] = column;
+        }
+    });
+    return JSON.stringify({ schema: newSchema, rows }, undefined, 2);
+}
+exports.jsonWithSchema = jsonWithSchema;
+var Format;
+(function (Format) {
+    Format["CSV"] = "csv";
+    Format["JSON"] = "json";
+    Format["JSON_WITH_SCHEMA"] = "jsonWithSchema";
+})(Format = exports.Format || (exports.Format = {}));
+function isFormat(format) {
+    return Object.values(Format).includes(format);
+}
+exports.isFormat = isFormat;

package/dist/api/controllers/view/index.js

@@ -1,27 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -49,7 +26,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.exportView = exports.destroy = exports.filterEvents = exports.calculationEvents = exports.save = exports.fetch = void 0;
 const viewBuilder_1 = __importDefault(require("./viewBuilder"));
 const fileSystem_1 = require("../../../utilities/fileSystem");
-const exporters = __importStar(require("./exporters"));
+const exporters_1 = require("./exporters");
 const utils_1 = require("./utils");
 const row_1 = require("../row");
 const constants_1 = require("../../../constants");
@@ -160,8 +137,8 @@ function exportView(ctx) {
         const viewName = decodeURIComponent(ctx.query.view);
         const view = yield (0, utils_1.getView)(viewName);
         const format = ctx.query.format;
-        if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
-            ctx.throw(400, "Format must be specified, either csv or json");
+        if (!(0, exporters_1.isFormat)(format)) {
+            ctx.throw(400, "Format must be specified, either csv, json or jsonWithSchema");
         }
         if (view) {
             ctx.params.viewName = viewName;
@@ -199,7 +176,7 @@ function exportView(ctx) {
             delete schema[column];
         });
         // make sure no "undefined" entries appear in the CSV
-        if (format === exporters.ExportFormats.CSV) {
+        if (format === exporters_1.Format.CSV) {
             const schemaKeys = Object.keys(schema);
             for (let key of schemaKeys) {
                 for (let row of rows) {
@@ -209,13 +186,21 @@ function exportView(ctx) {
                 }
             }
         }
-        // Export part
-        let headers = Object.keys(schema);
-        const exporter = format === "csv" ? exporters.csv : exporters.json;
-        const filename = `${viewName}.${format}`;
-        // send down the file
-        ctx.attachment(filename);
-        ctx.body = (0, fileSystem_1.apiFileReturn)(exporter(headers, rows));
+        if (format === exporters_1.Format.CSV) {
+            ctx.attachment(`${viewName}.csv`);
+            ctx.body = (0, fileSystem_1.apiFileReturn)((0, exporters_1.csv)(Object.keys(schema), rows));
+        }
+        else if (format === exporters_1.Format.JSON) {
+            ctx.attachment(`${viewName}.json`);
+            ctx.body = (0, fileSystem_1.apiFileReturn)((0, exporters_1.json)(rows));
+        }
+        else if (format === exporters_1.Format.JSON_WITH_SCHEMA) {
+            ctx.attachment(`${viewName}.json`);
+            ctx.body = (0, fileSystem_1.apiFileReturn)((0, exporters_1.jsonWithSchema)(schema, rows));
+        }
+        else {
+            throw "Format not recognised";
+        }
         if (viewName.startsWith(utils_2.DocumentType.TABLE)) {
             yield backend_core_1.events.table.exported(table, format);
         }

package/dist/api/routes/table.js

@@ -88,10 +88,7 @@ router
      * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
      * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
      * the rows in the table. This functionality is only available for internal tables.
-     * @apiParam (Body) {object} [dataImport] When creating an internal table it can be built from a CSV, by using the
-     * CSV validation endpoint. Send the CSV data to the validation endpoint, then put the results of that call
-     * into this property, along with the CSV and a table/rows will be built from it. This is not supported when updating
-     * or for external tables.
+     * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided.
      *
      * @apiParamExample {json} Example:
      * {
@@ -120,15 +117,7 @@ router
      *     "old": "columnName",
      *     "updated": "newColumnName",
      *   },
-     *   "dataImport": {
-     *     "csvString": "column\nvalue",
-     *     "primaryDisplay": "column",
-     *     "schema": {
-     *       "column": {
-     *         "type": "string"
-     *       }
-     *     }
-     *   }
+     *   "rows": []
      * }
      *
      * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
@@ -137,27 +126,9 @@ router
     .post("/api/tables", 
 // allows control over updating a table
 (0, resourceId_1.bodyResource)("_id"), (0, authorized_1.default)(BUILDER), (0, validators_1.tableValidator)(), tableController.save)
-    /**
-     * @api {post} /api/tables/csv/validate Validate a CSV for a table
-     * @apiName Validate a CSV for a table
-     * @apiGroup tables
-     * @apiPermission builder
-     * @apiDescription When creating a new table, or importing a CSV to an existing table the CSV must be validated and
-     * converted into a Budibase schema; this endpoint does this.
-     *
-     * @apiParam (Body) {string} csvString The CSV which is to be validated as a string.
-     * @apiParam (Body) {object} [schema] When a CSV has been validated it is possible to re-validate after changing the
-     * type of a field, by default everything will be strings as there is no way to infer types. The returned schema can
-     * be updated and then returned to the endpoint to re-validate and check if the type will work for the CSV, e.g.
-     * using a number instead of strings.
-     * @apiParam (Body) {string} [tableId] If importing data to an existing table this will pull the current table and
-     * remove any fields from the CSV schema which do not exist on the table/don't match the type of the table. When
-     * importing a CSV to an existing table only fields that are present on the table can be imported.
-     *
-     * @apiSuccess {object} schema The response body will contain a "schema" object that represents the schema found for
-     * the CSV - this will be in the same format used for table schema.s
-     */
-    .post("/api/tables/csv/validate", (0, authorized_1.default)(BUILDER), tableController.validateCSVSchema)
+    .post("/api/convert/csvToJson", (0, authorized_1.default)(BUILDER), tableController.csvToJson)
+    .post("/api/tables/validateNewTableImport", (0, authorized_1.default)(BUILDER), tableController.validateNewTableImport)
+    .post("/api/tables/validateExistingTableImport", (0, authorized_1.default)(BUILDER), tableController.validateExistingTableImport)
     /**
      * @api {post} /api/tables/:tableId/:revId Delete a table
      * @apiName Delete a table
@@ -184,9 +155,7 @@ router
      *
      * @apiParam {string} tableId The ID of the table which the data should be imported to.
      *
-     * @apiParam (Body) {object} dataImport This is the same as the structure used when creating an internal table with
-     * a CSV, it will have the "schema" returned from the CSV validation endpoint and the "csvString" which is to be
-     * turned into rows.
+     * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored.
      *
      * @apiSuccess {string} message A message stating that the data was imported successfully.
      */

package/dist/api/routes/utils/validators.js

@@ -22,7 +22,7 @@ function tableValidator() {
         schema: joi_1.default.object().required(),
         name: joi_1.default.string().required(),
         views: joi_1.default.object(),
-        dataImport: joi_1.default.object(),
+        rows: joi_1.default.array(),
     }).unknown(true));
 }
 exports.tableValidator = tableValidator;

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "2.2.12-alpha.19",
+    "version": "2.2.12-alpha.20",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -43,11 +43,11 @@
     "license": "GPL-3.0",
     "dependencies": {
         "@apidevtools/swagger-parser": "10.0.3",
-        "@budibase/backend-core": "2.2.12-alpha.19",
-        "@budibase/client": "2.2.12-alpha.19",
-        "@budibase/pro": "2.2.12-alpha.19",
-        "@budibase/string-templates": "2.2.12-alpha.19",
-        "@budibase/types": "2.2.12-alpha.19",
+        "@budibase/backend-core": "2.2.12-alpha.20",
+        "@budibase/client": "2.2.12-alpha.20",
+        "@budibase/pro": "2.2.12-alpha.20",
+        "@budibase/string-templates": "2.2.12-alpha.20",
+        "@budibase/types": "2.2.12-alpha.20",
         "@bull-board/api": "3.7.0",
         "@bull-board/koa": "3.9.4",
         "@elastic/elasticsearch": "7.10.0",