@budibase/server 2.4.26 → 2.4.27-alpha.0

package/dist/integrations/googlesheets.js

@@ -15,11 +15,21 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const types_1 = require("@budibase/types");
 const google_auth_library_1 = require("google-auth-library");
 const utils_1 = require("./utils");
-const constants_1 = require("../constants");
 const google_spreadsheet_1 = require("google-spreadsheet");
 const node_fetch_1 = __importDefault(require("node-fetch"));
 const backend_core_1 = require("@budibase/backend-core");
 const shared_core_1 = require("@budibase/shared-core");
+const constants_1 = require("../constants");
+const ALLOWED_TYPES = [
+    types_1.FieldType.STRING,
+    types_1.FieldType.FORMULA,
+    types_1.FieldType.NUMBER,
+    types_1.FieldType.LONGFORM,
+    types_1.FieldType.DATETIME,
+    types_1.FieldType.OPTIONS,
+    types_1.FieldType.BOOLEAN,
+    types_1.FieldType.BARCODEQR,
+];
 const SCHEMA = {
     plus: true,
     auth: {
@@ -135,6 +145,7 @@ class GoogleSheetsIntegration {
         });
     }
     connect() {
+        var _a;
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 // Initialise oAuth client
@@ -159,12 +170,36 @@ class GoogleSheetsIntegration {
                 yield this.client.loadInfo();
             }
             catch (err) {
+                // this happens for xlsx imports
+                if ((_a = err.message) === null || _a === void 0 ? void 0 : _a.includes("operation is not supported")) {
+                    err.message =
+                        "This operation is not supported - XLSX sheets must be converted.";
+                }
                 console.error("Error connecting to google sheets", err);
                 throw err;
             }
         });
     }
-    buildSchema(datasourceId) {
+    getTableSchema(title, headerValues, id) {
+        // base table
+        const table = {
+            name: title,
+            primary: [constants_1.GOOGLE_SHEETS_PRIMARY_KEY],
+            schema: {},
+        };
+        if (id) {
+            table._id = id;
+        }
+        // build schema from headers
+        for (let header of headerValues) {
+            table.schema[header] = {
+                name: header,
+                type: types_1.FieldType.STRING,
+            };
+        }
+        return table;
+    }
+    buildSchema(datasourceId, entities) {
         return __awaiter(this, void 0, void 0, function* () {
             yield this.connect();
             const sheets = this.client.sheetsByIndex;
@@ -172,56 +207,47 @@ class GoogleSheetsIntegration {
             for (let sheet of sheets) {
                 // must fetch rows to determine schema
                 yield sheet.getRows();
-                // build schema
-                const schema = {};
-                // build schema from headers
-                for (let header of sheet.headerValues) {
-                    schema[header] = {
-                        name: header,
-                        type: constants_1.FieldTypes.STRING,
-                    };
-                }
-                // create tables
-                tables[sheet.title] = {
-                    _id: (0, utils_1.buildExternalTableId)(datasourceId, sheet.title),
-                    name: sheet.title,
-                    primary: ["rowNumber"],
-                    schema,
-                };
+                const id = (0, utils_1.buildExternalTableId)(datasourceId, sheet.title);
+                tables[sheet.title] = this.getTableSchema(sheet.title, sheet.headerValues, id);
             }
-            this.tables = tables;
+            const final = (0, utils_1.finaliseExternalTables)(tables, entities);
+            this.tables = final.tables;
+            this.schemaErrors = final.errors;
         });
     }
     query(json) {
+        var _a, _b, _c, _d, _e, _f, _g, _h;
         return __awaiter(this, void 0, void 0, function* () {
             const sheet = json.endpoint.entityId;
-            const handlers = {
-                [constants_1.DataSourceOperation.CREATE]: () => this.create({ sheet, row: json.body }),
-                [constants_1.DataSourceOperation.READ]: () => this.read(Object.assign(Object.assign({}, json), { sheet })),
-                [constants_1.DataSourceOperation.UPDATE]: () => {
-                    var _a, _b, _c;
+            switch (json.endpoint.operation) {
+                case types_1.Operation.CREATE:
+                    return this.create({ sheet, row: json.body });
+                case types_1.Operation.BULK_CREATE:
+                    return this.createBulk({ sheet, rows: json.body });
+                case types_1.Operation.READ:
+                    return this.read(Object.assign(Object.assign({}, json), { sheet }));
+                case types_1.Operation.UPDATE:
                     return this.update({
                         // exclude the header row and zero index
                         rowIndex: ((_c = (_b = (_a = json.extra) === null || _a === void 0 ? void 0 : _a.idFilter) === null || _b === void 0 ? void 0 : _b.equal) === null || _c === void 0 ? void 0 : _c.rowNumber) - 2,
                         sheet,
                         row: json.body,
                     });
-                },
-                [constants_1.DataSourceOperation.DELETE]: () => {
-                    var _a, _b, _c;
+                case types_1.Operation.DELETE:
                     return this.delete({
                         // exclude the header row and zero index
-                        rowIndex: ((_c = (_b = (_a = json.extra) === null || _a === void 0 ? void 0 : _a.idFilter) === null || _b === void 0 ? void 0 : _b.equal) === null || _c === void 0 ? void 0 : _c.rowNumber) - 2,
+                        rowIndex: ((_f = (_e = (_d = json.extra) === null || _d === void 0 ? void 0 : _d.idFilter) === null || _e === void 0 ? void 0 : _e.equal) === null || _f === void 0 ? void 0 : _f.rowNumber) - 2,
                         sheet,
                     });
-                },
-                [constants_1.DataSourceOperation.CREATE_TABLE]: () => { var _a; return this.createTable((_a = json === null || json === void 0 ? void 0 : json.table) === null || _a === void 0 ? void 0 : _a.name); },
-                [constants_1.DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
-                [constants_1.DataSourceOperation.DELETE_TABLE]: () => { var _a; return this.deleteTable((_a = json === null || json === void 0 ? void 0 : json.table) === null || _a === void 0 ? void 0 : _a.name); },
-            };
-            // @ts-ignore
-            const internalQueryMethod = handlers[json.endpoint.operation];
-            return yield internalQueryMethod();
+                case types_1.Operation.CREATE_TABLE:
+                    return this.createTable((_g = json === null || json === void 0 ? void 0 : json.table) === null || _g === void 0 ? void 0 : _g.name);
+                case types_1.Operation.UPDATE_TABLE:
+                    return this.updateTable(json.table);
+                case types_1.Operation.DELETE_TABLE:
+                    return this.deleteTable((_h = json === null || json === void 0 ? void 0 : json.table) === null || _h === void 0 ? void 0 : _h.name);
+                default:
+                    throw new Error(`GSheets integration does not support "${json.endpoint.operation}".`);
+            }
         });
     }
     buildRowObject(headers, values, rowNumber) {
@@ -234,9 +260,12 @@ class GoogleSheetsIntegration {
     }
     createTable(name) {
         return __awaiter(this, void 0, void 0, function* () {
+            if (!name) {
+                throw new Error("Must provide name for new sheet.");
+            }
             try {
                 yield this.connect();
-                return yield this.client.addSheet({ title: name, headerValues: ["test"] });
+                return yield this.client.addSheet({ title: name, headerValues: [name] });
             }
             catch (err) {
                 console.error("Error creating new table in google sheets", err);
@@ -246,34 +275,53 @@ class GoogleSheetsIntegration {
     }
     updateTable(table) {
         return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.connect();
-                const sheet = this.client.sheetsByTitle[table.name];
-                yield sheet.loadHeaderRow();
-                if (table._rename) {
-                    const headers = [];
-                    for (let header of sheet.headerValues) {
-                        if (header === table._rename.old) {
-                            headers.push(table._rename.updated);
-                        }
-                        else {
-                            headers.push(header);
-                        }
+            yield this.connect();
+            const sheet = this.client.sheetsByTitle[table.name];
+            yield sheet.loadHeaderRow();
+            if (table._rename) {
+                const headers = [];
+                for (let header of sheet.headerValues) {
+                    if (header === table._rename.old) {
+                        headers.push(table._rename.updated);
+                    }
+                    else {
+                        headers.push(header);
                     }
+                }
+                try {
                     yield sheet.setHeaderRow(headers);
                 }
-                else {
-                    const updatedHeaderValues = [...sheet.headerValues];
-                    const newField = Object.keys(table.schema).find(key => !sheet.headerValues.includes(key));
-                    if (newField) {
-                        updatedHeaderValues.push(newField);
+                catch (err) {
+                    console.error("Error updating column name in google sheets", err);
+                    throw err;
+                }
+            }
+            else {
+                const updatedHeaderValues = [...sheet.headerValues];
+                // add new column - doesn't currently exist
+                for (let [key, column] of Object.entries(table.schema)) {
+                    if (!ALLOWED_TYPES.includes(column.type)) {
+                        throw new Error(`Column type: ${column.type} not allowed for GSheets integration.`);
+                    }
+                    if (!sheet.headerValues.includes(key) &&
+                        column.type !== types_1.FieldType.FORMULA) {
+                        updatedHeaderValues.push(key);
+                    }
+                }
+                // clear out deleted columns
+                for (let key of sheet.headerValues) {
+                    if (!Object.keys(table.schema).includes(key)) {
+                        const idx = updatedHeaderValues.indexOf(key);
+                        updatedHeaderValues.splice(idx, 1);
                     }
+                }
+                try {
                     yield sheet.setHeaderRow(updatedHeaderValues);
                 }
-            }
-            catch (err) {
-                console.error("Error updating table in google sheets", err);
-                throw err;
+                catch (err) {
+                    console.error("Error updating table in google sheets", err);
+                    throw err;
+                }
             }
         });
     }
@@ -307,6 +355,24 @@ class GoogleSheetsIntegration {
             }
         });
     }
+    createBulk(query) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.connect();
+                const sheet = this.client.sheetsByTitle[query.sheet];
+                let rowsToInsert = [];
+                for (let row of query.rows) {
+                    rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row);
+                }
+                const rows = yield sheet.addRows(rowsToInsert);
+                return rows.map(row => this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber));
+            }
+            catch (err) {
+                console.error("Error bulk writing to google sheets", err);
+                throw err;
+            }
+        });
+    }
     read(query) {
         return __awaiter(this, void 0, void 0, function* () {
             try {

package/dist/integrations/utils.js

@@ -1,11 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.finaliseExternalTables = exports.isIsoDateString = exports.isSQL = exports.getSqlQuery = exports.convertSqlType = exports.breakRowIdField = exports.convertRowId = exports.isRowId = exports.generateRowIdField = exports.breakExternalTableId = exports.buildExternalTableId = exports.isExternalTable = exports.SqlClient = void 0;
+exports.finaliseExternalTables = exports.shouldCopySpecialColumn = exports.shouldCopyRelationship = exports.isIsoDateString = exports.isSQL = exports.getSqlQuery = exports.convertSqlType = exports.breakRowIdField = exports.convertRowId = exports.isRowId = exports.generateRowIdField = exports.breakExternalTableId = exports.buildExternalTableId = exports.isExternalTable = exports.SqlClient = void 0;
 const types_1 = require("@budibase/types");
 const utils_1 = require("../db/utils");
 const constants_1 = require("../constants");
 const DOUBLE_SEPARATOR = `${utils_1.SEPARATOR}${utils_1.SEPARATOR}`;
 const ROW_ID_REGEX = /^\[.*]$/g;
+const ENCODED_SPACE = encodeURIComponent(" ");
 const SQL_NUMBER_TYPE_MAP = {
     integer: constants_1.FieldTypes.NUMBER,
     int: constants_1.FieldTypes.NUMBER,
@@ -67,6 +68,10 @@ function isExternalTable(tableId) {
 }
 exports.isExternalTable = isExternalTable;
 function buildExternalTableId(datasourceId, tableName) {
+    // encode spaces
+    if (tableName.includes(" ")) {
+        tableName = encodeURIComponent(tableName);
+    }
     return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`;
 }
 exports.buildExternalTableId = buildExternalTableId;
@@ -78,6 +83,10 @@ function breakExternalTableId(tableId) {
     let datasourceId = parts.shift();
     // if they need joined
     let tableName = parts.join(DOUBLE_SEPARATOR);
+    // if contains encoded spaces, decode it
+    if (tableName.includes(ENCODED_SPACE)) {
+        tableName = decodeURIComponent(tableName);
+    }
     return { datasourceId, tableName };
 }
 exports.breakExternalTableId = breakExternalTableId;
@@ -193,6 +202,7 @@ function shouldCopyRelationship(column, tableIds) {
         column.tableId &&
         tableIds.includes(column.tableId));
 }
+exports.shouldCopyRelationship = shouldCopyRelationship;
 /**
  * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
  * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
@@ -217,6 +227,7 @@ function shouldCopySpecialColumn(column, fetchedColumn) {
     return (specialTypes.indexOf(column.type) !== -1 ||
         (fetchedIsNumber && column.type === constants_1.FieldTypes.BOOLEAN));
 }
+exports.shouldCopySpecialColumn = shouldCopySpecialColumn;
 /**
  * Looks for columns which need to be copied over into the new table definitions, like relationships
  * and options types.
@@ -226,10 +237,14 @@ function shouldCopySpecialColumn(column, fetchedColumn) {
  * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
  */
 function copyExistingPropsOver(tableName, table, entities, tableIds) {
+    var _a, _b, _c;
     if (entities && entities[tableName]) {
-        if (entities[tableName].primaryDisplay) {
+        if ((_a = entities[tableName]) === null || _a === void 0 ? void 0 : _a.primaryDisplay) {
             table.primaryDisplay = entities[tableName].primaryDisplay;
         }
+        if ((_b = entities[tableName]) === null || _b === void 0 ? void 0 : _b.created) {
+            table.created = (_c = entities[tableName]) === null || _c === void 0 ? void 0 : _c.created;
+        }
         const existingTableSchema = entities[tableName].schema;
         for (let key in existingTableSchema) {
             if (!existingTableSchema.hasOwnProperty(key)) {

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "2.4.25",
+    "version": "2.4.26",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -43,12 +43,12 @@
     "license": "GPL-3.0",
     "dependencies": {
         "@apidevtools/swagger-parser": "10.0.3",
-        "@budibase/backend-core": "^2.4.25",
-        "@budibase/client": "^2.4.25",
-        "@budibase/pro": "2.4.25",
-        "@budibase/shared-core": "^2.4.25",
-        "@budibase/string-templates": "^2.4.25",
-        "@budibase/types": "^2.4.25",
+        "@budibase/backend-core": "^2.4.26",
+        "@budibase/client": "^2.4.26",
+        "@budibase/pro": "2.4.26",
+        "@budibase/shared-core": "^2.4.26",
+        "@budibase/string-templates": "^2.4.26",
+        "@budibase/types": "^2.4.26",
         "@bull-board/api": "3.7.0",
         "@bull-board/koa": "3.9.4",
         "@elastic/elasticsearch": "7.10.0",