@budibase/server 1.0.49 → 1.0.50-alpha.0

package/dist/integrations/base/sql.js

@@ -69,10 +69,16 @@ class InternalBuilder {
         this.client = client;
     }
     // right now we only do filters on the specific table being queried
-    addFilters(tableName, query, filters) {
+    addFilters(query, filters, opts) {
         function iterate(structure, fn) {
             for (let [key, value] of Object.entries(structure)) {
-                fn(`${tableName}.${key}`, value);
+                const isRelationshipField = key.includes(".");
+                if (!opts.relationship && !isRelationshipField) {
+                    fn(`${opts.tableName}.${key}`, value);
+                }
+                if (opts.relationship && isRelationshipField) {
+                    fn(key, value);
+                }
             }
         }
         if (!filters) {
@@ -153,15 +159,14 @@ class InternalBuilder {
     addSorting(query, json) {
         var _a;
         let { sort, paginate } = json;
-        if (!sort) {
-            return query;
-        }
         const table = (_a = json.meta) === null || _a === void 0 ? void 0 : _a.table;
-        for (let [key, value] of Object.entries(sort)) {
-            const direction = value === datasource_1.SortDirection.ASCENDING ? "asc" : "desc";
-            query = query.orderBy(`${table === null || table === void 0 ? void 0 : table.name}.${key}`, direction);
+        if (sort) {
+            for (let [key, value] of Object.entries(sort)) {
+                const direction = value === datasource_1.SortDirection.ASCENDING ? "asc" : "desc";
+                query = query.orderBy(`${table === null || table === void 0 ? void 0 : table.name}.${key}`, direction);
+            }
         }
-        if (this.client === utils_1.SqlClients.MS_SQL && !sort && (paginate === null || paginate === void 0 ? void 0 : paginate.limit)) {
+        else if (this.client === utils_1.SqlClients.MS_SQL && (paginate === null || paginate === void 0 ? void 0 : paginate.limit)) {
             // @ts-ignore
             query = query.orderBy(`${table === null || table === void 0 ? void 0 : table.name}.${table === null || table === void 0 ? void 0 : table.primary[0]}`);
         }
@@ -171,20 +176,55 @@ class InternalBuilder {
         if (!relationships) {
             return query;
         }
+        const tableSets = {};
+        // aggregate into table sets (all the same to tables)
         for (let relationship of relationships) {
-            const from = relationship.from, to = relationship.to, toTable = relationship.tableName;
-            if (!relationship.through) {
+            const keyObj = {
+                toTable: relationship.tableName,
+                throughTable: undefined,
+            };
+            if (relationship.through) {
+                keyObj.throughTable = relationship.through;
+            }
+            const key = JSON.stringify(keyObj);
+            if (tableSets[key]) {
+                tableSets[key].push(relationship);
+            }
+            else {
+                tableSets[key] = [relationship];
+            }
+        }
+        for (let [key, relationships] of Object.entries(tableSets)) {
+            const { toTable, throughTable } = JSON.parse(key);
+            if (!throughTable) {
                 // @ts-ignore
-                query = query.leftJoin(toTable, `${fromTable}.${from}`, `${toTable}.${to}`);
+                query = query.join(toTable, function () {
+                    for (let relationship of relationships) {
+                        const from = relationship.from, to = relationship.to;
+                        // @ts-ignore
+                        this.orOn(`${fromTable}.${from}`, "=", `${toTable}.${to}`);
+                    }
+                }, "left");
             }
             else {
-                const throughTable = relationship.through;
-                const fromPrimary = relationship.fromPrimary;
-                const toPrimary = relationship.toPrimary;
                 query = query
                     // @ts-ignore
-                    .leftJoin(throughTable, `${fromTable}.${fromPrimary}`, `${throughTable}.${from}`)
-                    .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`);
+                    .join(throughTable, function () {
+                    for (let relationship of relationships) {
+                        const fromPrimary = relationship.fromPrimary;
+                        const from = relationship.from;
+                        // @ts-ignore
+                        this.orOn(`${fromTable}.${fromPrimary}`, "=", `${throughTable}.${from}`);
+                    }
+                }, "left")
+                    .join(toTable, function () {
+                    for (let relationship of relationships) {
+                        const toPrimary = relationship.toPrimary;
+                        const to = relationship.to;
+                        // @ts-ignore
+                        this.orOn(`${toTable}.${toPrimary}`, `${throughTable}.${to}`);
+                    }
+                }, "left");
             }
         }
         return query.limit(BASE_LIMIT);
@@ -248,7 +288,7 @@ class InternalBuilder {
         if (foundOffset) {
             query = query.offset(foundOffset);
         }
-        query = this.addFilters(tableName, query, filters);
+        query = this.addFilters(query, filters, { tableName });
         // add sorting to pre-query
         query = this.addSorting(query, json);
         // @ts-ignore
@@ -261,13 +301,14 @@ class InternalBuilder {
             preQuery = this.addSorting(preQuery, json);
         }
         // handle joins
-        return this.addRelationships(knex, preQuery, selectStatement, tableName, relationships);
+        query = this.addRelationships(knex, preQuery, selectStatement, tableName, relationships);
+        return this.addFilters(query, filters, { relationship: true });
     }
     update(knex, json, opts) {
         const { endpoint, body, filters } = json;
         let query = knex(endpoint.entityId);
         const parsedBody = parseBody(body);
-        query = this.addFilters(endpoint.entityId, query, filters);
+        query = this.addFilters(query, filters, { tableName: endpoint.entityId });
         // mysql can't use returning
         if (opts.disableReturning) {
             return query.update(parsedBody);
@@ -279,7 +320,7 @@ class InternalBuilder {
     delete(knex, json, opts) {
         const { endpoint, filters } = json;
         let query = knex(endpoint.entityId);
-        query = this.addFilters(endpoint.entityId, query, filters);
+        query = this.addFilters(query, filters, { tableName: endpoint.entityId });
         // mysql can't use returning
         if (opts.disableReturning) {
             return query.delete();

package/dist/integrations/googlesheets.js

@@ -0,0 +1,340 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const datasource_1 = require("../definitions/datasource");
+const google_auth_library_1 = require("google-auth-library");
+const utils_1 = require("./utils");
+const constants_1 = require("../constants");
+const google_spreadsheet_1 = require("google-spreadsheet");
+var GoogleSheetsModule;
+(function (GoogleSheetsModule) {
+    const { getGlobalDB } = require("@budibase/backend-core/tenancy");
+    const { getScopedConfig } = require("@budibase/backend-core/db");
+    const { Configs } = require("@budibase/backend-core/constants");
+    const SCHEMA = {
+        plus: true,
+        auth: {
+            type: "google",
+        },
+        relationships: false,
+        docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
+        description: "Create and collaborate on online spreadsheets in real-time and from any device. ",
+        friendlyName: "Google Sheets",
+        datasource: {
+            spreadsheetId: {
+                type: datasource_1.DatasourceFieldTypes.STRING,
+                required: true,
+            },
+        },
+        query: {
+            create: {
+                type: datasource_1.QueryTypes.FIELDS,
+                fields: {
+                    sheet: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                    row: {
+                        type: datasource_1.QueryTypes.JSON,
+                        required: true,
+                    },
+                },
+            },
+            read: {
+                type: datasource_1.QueryTypes.FIELDS,
+                fields: {
+                    sheet: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                },
+            },
+            update: {
+                type: datasource_1.QueryTypes.FIELDS,
+                fields: {
+                    sheet: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                    rowIndex: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                    row: {
+                        type: datasource_1.QueryTypes.JSON,
+                        required: true,
+                    },
+                },
+            },
+            delete: {
+                type: datasource_1.QueryTypes.FIELDS,
+                fields: {
+                    sheet: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                    rowIndex: {
+                        type: datasource_1.DatasourceFieldTypes.NUMBER,
+                        required: true,
+                    },
+                },
+            },
+        },
+    };
+    class GoogleSheetsIntegration {
+        constructor(config) {
+            this.tables = {};
+            this.schemaErrors = {};
+            this.config = config;
+            const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId);
+            this.client = new google_spreadsheet_1.GoogleSpreadsheet(spreadsheetId);
+        }
+        /**
+         * Pull the spreadsheet ID out from a valid google sheets URL
+         * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
+         * @returns spreadsheet Id of the google sheet
+         */
+        cleanSpreadsheetUrl(spreadsheetId) {
+            if (!spreadsheetId) {
+                throw new Error("You must set a spreadsheet ID in your configuration to fetch tables.");
+            }
+            const parts = spreadsheetId.split("/");
+            return parts.length > 5 ? parts[5] : spreadsheetId;
+        }
+        connect() {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    // Initialise oAuth client
+                    const db = getGlobalDB();
+                    const googleConfig = yield getScopedConfig(db, {
+                        type: Configs.GOOGLE,
+                    });
+                    const oauthClient = new google_auth_library_1.OAuth2Client({
+                        clientId: googleConfig.clientID,
+                        clientSecret: googleConfig.clientSecret,
+                    });
+                    oauthClient.credentials.access_token = this.config.auth.accessToken;
+                    oauthClient.credentials.refresh_token = this.config.auth.refreshToken;
+                    this.client.useOAuth2Client(oauthClient);
+                    yield this.client.loadInfo();
+                }
+                catch (err) {
+                    console.error("Error connecting to google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        buildSchema(datasourceId) {
+            return __awaiter(this, void 0, void 0, function* () {
+                yield this.connect();
+                const sheets = yield this.client.sheetsByIndex;
+                const tables = {};
+                for (let sheet of sheets) {
+                    // must fetch rows to determine schema
+                    yield sheet.getRows();
+                    // build schema
+                    const schema = {};
+                    // build schema from headers
+                    for (let header of sheet.headerValues) {
+                        schema[header] = {
+                            name: header,
+                            type: constants_1.FieldTypes.STRING,
+                        };
+                    }
+                    // create tables
+                    tables[sheet.title] = {
+                        _id: utils_1.buildExternalTableId(datasourceId, sheet.title),
+                        name: sheet.title,
+                        primary: ["rowNumber"],
+                        schema,
+                    };
+                }
+                this.tables = tables;
+            });
+        }
+        query(json) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const sheet = json.endpoint.entityId;
+                const handlers = {
+                    [constants_1.DataSourceOperation.CREATE]: () => this.create({ sheet, row: json.body }),
+                    [constants_1.DataSourceOperation.READ]: () => this.read({ sheet }),
+                    [constants_1.DataSourceOperation.UPDATE]: () => {
+                        var _a, _b, _c;
+                        return this.update({
+                            // exclude the header row and zero index
+                            rowIndex: ((_c = (_b = (_a = json.extra) === null || _a === void 0 ? void 0 : _a.idFilter) === null || _b === void 0 ? void 0 : _b.equal) === null || _c === void 0 ? void 0 : _c.rowNumber) - 2,
+                            sheet,
+                            row: json.body,
+                        });
+                    },
+                    [constants_1.DataSourceOperation.DELETE]: () => {
+                        var _a, _b, _c;
+                        return this.delete({
+                            // exclude the header row and zero index
+                            rowIndex: ((_c = (_b = (_a = json.extra) === null || _a === void 0 ? void 0 : _a.idFilter) === null || _b === void 0 ? void 0 : _b.equal) === null || _c === void 0 ? void 0 : _c.rowNumber) - 2,
+                            sheet,
+                        });
+                    },
+                    [constants_1.DataSourceOperation.CREATE_TABLE]: () => { var _a; return this.createTable((_a = json === null || json === void 0 ? void 0 : json.table) === null || _a === void 0 ? void 0 : _a.name); },
+                    [constants_1.DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
+                    [constants_1.DataSourceOperation.DELETE_TABLE]: () => { var _a; return this.deleteTable((_a = json === null || json === void 0 ? void 0 : json.table) === null || _a === void 0 ? void 0 : _a.name); },
+                };
+                const internalQueryMethod = handlers[json.endpoint.operation];
+                return yield internalQueryMethod();
+            });
+        }
+        buildRowObject(headers, values, rowNumber) {
+            const rowObject = { rowNumber };
+            for (let i = 0; i < headers.length; i++) {
+                rowObject._id = rowNumber;
+                rowObject[headers[i]] = values[i];
+            }
+            return rowObject;
+        }
+        createTable(name) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.addSheet({ title: name });
+                    return sheet;
+                }
+                catch (err) {
+                    console.error("Error creating new table in google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        updateTable(table) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.sheetsByTitle[table.name];
+                    yield sheet.loadHeaderRow();
+                    if (table._rename) {
+                        const headers = [];
+                        for (let header of sheet.headerValues) {
+                            if (header === table._rename.old) {
+                                headers.push(table._rename.updated);
+                            }
+                            else {
+                                headers.push(header);
+                            }
+                        }
+                        yield sheet.setHeaderRow(headers);
+                    }
+                    else {
+                        let newField = Object.keys(table.schema).find(key => !sheet.headerValues.includes(key));
+                        yield sheet.setHeaderRow([...sheet.headerValues, newField]);
+                    }
+                }
+                catch (err) {
+                    console.error("Error updating table in google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        deleteTable(sheet) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheetToDelete = yield this.client.sheetsByTitle[sheet];
+                    return yield sheetToDelete.delete();
+                }
+                catch (err) {
+                    console.error("Error deleting table in google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        create(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.sheetsByTitle[query.sheet];
+                    const rowToInsert = typeof query.row === "string" ? JSON.parse(query.row) : query.row;
+                    const row = yield sheet.addRow(rowToInsert);
+                    return [
+                        this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
+                    ];
+                }
+                catch (err) {
+                    console.error("Error writing to google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        read(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.sheetsByTitle[query.sheet];
+                    const rows = yield sheet.getRows();
+                    const headerValues = sheet.headerValues;
+                    const response = [];
+                    for (let row of rows) {
+                        response.push(this.buildRowObject(headerValues, row._rawData, row._rowNumber));
+                    }
+                    return response;
+                }
+                catch (err) {
+                    console.error("Error reading from google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        update(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.sheetsByTitle[query.sheet];
+                    const rows = yield sheet.getRows();
+                    const row = rows[query.rowIndex];
+                    if (row) {
+                        const updateValues = query.row;
+                        for (let key in updateValues) {
+                            row[key] = updateValues[key];
+                        }
+                        yield row.save();
+                        return [
+                            this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
+                        ];
+                    }
+                    else {
+                        throw new Error("Row does not exist.");
+                    }
+                }
+                catch (err) {
+                    console.error("Error reading from google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        delete(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                yield this.connect();
+                const sheet = yield this.client.sheetsByTitle[query.sheet];
+                const rows = yield sheet.getRows();
+                const row = rows[query.rowIndex];
+                if (row) {
+                    yield row.delete();
+                    return [{ deleted: query.rowIndex }];
+                }
+                else {
+                    throw new Error("Row does not exist.");
+                }
+            });
+        }
+    }
+    module.exports = {
+        schema: SCHEMA,
+        integration: GoogleSheetsIntegration,
+    };
+})(GoogleSheetsModule || (GoogleSheetsModule = {}));

package/dist/integrations/index.js

@@ -10,6 +10,7 @@ const airtable = require("./airtable");
 const mysql = require("./mysql");
 const arangodb = require("./arangodb");
 const rest = require("./rest");
+const googlesheets = require("./googlesheets");
 const { SourceNames } = require("../definitions/datasource");
 const DEFINITIONS = {
     [SourceNames.POSTGRES]: postgres.schema,
@@ -23,6 +24,7 @@ const DEFINITIONS = {
     [SourceNames.MYSQL]: mysql.schema,
     [SourceNames.ARANGODB]: arangodb.schema,
     [SourceNames.REST]: rest.schema,
+    [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 };
 const INTEGRATIONS = {
     [SourceNames.POSTGRES]: postgres.integration,
@@ -36,6 +38,7 @@ const INTEGRATIONS = {
     [SourceNames.MYSQL]: mysql.integration,
     [SourceNames.ARANGODB]: arangodb.integration,
     [SourceNames.REST]: rest.integration,
+    [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 };
 // optionally add oracle integration if the oracle binary can be installed
 if (!(process.arch === "arm64" && process.platform === "darwin")) {

package/dist/integrations/utils.js

@@ -52,6 +52,9 @@ function buildExternalTableId(datasourceId, tableName) {
 }
 exports.buildExternalTableId = buildExternalTableId;
 function breakExternalTableId(tableId) {
+    if (!tableId) {
+        return {};
+    }
     const parts = tableId.split(DOUBLE_SEPARATOR);
     let tableName = parts.pop();
     // if they need joined
@@ -125,7 +128,12 @@ function isSQL(datasource) {
     if (!datasource || !datasource.source) {
         return false;
     }
-    const SQL = [datasource_1.SourceNames.POSTGRES, datasource_1.SourceNames.SQL_SERVER, datasource_1.SourceNames.MYSQL];
+    const SQL = [
+        datasource_1.SourceNames.POSTGRES,
+        datasource_1.SourceNames.SQL_SERVER,
+        datasource_1.SourceNames.MYSQL,
+        datasource_1.SourceNames.ORACLE,
+    ];
     return SQL.indexOf(datasource.source) !== -1;
 }
 exports.isSQL = isSQL;
@@ -137,8 +145,44 @@ function isIsoDateString(str) {
     return d.toISOString() === str;
 }
 exports.isIsoDateString = isIsoDateString;
-// add the existing relationships from the entities if they exist, to prevent them from being overridden
-function copyExistingPropsOver(tableName, table, entities) {
+/**
+ * This function will determine whether a column is a relationship and whether it
+ * is currently valid. The reason for the validity check is that tables can be deleted
+ * outside of Budibase control and if this is the case it will break Budibase relationships.
+ * The tableIds is a list passed down from the main finalise tables function, which is
+ * based on the tables that have just been fetched. This will only really be used on subsequent
+ * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
+ * @param column The column to check, to see if it is a valid relationship.
+ * @param tableIds The IDs of the tables which currently exist.
+ */
+function shouldCopyRelationship(column, tableIds) {
+    return (column.type === constants_1.FieldTypes.LINK &&
+        column.tableId &&
+        tableIds.includes(column.tableId));
+}
+/**
+ * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
+ * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
+ * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
+ * column has changed type in the external database then copying it over may not be possible.
+ * @param column The column to check for options or boolean type.
+ * @param fetchedColumn The fetched column to check for the type in the external database.
+ */
+function shouldCopySpecialColumn(column, fetchedColumn) {
+    return (column.type === constants_1.FieldTypes.OPTIONS ||
+        column.type === constants_1.FieldTypes.ARRAY ||
+        ((!fetchedColumn || fetchedColumn.type === constants_1.FieldTypes.NUMBER) &&
+            column.type === constants_1.FieldTypes.BOOLEAN));
+}
+/**
+ * Looks for columns which need to be copied over into the new table definitions, like relationships
+ * and options types.
+ * @param tableName The name of the table which is being checked.
+ * @param table The specific table which is being checked.
+ * @param entities All the tables that existed before - the old table definitions.
+ * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
+ */
+function copyExistingPropsOver(tableName, table, entities, tableIds) {
     if (entities && entities[tableName]) {
         if (entities[tableName].primaryDisplay) {
             table.primaryDisplay = entities[tableName].primaryDisplay;
@@ -148,21 +192,28 @@ function copyExistingPropsOver(tableName, table, entities) {
             if (!existingTableSchema.hasOwnProperty(key)) {
                 continue;
             }
-            if (existingTableSchema[key].type === constants_1.FieldTypes.LINK ||
-                existingTableSchema[key].type === constants_1.FieldTypes.OPTIONS ||
-                existingTableSchema[key].type === constants_1.FieldTypes.ARRAY ||
-                ((!table.schema[key] || table.schema[key].type === constants_1.FieldTypes.NUMBER) &&
-                    existingTableSchema[key].type === constants_1.FieldTypes.BOOLEAN)) {
+            const column = existingTableSchema[key];
+            if (shouldCopyRelationship(column, tableIds) ||
+                shouldCopySpecialColumn(column, table.schema[key])) {
                 table.schema[key] = existingTableSchema[key];
             }
         }
     }
     return table;
 }
+/**
+ * Look through the final table definitions to see if anything needs to be
+ * copied over from the old and if any errors have occurred mark them so
+ * that the user can be made aware.
+ * @param tables The list of tables that have been retrieved from the external database.
+ * @param entities The old list of tables, if there was any to look for definitions in.
+ */
 function finaliseExternalTables(tables, entities) {
     const invalidColumns = Object.values(constants_1.InvalidColumns);
     let finalTables = {};
     const errors = {};
+    // @ts-ignore
+    const tableIds = Object.values(tables).map(table => table._id);
     for (let [name, table] of Object.entries(tables)) {
         const schemaFields = Object.keys(table.schema);
         // make sure every table has a key
@@ -175,7 +226,7 @@ function finaliseExternalTables(tables, entities) {
             continue;
         }
         // make sure all previous props have been added back
-        finalTables[name] = copyExistingPropsOver(name, table, entities);
+        finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds);
     }
     // sort the tables by name
     finalTables = Object.entries(finalTables)