@budibase/server 1.0.48 → 1.0.49-alpha.3

package/dist/db/linkedRows/index.js

@@ -61,7 +61,7 @@ function getLinksForRows(appId, rows) {
             .map(el => (Object.assign(Object.assign({}, el), { unique: el.id + el.thisId + el.fieldName }))), "unique");
     });
 }
-function getFullLinkedDocs(ctx, appId, links) {
+function getFullLinkedDocs(appId, links) {
     return __awaiter(this, void 0, void 0, function* () {
         // create DBs
         const db = new CouchDB(appId);
@@ -132,13 +132,12 @@ exports.updateLinks = function (args) {
 /**
  * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
  * This is required for formula fields, this may only be utilised internally (for now).
- * @param {object} ctx The request which is looking for rows.
+ * @param {string} appId The ID of the app which this request is in the context of.
  * @param {object} table The table from which the rows originated.
  * @param {array<object>} rows The rows which are to be enriched.
  * @return {Promise<*>} returns the rows with all of the enriched relationships on it.
  */
-exports.attachFullLinkedDocs = (ctx, table, rows) => __awaiter(void 0, void 0, void 0, function* () {
-    const appId = ctx.appId;
+exports.attachFullLinkedDocs = (appId, table, rows) => __awaiter(void 0, void 0, void 0, function* () {
     const linkedTableIds = getLinkedTableIDs(table);
     if (linkedTableIds.length === 0) {
         return rows;
@@ -150,7 +149,7 @@ exports.attachFullLinkedDocs = (ctx, table, rows) => __awaiter(void 0, void 0, v
     // clear any existing links that could be dupe'd
     rows = clearRelationshipFields(table, rows);
     // now get the docs and combine into the rows
-    let linked = yield getFullLinkedDocs(ctx, appId, links);
+    let linked = yield getFullLinkedDocs(appId, links);
     const linkedTables = [];
     for (let row of rows) {
         for (let link of links.filter(link => link.thisId === row._id)) {

package/dist/definitions/datasource.js

@@ -49,6 +49,7 @@ var SourceNames;
     SourceNames["ARANGODB"] = "ARANGODB";
     SourceNames["REST"] = "REST";
     SourceNames["ORACLE"] = "ORACLE";
+    SourceNames["GOOGLE_SHEETS"] = "GOOGLE_SHEETS";
 })(SourceNames = exports.SourceNames || (exports.SourceNames = {}));
 var IncludeRelationships;
 (function (IncludeRelationships) {

package/dist/environment.js

@@ -30,10 +30,12 @@ module.exports = {
     MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
     MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
     USE_QUOTAS: process.env.USE_QUOTAS,
+    EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
     REDIS_URL: process.env.REDIS_URL,
     REDIS_PASSWORD: process.env.REDIS_PASSWORD,
     INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
     MULTI_TENANCY: process.env.MULTI_TENANCY,
+    HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
     // environment
     NODE_ENV: process.env.NODE_ENV,
     JEST_WORKER_ID: process.env.JEST_WORKER_ID,

package/dist/integrations/base/sql.js

@@ -69,10 +69,16 @@ class InternalBuilder {
         this.client = client;
     }
     // right now we only do filters on the specific table being queried
-    addFilters(tableName, query, filters) {
+    addFilters(query, filters, opts) {
         function iterate(structure, fn) {
             for (let [key, value] of Object.entries(structure)) {
-                fn(`${tableName}.${key}`, value);
+                const isRelationshipField = key.includes(".");
+                if (!opts.relationship && !isRelationshipField) {
+                    fn(`${opts.tableName}.${key}`, value);
+                }
+                if (opts.relationship && isRelationshipField) {
+                    fn(key, value);
+                }
             }
         }
         if (!filters) {
@@ -248,7 +254,7 @@ class InternalBuilder {
         if (foundOffset) {
             query = query.offset(foundOffset);
         }
-        query = this.addFilters(tableName, query, filters);
+        query = this.addFilters(query, filters, { tableName });
         // add sorting to pre-query
         query = this.addSorting(query, json);
         // @ts-ignore
@@ -261,13 +267,14 @@ class InternalBuilder {
             preQuery = this.addSorting(preQuery, json);
         }
         // handle joins
-        return this.addRelationships(knex, preQuery, selectStatement, tableName, relationships);
+        query = this.addRelationships(knex, preQuery, selectStatement, tableName, relationships);
+        return this.addFilters(query, filters, { relationship: true });
     }
     update(knex, json, opts) {
         const { endpoint, body, filters } = json;
         let query = knex(endpoint.entityId);
         const parsedBody = parseBody(body);
-        query = this.addFilters(endpoint.entityId, query, filters);
+        query = this.addFilters(query, filters, { tableName: endpoint.entityId });
         // mysql can't use returning
         if (opts.disableReturning) {
             return query.update(parsedBody);
@@ -279,7 +286,7 @@ class InternalBuilder {
     delete(knex, json, opts) {
         const { endpoint, filters } = json;
         let query = knex(endpoint.entityId);
-        query = this.addFilters(endpoint.entityId, query, filters);
+        query = this.addFilters(query, filters, { tableName: endpoint.entityId });
         // mysql can't use returning
         if (opts.disableReturning) {
             return query.delete();

package/dist/integrations/googlesheets.js

@@ -0,0 +1,340 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const datasource_1 = require("../definitions/datasource");
+const google_auth_library_1 = require("google-auth-library");
+const utils_1 = require("./utils");
+const constants_1 = require("../constants");
+const google_spreadsheet_1 = require("google-spreadsheet");
+var GoogleSheetsModule;
+(function (GoogleSheetsModule) {
+    const { getGlobalDB } = require("@budibase/backend-core/tenancy");
+    const { getScopedConfig } = require("@budibase/backend-core/db");
+    const { Configs } = require("@budibase/backend-core/constants");
+    const SCHEMA = {
+        plus: true,
+        auth: {
+            type: "google",
+        },
+        relationships: false,
+        docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
+        description: "Create and collaborate on online spreadsheets in real-time and from any device. ",
+        friendlyName: "Google Sheets",
+        datasource: {
+            spreadsheetId: {
+                type: datasource_1.DatasourceFieldTypes.STRING,
+                required: true,
+            },
+        },
+        query: {
+            create: {
+                type: datasource_1.QueryTypes.FIELDS,
+                fields: {
+                    sheet: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                    row: {
+                        type: datasource_1.QueryTypes.JSON,
+                        required: true,
+                    },
+                },
+            },
+            read: {
+                type: datasource_1.QueryTypes.FIELDS,
+                fields: {
+                    sheet: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                },
+            },
+            update: {
+                type: datasource_1.QueryTypes.FIELDS,
+                fields: {
+                    sheet: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                    rowIndex: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                    row: {
+                        type: datasource_1.QueryTypes.JSON,
+                        required: true,
+                    },
+                },
+            },
+            delete: {
+                type: datasource_1.QueryTypes.FIELDS,
+                fields: {
+                    sheet: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                    rowIndex: {
+                        type: datasource_1.DatasourceFieldTypes.NUMBER,
+                        required: true,
+                    },
+                },
+            },
+        },
+    };
+    class GoogleSheetsIntegration {
+        constructor(config) {
+            this.tables = {};
+            this.schemaErrors = {};
+            this.config = config;
+            const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId);
+            this.client = new google_spreadsheet_1.GoogleSpreadsheet(spreadsheetId);
+        }
+        /**
+         * Pull the spreadsheet ID out from a valid google sheets URL
+         * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
+         * @returns spreadsheet Id of the google sheet
+         */
+        cleanSpreadsheetUrl(spreadsheetId) {
+            if (!spreadsheetId) {
+                throw new Error("You must set a spreadsheet ID in your configuration to fetch tables.");
+            }
+            const parts = spreadsheetId.split("/");
+            return parts.length > 5 ? parts[5] : spreadsheetId;
+        }
+        connect() {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    // Initialise oAuth client
+                    const db = getGlobalDB();
+                    const googleConfig = yield getScopedConfig(db, {
+                        type: Configs.GOOGLE,
+                    });
+                    const oauthClient = new google_auth_library_1.OAuth2Client({
+                        clientId: googleConfig.clientID,
+                        clientSecret: googleConfig.clientSecret,
+                    });
+                    oauthClient.credentials.access_token = this.config.auth.accessToken;
+                    oauthClient.credentials.refresh_token = this.config.auth.refreshToken;
+                    this.client.useOAuth2Client(oauthClient);
+                    yield this.client.loadInfo();
+                }
+                catch (err) {
+                    console.error("Error connecting to google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        buildSchema(datasourceId) {
+            return __awaiter(this, void 0, void 0, function* () {
+                yield this.connect();
+                const sheets = yield this.client.sheetsByIndex;
+                const tables = {};
+                for (let sheet of sheets) {
+                    // must fetch rows to determine schema
+                    yield sheet.getRows();
+                    // build schema
+                    const schema = {};
+                    // build schema from headers
+                    for (let header of sheet.headerValues) {
+                        schema[header] = {
+                            name: header,
+                            type: constants_1.FieldTypes.STRING,
+                        };
+                    }
+                    // create tables
+                    tables[sheet.title] = {
+                        _id: utils_1.buildExternalTableId(datasourceId, sheet.title),
+                        name: sheet.title,
+                        primary: ["rowNumber"],
+                        schema,
+                    };
+                }
+                this.tables = tables;
+            });
+        }
+        query(json) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const sheet = json.endpoint.entityId;
+                const handlers = {
+                    [constants_1.DataSourceOperation.CREATE]: () => this.create({ sheet, row: json.body }),
+                    [constants_1.DataSourceOperation.READ]: () => this.read({ sheet }),
+                    [constants_1.DataSourceOperation.UPDATE]: () => {
+                        var _a, _b, _c;
+                        return this.update({
+                            // exclude the header row and zero index
+                            rowIndex: ((_c = (_b = (_a = json.extra) === null || _a === void 0 ? void 0 : _a.idFilter) === null || _b === void 0 ? void 0 : _b.equal) === null || _c === void 0 ? void 0 : _c.rowNumber) - 2,
+                            sheet,
+                            row: json.body,
+                        });
+                    },
+                    [constants_1.DataSourceOperation.DELETE]: () => {
+                        var _a, _b, _c;
+                        return this.delete({
+                            // exclude the header row and zero index
+                            rowIndex: ((_c = (_b = (_a = json.extra) === null || _a === void 0 ? void 0 : _a.idFilter) === null || _b === void 0 ? void 0 : _b.equal) === null || _c === void 0 ? void 0 : _c.rowNumber) - 2,
+                            sheet,
+                        });
+                    },
+                    [constants_1.DataSourceOperation.CREATE_TABLE]: () => { var _a; return this.createTable((_a = json === null || json === void 0 ? void 0 : json.table) === null || _a === void 0 ? void 0 : _a.name); },
+                    [constants_1.DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
+                    [constants_1.DataSourceOperation.DELETE_TABLE]: () => { var _a; return this.deleteTable((_a = json === null || json === void 0 ? void 0 : json.table) === null || _a === void 0 ? void 0 : _a.name); },
+                };
+                const internalQueryMethod = handlers[json.endpoint.operation];
+                return yield internalQueryMethod();
+            });
+        }
+        buildRowObject(headers, values, rowNumber) {
+            const rowObject = { rowNumber };
+            for (let i = 0; i < headers.length; i++) {
+                rowObject._id = rowNumber;
+                rowObject[headers[i]] = values[i];
+            }
+            return rowObject;
+        }
+        createTable(name) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.addSheet({ title: name });
+                    return sheet;
+                }
+                catch (err) {
+                    console.error("Error creating new table in google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        updateTable(table) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.sheetsByTitle[table.name];
+                    yield sheet.loadHeaderRow();
+                    if (table._rename) {
+                        const headers = [];
+                        for (let header of sheet.headerValues) {
+                            if (header === table._rename.old) {
+                                headers.push(table._rename.updated);
+                            }
+                            else {
+                                headers.push(header);
+                            }
+                        }
+                        yield sheet.setHeaderRow(headers);
+                    }
+                    else {
+                        let newField = Object.keys(table.schema).find(key => !sheet.headerValues.includes(key));
+                        yield sheet.setHeaderRow([...sheet.headerValues, newField]);
+                    }
+                }
+                catch (err) {
+                    console.error("Error updating table in google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        deleteTable(sheet) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheetToDelete = yield this.client.sheetsByTitle[sheet];
+                    return yield sheetToDelete.delete();
+                }
+                catch (err) {
+                    console.error("Error deleting table in google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        create(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.sheetsByTitle[query.sheet];
+                    const rowToInsert = typeof query.row === "string" ? JSON.parse(query.row) : query.row;
+                    const row = yield sheet.addRow(rowToInsert);
+                    return [
+                        this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
+                    ];
+                }
+                catch (err) {
+                    console.error("Error writing to google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        read(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.sheetsByTitle[query.sheet];
+                    const rows = yield sheet.getRows();
+                    const headerValues = sheet.headerValues;
+                    const response = [];
+                    for (let row of rows) {
+                        response.push(this.buildRowObject(headerValues, row._rawData, row._rowNumber));
+                    }
+                    return response;
+                }
+                catch (err) {
+                    console.error("Error reading from google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        update(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.connect();
+                    const sheet = yield this.client.sheetsByTitle[query.sheet];
+                    const rows = yield sheet.getRows();
+                    const row = rows[query.rowIndex];
+                    if (row) {
+                        const updateValues = query.row;
+                        for (let key in updateValues) {
+                            row[key] = updateValues[key];
+                        }
+                        yield row.save();
+                        return [
+                            this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
+                        ];
+                    }
+                    else {
+                        throw new Error("Row does not exist.");
+                    }
+                }
+                catch (err) {
+                    console.error("Error reading from google sheets", err);
+                    throw err;
+                }
+            });
+        }
+        delete(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                yield this.connect();
+                const sheet = yield this.client.sheetsByTitle[query.sheet];
+                const rows = yield sheet.getRows();
+                const row = rows[query.rowIndex];
+                if (row) {
+                    yield row.delete();
+                    return [{ deleted: query.rowIndex }];
+                }
+                else {
+                    throw new Error("Row does not exist.");
+                }
+            });
+        }
+    }
+    module.exports = {
+        schema: SCHEMA,
+        integration: GoogleSheetsIntegration,
+    };
+})(GoogleSheetsModule || (GoogleSheetsModule = {}));

package/dist/integrations/index.js

@@ -10,6 +10,7 @@ const airtable = require("./airtable");
 const mysql = require("./mysql");
 const arangodb = require("./arangodb");
 const rest = require("./rest");
+const googlesheets = require("./googlesheets");
 const { SourceNames } = require("../definitions/datasource");
 const DEFINITIONS = {
     [SourceNames.POSTGRES]: postgres.schema,
@@ -23,6 +24,7 @@ const DEFINITIONS = {
     [SourceNames.MYSQL]: mysql.schema,
     [SourceNames.ARANGODB]: arangodb.schema,
     [SourceNames.REST]: rest.schema,
+    [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 };
 const INTEGRATIONS = {
     [SourceNames.POSTGRES]: postgres.integration,
@@ -36,6 +38,7 @@ const INTEGRATIONS = {
     [SourceNames.MYSQL]: mysql.integration,
     [SourceNames.ARANGODB]: arangodb.integration,
     [SourceNames.REST]: rest.integration,
+    [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 };
 // optionally add oracle integration if the oracle binary can be installed
 if (!(process.arch === "arm64" && process.platform === "darwin")) {

package/dist/integrations/utils.js

@@ -125,7 +125,12 @@ function isSQL(datasource) {
     if (!datasource || !datasource.source) {
         return false;
     }
-    const SQL = [datasource_1.SourceNames.POSTGRES, datasource_1.SourceNames.SQL_SERVER, datasource_1.SourceNames.MYSQL];
+    const SQL = [
+        datasource_1.SourceNames.POSTGRES,
+        datasource_1.SourceNames.SQL_SERVER,
+        datasource_1.SourceNames.MYSQL,
+        datasource_1.SourceNames.ORACLE,
+    ];
     return SQL.indexOf(datasource.source) !== -1;
 }
 exports.isSQL = isSQL;
@@ -137,8 +142,44 @@ function isIsoDateString(str) {
     return d.toISOString() === str;
 }
 exports.isIsoDateString = isIsoDateString;
-// add the existing relationships from the entities if they exist, to prevent them from being overridden
-function copyExistingPropsOver(tableName, table, entities) {
+/**
+ * This function will determine whether a column is a relationship and whether it
+ * is currently valid. The reason for the validity check is that tables can be deleted
+ * outside of Budibase control and if this is the case it will break Budibase relationships.
+ * The tableIds is a list passed down from the main finalise tables function, which is
+ * based on the tables that have just been fetched. This will only really be used on subsequent
+ * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
+ * @param column The column to check, to see if it is a valid relationship.
+ * @param tableIds The IDs of the tables which currently exist.
+ */
+function shouldCopyRelationship(column, tableIds) {
+    return (column.type === constants_1.FieldTypes.LINK &&
+        column.tableId &&
+        tableIds.includes(column.tableId));
+}
+/**
+ * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
+ * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
+ * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
+ * column has changed type in the external database then copying it over may not be possible.
+ * @param column The column to check for options or boolean type.
+ * @param fetchedColumn The fetched column to check for the type in the external database.
+ */
+function shouldCopySpecialColumn(column, fetchedColumn) {
+    return (column.type === constants_1.FieldTypes.OPTIONS ||
+        column.type === constants_1.FieldTypes.ARRAY ||
+        ((!fetchedColumn || fetchedColumn.type === constants_1.FieldTypes.NUMBER) &&
+            column.type === constants_1.FieldTypes.BOOLEAN));
+}
+/**
+ * Looks for columns which need to be copied over into the new table definitions, like relationships
+ * and options types.
+ * @param tableName The name of the table which is being checked.
+ * @param table The specific table which is being checked.
+ * @param entities All the tables that existed before - the old table definitions.
+ * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
+ */
+function copyExistingPropsOver(tableName, table, entities, tableIds) {
     if (entities && entities[tableName]) {
         if (entities[tableName].primaryDisplay) {
             table.primaryDisplay = entities[tableName].primaryDisplay;
@@ -148,21 +189,28 @@ function copyExistingPropsOver(tableName, table, entities) {
             if (!existingTableSchema.hasOwnProperty(key)) {
                 continue;
             }
-            if (existingTableSchema[key].type === constants_1.FieldTypes.LINK ||
-                existingTableSchema[key].type === constants_1.FieldTypes.OPTIONS ||
-                existingTableSchema[key].type === constants_1.FieldTypes.ARRAY ||
-                ((!table.schema[key] || table.schema[key].type === constants_1.FieldTypes.NUMBER) &&
-                    existingTableSchema[key].type === constants_1.FieldTypes.BOOLEAN)) {
+            const column = existingTableSchema[key];
+            if (shouldCopyRelationship(column, tableIds) ||
+                shouldCopySpecialColumn(column, table.schema[key])) {
                 table.schema[key] = existingTableSchema[key];
             }
         }
     }
     return table;
 }
+/**
+ * Look through the final table definitions to see if anything needs to be
+ * copied over from the old and if any errors have occurred mark them so
+ * that the user can be made aware.
+ * @param tables The list of tables that have been retrieved from the external database.
+ * @param entities The old list of tables, if there was any to look for definitions in.
+ */
 function finaliseExternalTables(tables, entities) {
     const invalidColumns = Object.values(constants_1.InvalidColumns);
     let finalTables = {};
     const errors = {};
+    // @ts-ignore
+    const tableIds = Object.values(tables).map(table => table._id);
     for (let [name, table] of Object.entries(tables)) {
         const schemaFields = Object.keys(table.schema);
         // make sure every table has a key
@@ -175,7 +223,7 @@ function finaliseExternalTables(tables, entities) {
             continue;
         }
         // make sure all previous props have been added back
-        finalTables[name] = copyExistingPropsOver(name, table, entities);
+        finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds);
     }
     // sort the tables by name
     finalTables = Object.entries(finalTables)