@budibase/server 1.0.49 → 1.0.50-alpha.0

package/dist/api/controllers/screen.js

@@ -8,20 +8,18 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const CouchDB = require("../../db");
 const { getScreenParams, generateScreenID } = require("../../db/utils");
 const { AccessController } = require("@budibase/backend-core/roles");
+const { getAppDB } = require("@budibase/backend-core/context");
 exports.fetch = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
-    const appId = ctx.appId;
-    const db = new CouchDB(appId);
+    const db = getAppDB();
     const screens = (yield db.allDocs(getScreenParams(null, {
         include_docs: true,
     }))).rows.map(element => element.doc);
-    ctx.body = yield new AccessController(appId).checkScreensAccess(screens, ctx.user.role._id);
+    ctx.body = yield new AccessController().checkScreensAccess(screens, ctx.user.role._id);
 });
 exports.save = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
-    const appId = ctx.appId;
-    const db = new CouchDB(appId);
+    const db = getAppDB();
     let screen = ctx.request.body;
     if (!screen._id) {
         screen._id = generateScreenID();
@@ -31,7 +29,7 @@ exports.save = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
     ctx.body = Object.assign(Object.assign({}, screen), { _id: response.id, _rev: response.rev });
 });
 exports.destroy = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
-    const db = new CouchDB(ctx.appId);
+    const db = getAppDB();
     yield db.remove(ctx.params.screenId, ctx.params.screenRev);
     ctx.body = {
         message: "Screen deleted successfully",

package/dist/api/controllers/static/index.js

@@ -14,14 +14,16 @@ const { resolve, join } = require("../../../utilities/centralPath");
 const uuid = require("uuid");
 const { ObjectStoreBuckets } = require("../../../constants");
 const { processString } = require("@budibase/string-templates");
-const { getDeployedApps } = require("../../../utilities/workerRequests");
-const CouchDB = require("../../../db");
+const { getAllApps } = require("@budibase/backend-core/db");
 const { loadHandlebarsFile, NODE_MODULES_PATH, TOP_LEVEL_PATH, } = require("../../../utilities/fileSystem");
 const env = require("../../../environment");
 const { clientLibraryPath } = require("../../../utilities");
 const { upload } = require("../../../utilities/fileSystem");
 const { attachmentsRelativeURL } = require("../../../utilities");
 const { DocumentTypes } = require("../../../db/utils");
+const { getAppDB, updateAppId } = require("@budibase/backend-core/context");
+const AWS = require("aws-sdk");
+const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1";
 function prepareUpload({ s3Key, bucket, metadata, file }) {
     return __awaiter(this, void 0, void 0, function* () {
         const response = yield upload({
@@ -41,17 +43,16 @@ function prepareUpload({ s3Key, bucket, metadata, file }) {
         };
     });
 }
-function checkForSelfHostedURL(ctx) {
+function getAppIdFromUrl(ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        // the "appId" component of the URL may actually be a specific self hosted URL
+        // the "appId" component of the URL can be the id or the custom url
         let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`;
-        const apps = yield getDeployedApps();
-        if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
-            return apps[possibleAppUrl].appId;
-        }
-        else {
-            return ctx.params.appId;
-        }
+        // search prod apps for a url that matches, exclude dev where id is always used
+        const apps = yield getAllApps({ dev: false });
+        const app = apps.filter(a => a.url && a.url.toLowerCase() === possibleAppUrl)[0];
+        const appId = app && app.appId ? app.appId : ctx.params.appId;
+        updateAppId(appId);
+        return appId;
     });
 }
 exports.serveBuilder = function (ctx) {
@@ -80,12 +81,9 @@ exports.uploadFile = function (ctx) {
 };
 exports.serveApp = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        let appId = ctx.params.appId;
-        if (env.SELF_HOSTED) {
-            appId = yield checkForSelfHostedURL(ctx);
-        }
+        let appId = yield getAppIdFromUrl(ctx);
         const App = require("./templates/BudibaseApp.svelte").default;
-        const db = new CouchDB(appId, { skip_setup: true });
+        const db = getAppDB({ skip_setup: true });
         const appInfo = yield db.get(DocumentTypes.APP_METADATA);
         const { head, html, css } = App.render({
             title: appInfo.name,
@@ -109,3 +107,51 @@ exports.serveClientLibrary = function (ctx) {
         });
     });
 };
+exports.getSignedUploadURL = function (ctx) {
+    var _a, _b, _c;
+    return __awaiter(this, void 0, void 0, function* () {
+        const database = getAppDB();
+        // Ensure datasource is valid
+        let datasource;
+        try {
+            const { datasourceId } = ctx.params;
+            datasource = yield database.get(datasourceId);
+            if (!datasource) {
+                ctx.throw(400, "The specified datasource could not be found");
+            }
+        }
+        catch (error) {
+            ctx.throw(400, "The specified datasource could not be found");
+        }
+        // Ensure we aren't using a custom endpoint
+        if ((_a = datasource === null || datasource === void 0 ? void 0 : datasource.config) === null || _a === void 0 ? void 0 : _a.endpoint) {
+            ctx.throw(400, "S3 datasources with custom endpoints are not supported");
+        }
+        // Determine type of datasource and generate signed URL
+        let signedUrl;
+        let publicUrl;
+        if (datasource.source === "S3") {
+            const { bucket, key } = ctx.request.body || {};
+            if (!bucket || !key) {
+                ctx.throw(400, "bucket and key values are required");
+                return;
+            }
+            try {
+                const s3 = new AWS.S3({
+                    region: AWS_REGION,
+                    accessKeyId: (_b = datasource === null || datasource === void 0 ? void 0 : datasource.config) === null || _b === void 0 ? void 0 : _b.accessKeyId,
+                    secretAccessKey: (_c = datasource === null || datasource === void 0 ? void 0 : datasource.config) === null || _c === void 0 ? void 0 : _c.secretAccessKey,
+                    apiVersion: "2006-03-01",
+                    signatureVersion: "v4",
+                });
+                const params = { Bucket: bucket, Key: key };
+                signedUrl = s3.getSignedUrl("putObject", params);
+                publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`;
+            }
+            catch (error) {
+                ctx.throw(400, error);
+            }
+        }
+        ctx.body = { signedUrl, publicUrl };
+    });
+};

package/dist/api/controllers/table/bulkFormula.js

@@ -0,0 +1,159 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { FieldTypes, FormulaTypes } = require("../../../constants");
+const { getAllInternalTables, clearColumns } = require("./utils");
+const { doesContainStrings } = require("@budibase/string-templates");
+const { cloneDeep } = require("lodash/fp");
+const { isEqual, uniq } = require("lodash");
+const { updateAllFormulasInTable } = require("../row/staticFormula");
+const { getAppDB } = require("@budibase/backend-core/context");
+function isStaticFormula(column) {
+    return (column.type === FieldTypes.FORMULA &&
+        column.formulaType === FormulaTypes.STATIC);
+}
+/**
+ * This retrieves the formula columns from a table schema that use a specified column name
+ * in the formula.
+ */
+function getFormulaThatUseColumn(table, columnNames) {
+    let formula = [];
+    columnNames = Array.isArray(columnNames) ? columnNames : [columnNames];
+    for (let column of Object.values(table.schema)) {
+        // not a static formula, or doesn't contain a relationship
+        if (!isStaticFormula(column)) {
+            continue;
+        }
+        if (!doesContainStrings(column.formula, columnNames)) {
+            continue;
+        }
+        formula.push(column.name);
+    }
+    return formula;
+}
+/**
+ * This functions checks for when a related table, column or related column is deleted, if any
+ * tables need to have the formula column removed.
+ */
+function checkIfFormulaNeedsCleared(table, { oldTable, deletion }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // start by retrieving all tables, remove the current table from the list
+        const tables = (yield getAllInternalTables()).filter(tbl => tbl._id !== table._id);
+        const schemaToUse = oldTable ? oldTable.schema : table.schema;
+        let removedColumns = Object.values(schemaToUse).filter(column => deletion || !table.schema[column.name]);
+        // remove any formula columns that used related columns
+        for (let removed of removedColumns) {
+            let tableToUse = table;
+            // if relationship, get the related table
+            if (removed.type === FieldTypes.LINK) {
+                tableToUse = tables.find(table => table._id === removed.tableId);
+            }
+            const columnsToDelete = getFormulaThatUseColumn(tableToUse, removed.name);
+            if (columnsToDelete.length > 0) {
+                yield clearColumns(table, columnsToDelete);
+            }
+            // need a special case, where a column has been removed from this table, but was used
+            // in a different, related tables formula
+            if (!table.relatedFormula) {
+                continue;
+            }
+            for (let relatedTableId of table.relatedFormula) {
+                const relatedColumns = Object.values(table.schema).filter(column => column.tableId === relatedTableId);
+                const relatedTable = tables.find(table => table._id === relatedTableId);
+                // look to see if the column was used in a relationship formula,
+                // relationships won't be used for this
+                if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
+                    let relatedFormulaToRemove = [];
+                    for (let column of relatedColumns) {
+                        relatedFormulaToRemove = relatedFormulaToRemove.concat(getFormulaThatUseColumn(relatedTable, [
+                            column.fieldName,
+                            removed.name,
+                        ]));
+                    }
+                    if (relatedFormulaToRemove.length > 0) {
+                        yield clearColumns(relatedTable, uniq(relatedFormulaToRemove));
+                    }
+                }
+            }
+        }
+    });
+}
+/**
+ * This function adds a note to related tables that they are
+ * used in a static formula - so that the link controller
+ * can manage hydrating related rows formula fields. This is
+ * specifically only for static formula.
+ */
+function updateRelatedFormulaLinksOnTables(table, { deletion } = { deletion: false }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const db = getAppDB();
+        // start by retrieving all tables, remove the current table from the list
+        const tables = (yield getAllInternalTables()).filter(tbl => tbl._id !== table._id);
+        // clone the tables, so we can compare at end
+        const initialTables = cloneDeep(tables);
+        // first find the related column names
+        const relatedColumns = Object.values(table.schema).filter(col => col.type === FieldTypes.LINK);
+        // we start by removing the formula field from all tables
+        for (let otherTable of tables) {
+            if (!otherTable.relatedFormula) {
+                continue;
+            }
+            const index = otherTable.relatedFormula.indexOf(table._id);
+            if (index !== -1) {
+                otherTable.relatedFormula.splice(index, 1);
+            }
+        }
+        // if deleting, just remove the table IDs, don't try add
+        if (!deletion) {
+            for (let relatedCol of relatedColumns) {
+                let columns = getFormulaThatUseColumn(table, relatedCol.name);
+                if (!columns || columns.length === 0) {
+                    continue;
+                }
+                const relatedTable = tables.find(related => related._id === relatedCol.tableId);
+                // check if the table is already in the list of related formula, if it isn't, then add it
+                if (relatedTable &&
+                    (!relatedTable.relatedFormula ||
+                        !relatedTable.relatedFormula.includes(table._id))) {
+                    relatedTable.relatedFormula = relatedTable.relatedFormula
+                        ? [...relatedTable.relatedFormula, table._id]
+                        : [table._id];
+                }
+            }
+        }
+        // now we just need to compare all the tables and see if any need saved
+        for (let initial of initialTables) {
+            const found = tables.find(tbl => initial._id === tbl._id);
+            if (found && !isEqual(initial, found)) {
+                yield db.put(found);
+            }
+        }
+    });
+}
+function checkIfFormulaUpdated(table, { oldTable }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // look to see if any formula values have changed
+        const shouldUpdate = Object.values(table.schema).find(column => isStaticFormula(column) &&
+            (!oldTable ||
+                !oldTable.schema[column.name] ||
+                !isEqual(oldTable.schema[column.name], column)));
+        // if a static formula column has updated, then need to run the update
+        if (shouldUpdate != null) {
+            yield updateAllFormulasInTable(table);
+        }
+    });
+}
+exports.runStaticFormulaChecks = (table, { oldTable, deletion }) => __awaiter(void 0, void 0, void 0, function* () {
+    yield updateRelatedFormulaLinksOnTables(table, { deletion });
+    yield checkIfFormulaNeedsCleared(table, { oldTable, deletion });
+    if (!deletion) {
+        yield checkIfFormulaUpdated(table, { oldTable });
+    }
+});

package/dist/api/controllers/table/external.js

@@ -8,7 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const CouchDB = require("../../../db");
 const { buildExternalTableId, breakExternalTableId, } = require("../../../integrations/utils");
 const { getTable, generateForeignKey, generateJunctionTableName, foreignKeyStructure, hasTypeChanged, } = require("./utils");
 const { DataSourceOperation, FieldTypes, RelationshipTypes, } = require("../../../constants");
@@ -16,6 +15,7 @@ const { makeExternalQuery } = require("../../../integrations/base/utils");
 const { cloneDeep } = require("lodash/fp");
 const csvParser = require("../../../utilities/csvParser");
 const { handleRequest } = require("../row/external");
+const { getAppDB } = require("@budibase/backend-core/context");
 function makeTableRequest(datasource, operation, table, tables, oldTable = null) {
     return __awaiter(this, void 0, void 0, function* () {
         const json = {
@@ -134,7 +134,6 @@ function isRelationshipSetup(column) {
 }
 exports.save = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        const appId = ctx.appId;
         const table = ctx.request.body;
         // can't do this right now
         delete table.dataImport;
@@ -146,12 +145,12 @@ exports.save = function (ctx) {
         let tableToSave = Object.assign({ type: "table", _id: buildExternalTableId(datasourceId, table.name) }, table);
         let oldTable;
         if (ctx.request.body && ctx.request.body._id) {
-            oldTable = yield getTable(appId, ctx.request.body._id);
+            oldTable = yield getTable(ctx.request.body._id);
         }
         if (hasTypeChanged(tableToSave, oldTable)) {
             ctx.throw(400, "A column type has changed.");
         }
-        const db = new CouchDB(appId);
+        const db = getAppDB();
         const datasource = yield db.get(datasourceId);
         const oldTables = cloneDeep(datasource.entities);
         const tables = datasource.entities;
@@ -215,13 +214,12 @@ exports.save = function (ctx) {
 };
 exports.destroy = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        const appId = ctx.appId;
-        const tableToDelete = yield getTable(appId, ctx.params.tableId);
+        const tableToDelete = yield getTable(ctx.params.tableId);
         if (!tableToDelete || !tableToDelete.created) {
             ctx.throw(400, "Cannot delete tables which weren't created in Budibase.");
         }
         const datasourceId = getDatasourceId(tableToDelete);
-        const db = new CouchDB(appId);
+        const db = getAppDB();
         const datasource = yield db.get(datasourceId);
         const tables = datasource.entities;
         const operation = DataSourceOperation.DELETE_TABLE;
@@ -234,14 +232,13 @@ exports.destroy = function (ctx) {
 };
 exports.bulkImport = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        const appId = ctx.appId;
-        const table = yield getTable(appId, ctx.params.tableId);
+        const table = yield getTable(ctx.params.tableId);
         const { dataImport } = ctx.request.body;
         if (!dataImport || !dataImport.schema || !dataImport.csvString) {
             ctx.throw(400, "Provided data import information is invalid.");
         }
         const rows = yield csvParser.transform(Object.assign(Object.assign({}, dataImport), { existingTable: table }));
-        yield handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
+        yield handleRequest(DataSourceOperation.BULK_CREATE, table._id, {
             rows,
         });
         return table;

package/dist/api/controllers/table/index.js

@@ -8,13 +8,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const CouchDB = require("../../../db");
 const internal = require("./internal");
 const external = require("./external");
 const csvParser = require("../../../utilities/csvParser");
-const { isExternalTable } = require("../../../integrations/utils");
-const { getTableParams, getDatasourceParams, BudibaseInternalDB, } = require("../../../db/utils");
-const { getTable } = require("./utils");
+const { isExternalTable, isSQL } = require("../../../integrations/utils");
+const { getDatasourceParams } = require("../../../db/utils");
+const { getAppDB } = require("@budibase/backend-core/context");
+const { getTable, getAllInternalTables } = require("./utils");
 function pickApi({ tableId, table }) {
     if (table && !tableId) {
         tableId = table._id;
@@ -30,16 +30,19 @@ function pickApi({ tableId, table }) {
 // covers both internal and external
 exports.fetch = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        const db = new CouchDB(ctx.appId);
-        const internalTables = yield db.allDocs(getTableParams(null, {
-            include_docs: true,
-        }));
-        const internal = internalTables.rows.map(row => (Object.assign(Object.assign({}, row.doc), { type: "internal", sourceId: BudibaseInternalDB._id })));
+        const db = getAppDB();
+        const internal = yield getAllInternalTables();
         const externalTables = yield db.allDocs(getDatasourceParams("plus", {
             include_docs: true,
         }));
-        const external = externalTables.rows.flatMap(row => {
-            return Object.values(row.doc.entities || {}).map(entity => (Object.assign(Object.assign({}, entity), { type: "external", sourceId: row.doc._id })));
+        const external = externalTables.rows.flatMap(tableDoc => {
+            let entities = tableDoc.doc.entities;
+            if (entities) {
+                return Object.values(entities).map(entity => (Object.assign(Object.assign({}, entity), { type: "external", sourceId: tableDoc.doc._id, sql: isSQL(tableDoc.doc) })));
+            }
+            else {
+                return [];
+            }
         });
         ctx.body = [...internal, ...external];
     });
@@ -47,7 +50,7 @@ exports.fetch = function (ctx) {
 exports.find = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const tableId = ctx.params.id;
-        ctx.body = yield getTable(ctx.appId, tableId);
+        ctx.body = yield getTable(tableId);
     });
 };
 exports.save = function (ctx) {
@@ -90,7 +93,7 @@ exports.validateCSVSchema = function (ctx) {
         const { csvString, schema = {}, tableId } = ctx.request.body;
         let existingTable;
         if (tableId) {
-            existingTable = yield getTable(ctx.appId, tableId);
+            existingTable = yield getTable(tableId);
         }
         let result = yield csvParser.parse(csvString, schema);
         if (existingTable) {

package/dist/api/controllers/table/internal.js

@@ -19,15 +19,18 @@ var __rest = (this && this.__rest) || function (s, e) {
         }
     return t;
 };
-const CouchDB = require("../../../db");
 const linkRows = require("../../../db/linkedRows");
 const { getRowParams, generateTableID } = require("../../../db/utils");
 const { FieldTypes } = require("../../../constants");
 const { TableSaveFunctions, hasTypeChanged, getTable, handleDataImport, } = require("./utils");
+const usageQuota = require("../../../utilities/usageQuota");
+const { getAppDB } = require("@budibase/backend-core/context");
+const env = require("../../../environment");
+const { cleanupAttachments } = require("../../../utilities/rowProcessor");
+const { runStaticFormulaChecks } = require("./bulkFormula");
 exports.save = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        const appId = ctx.appId;
-        const db = new CouchDB(appId);
+        const db = getAppDB();
         const _a = ctx.request.body, { dataImport } = _a, rest = __rest(_a, ["dataImport"]);
         let tableToSave = Object.assign({ type: "table", _id: generateTableID(), views: {} }, rest);
         // if the table obj had an _id then it will have been retrieved
@@ -41,8 +44,7 @@ exports.save = function (ctx) {
         // saving a table is a complex operation, involving many different steps, this
         // has been broken out into a utility to make it more obvious/easier to manipulate
         const tableSaveFunctions = new TableSaveFunctions({
-            db,
-            ctx,
+            user: ctx.user,
             oldTable,
             dataImport,
         });
@@ -82,7 +84,6 @@ exports.save = function (ctx) {
         // update linked rows
         try {
             const linkResp = yield linkRows.updateLinks({
-                appId,
                 eventType: oldTable
                     ? linkRows.EventType.TABLE_UPDATED
                     : linkRows.EventType.TABLE_SAVE,
@@ -105,42 +106,47 @@ exports.save = function (ctx) {
         const result = yield db.put(tableToSave);
         tableToSave._rev = result.rev;
         tableToSave = yield tableSaveFunctions.after(tableToSave);
+        // has to run after, make sure it has _id
+        yield runStaticFormulaChecks(tableToSave, { oldTable });
         return tableToSave;
     });
 };
 exports.destroy = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        const appId = ctx.appId;
-        const db = new CouchDB(appId);
+        const db = getAppDB();
         const tableToDelete = yield db.get(ctx.params.tableId);
         // Delete all rows for that table
         const rows = yield db.allDocs(getRowParams(ctx.params.tableId, null, {
             include_docs: true,
         }));
         yield db.bulkDocs(rows.rows.map(row => (Object.assign(Object.assign({}, row.doc), { _deleted: true }))));
+        yield usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length);
         // update linked rows
         yield linkRows.updateLinks({
-            appId,
             eventType: linkRows.EventType.TABLE_DELETE,
             table: tableToDelete,
         });
         // don't remove the table itself until very end
         yield db.remove(tableToDelete);
         // remove table search index
-        const currentIndexes = yield db.getIndexes();
-        const existingIndex = currentIndexes.indexes.find(existing => existing.name === `search:${ctx.params.tableId}`);
-        if (existingIndex) {
-            yield db.deleteIndex(existingIndex);
+        if (!env.isTest()) {
+            const currentIndexes = yield db.getIndexes();
+            const existingIndex = currentIndexes.indexes.find(existing => existing.name === `search:${ctx.params.tableId}`);
+            if (existingIndex) {
+                yield db.deleteIndex(existingIndex);
+            }
         }
+        // has to run after, make sure it has _id
+        yield runStaticFormulaChecks(tableToDelete, { deletion: true });
+        yield cleanupAttachments(tableToDelete, { rows });
         return tableToDelete;
     });
 };
 exports.bulkImport = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
-        const appId = ctx.appId;
-        const table = yield getTable(appId, ctx.params.tableId);
+        const table = yield getTable(ctx.params.tableId);
         const { dataImport } = ctx.request.body;
-        yield handleDataImport(appId, ctx.user, table, dataImport);
+        yield handleDataImport(ctx.user, table, dataImport);
         return table;
     });
 };