@budibase/server 0.9.188 → 0.9.190-alpha.10

package/dist/api/controllers/application.js

@@ -13,10 +13,9 @@ const env = require("../../environment");
 const packageJson = require("../../../package.json");
 const { createLinkView, createRoutingView, createAllSearchIndex, } = require("../../db/views/staticViews");
 const { getTemplateStream, createApp, deleteApp, } = require("../../utilities/fileSystem");
-const { generateAppID, getLayoutParams, getScreenParams, generateScreenID, generateDevAppID, DocumentTypes, AppStatus, } = require("../../db/utils");
+const { generateAppID, getLayoutParams, getScreenParams, generateDevAppID, DocumentTypes, AppStatus, } = require("../../db/utils");
 const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles");
 const { BASE_LAYOUTS } = require("../../constants/layouts");
-const { createHomeScreen } = require("../../constants/screens");
 const { cloneDeep } = require("lodash/fp");
 const { processObject } = require("@budibase/string-templates");
 const { getAllApps, isDevAppID, getDeployedAppID, Replication, } = require("@budibase/auth/db");
@@ -26,6 +25,9 @@ const { clientLibraryPath, stringToReadStream } = require("../../utilities");
 const { getAllLocks } = require("../../utilities/redis");
 const { updateClientLibrary, backupClientLibrary, revertClientLibrary, } = require("../../utilities/fileSystem/clientLibrary");
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy");
+const { syncGlobalUsers } = require("./user");
+const { app: appCache } = require("@budibase/auth/cache");
+const { cleanupAutomations } = require("../../automations/utils");
 const URL_REGEX_SLASH = /\/|\\/g;
 // utility function, need to do away with this
 function getLayouts(db) {
@@ -213,6 +215,7 @@ exports.create = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
     if (!env.isTest()) {
         yield createApp(appId);
     }
+    yield appCache.invalidateAppMetadata(appId, newApplication);
     ctx.status = 200;
     ctx.body = newApplication;
 });
@@ -267,8 +270,12 @@ exports.delete = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
     if (!env.isTest() && !ctx.query.unpublish) {
         yield deleteApp(ctx.params.appId);
     }
+    if (ctx.query && ctx.query.unpublish) {
+        yield cleanupAutomations(ctx.params.appId);
+    }
     // make sure the app/role doesn't stick around after the app has been deleted
     yield removeAppFromUserRoles(ctx, ctx.params.appId);
+    yield appCache.invalidateAppMetadata(ctx.params.appId);
     ctx.status = 200;
     ctx.body = result;
 });
@@ -277,6 +284,7 @@ exports.sync = (ctx, next) => __awaiter(void 0, void 0, void 0, function* () {
     if (!isDevAppID(appId)) {
         ctx.throw(400, "This action cannot be performed for production apps");
     }
+    // replicate prod to dev
     const prodAppId = getDeployedAppID(appId);
     try {
         const prodDb = new CouchDB(prodAppId, { skip_setup: true });
@@ -307,6 +315,8 @@ exports.sync = (ctx, next) => __awaiter(void 0, void 0, void 0, function* () {
     catch (err) {
         error = err;
     }
+    // sync the users
+    yield syncGlobalUsers(appId);
     if (error) {
         ctx.throw(400, error);
     }
@@ -327,7 +337,10 @@ const updateAppPackage = (ctx, appPackage, appId) => __awaiter(void 0, void 0, v
     // the locked by property is attached by server but generated from
     // Redis, shouldn't ever store it
     delete newAppPackage.lockedBy;
-    return yield db.put(newAppPackage);
+    const response = yield db.put(newAppPackage);
+    // remove any cached metadata, so that it will be updated
+    yield appCache.invalidateAppMetadata(appId);
+    return response;
 });
 const createEmptyAppPackage = (ctx, app) => __awaiter(void 0, void 0, void 0, function* () {
     const db = new CouchDB(app.appId);
@@ -336,8 +349,5 @@ const createEmptyAppPackage = (ctx, app) => __awaiter(void 0, void 0, void 0, fu
         const cloned = cloneDeep(layout);
         screensAndLayouts.push(yield processObject(cloned, app));
     }
-    const homeScreen = createHomeScreen(app);
-    homeScreen._id = generateScreenID();
-    screensAndLayouts.push(homeScreen);
     yield db.bulkDocs(screensAndLayouts);
 });

package/dist/api/controllers/auth.js

@@ -22,6 +22,8 @@ exports.fetchSelf = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
         return;
     }
     const user = yield getFullUser(ctx, userId);
+    // this shouldn't be returned by the app self
+    delete user.roles;
     if (appId) {
         const db = new CouchDB(appId);
         // remove the full roles structure
@@ -40,9 +42,7 @@ exports.fetchSelf = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
             }
             // user has a role of some sort, return them
             else if (err.status === 404) {
-                const metadata = {
-                    _id: userId,
-                };
+                const metadata = Object.assign(Object.assign({}, user), { _id: userId });
                 const dbResp = yield db.put(metadata);
                 user._rev = dbResp.rev;
                 response = user;

package/dist/api/controllers/datasource.js

@@ -10,8 +10,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 const CouchDB = require("../../db");
 const { generateDatasourceID, getDatasourceParams, getQueryParams, DocumentTypes, BudibaseInternalDB, getTableParams, } = require("../../db/utils");
+const { BuildSchemaErrors, InvalidColumns } = require("../../constants");
 const { integrations } = require("../../integrations");
-const { makeExternalQuery } = require("./row/utils");
+const { getDatasourceAndQuery } = require("./row/utils");
 exports.fetch = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const database = new CouchDB(ctx.appId);
@@ -33,11 +34,15 @@ exports.buildSchemaFromDb = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = new CouchDB(ctx.appId);
         const datasource = yield db.get(ctx.params.datasourceId);
-        const tables = yield buildSchemaHelper(datasource);
+        const { tables, error } = yield buildSchemaHelper(datasource);
         datasource.entities = tables;
-        const response = yield db.put(datasource);
-        datasource._rev = response.rev;
-        ctx.body = datasource;
+        const dbResp = yield db.put(datasource);
+        datasource._rev = dbResp.rev;
+        const response = { datasource };
+        if (error) {
+            response.error = error;
+        }
+        ctx.body = response;
     });
 };
 exports.update = function (ctx) {
@@ -57,7 +62,7 @@ exports.update = function (ctx) {
         }
         ctx.status = 200;
         ctx.message = "Datasource saved successfully.";
-        ctx.body = datasource;
+        ctx.body = { datasource };
     });
 };
 exports.save = function (ctx) {
@@ -66,12 +71,14 @@ exports.save = function (ctx) {
         const plus = ctx.request.body.datasource.plus;
         const fetchSchema = ctx.request.body.fetchSchema;
         const datasource = Object.assign({ _id: generateDatasourceID({ plus }), type: plus ? DocumentTypes.DATASOURCE_PLUS : DocumentTypes.DATASOURCE }, ctx.request.body.datasource);
+        let schemaError = null;
         if (fetchSchema) {
-            let tables = yield buildSchemaHelper(datasource);
+            const { tables, error } = yield buildSchemaHelper(datasource);
+            schemaError = error;
             datasource.entities = tables;
         }
-        const response = yield db.put(datasource);
-        datasource._rev = response.rev;
+        const dbResp = yield db.put(datasource);
+        datasource._rev = dbResp.rev;
         // Drain connection pools when configuration is changed
         if (datasource.source) {
             const source = integrations[datasource.source];
@@ -79,17 +86,23 @@ exports.save = function (ctx) {
                 yield source.pool.end();
             }
         }
-        ctx.status = 200;
-        ctx.message = "Datasource saved successfully.";
-        ctx.body = datasource;
+        const response = { datasource };
+        if (schemaError) {
+            response.error = schemaError;
+        }
+        ctx.body = response;
     });
 };
 exports.destroy = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = new CouchDB(ctx.appId);
         // Delete all queries for the datasource
-        const rows = yield db.allDocs(getQueryParams(ctx.params.datasourceId, null));
-        yield db.bulkDocs(rows.rows.map(row => (Object.assign(Object.assign({}, row.doc), { _deleted: true }))));
+        const queries = yield db.allDocs(getQueryParams(ctx.params.datasourceId, null));
+        yield db.bulkDocs(queries.rows.map(row => ({
+            _id: row.id,
+            _rev: row.value.rev,
+            _deleted: true,
+        })));
         // delete the datasource
         yield db.remove(ctx.params.datasourceId, ctx.params.revId);
         ctx.message = `Datasource deleted.`;
@@ -107,18 +120,56 @@ exports.query = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const queryJson = ctx.request.body;
         try {
-            ctx.body = yield makeExternalQuery(ctx.appId, queryJson);
+            ctx.body = yield getDatasourceAndQuery(ctx.appId, queryJson);
         }
         catch (err) {
             ctx.throw(400, err);
         }
     });
 };
+function getErrorTables(errors, errorType) {
+    return Object.entries(errors)
+        .filter(entry => entry[1] === errorType)
+        .map(([name]) => name);
+}
+function updateError(error, newError, tables) {
+    if (!error) {
+        error = "";
+    }
+    if (error.length > 0) {
+        error += "\n";
+    }
+    error += `${newError} ${tables.join(", ")}`;
+    return error;
+}
 const buildSchemaHelper = (datasource) => __awaiter(void 0, void 0, void 0, function* () {
     const Connector = integrations[datasource.source];
     // Connect to the DB and build the schema
     const connector = new Connector(datasource.config);
     yield connector.buildSchema(datasource._id, datasource.entities);
     datasource.entities = connector.tables;
-    return connector.tables;
+    // make sure they all have a display name selected
+    for (let entity of Object.values(datasource.entities)) {
+        if (entity.primaryDisplay) {
+            continue;
+        }
+        const notAutoColumn = Object.values(entity.schema).find(schema => !schema.autocolumn);
+        if (notAutoColumn) {
+            entity.primaryDisplay = notAutoColumn.name;
+        }
+    }
+    const errors = connector.schemaErrors;
+    let error = null;
+    if (errors && Object.keys(errors).length > 0) {
+        const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY);
+        const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN);
+        if (noKey.length) {
+            error = updateError(error, "No primary key constraint found for the following:", noKey);
+        }
+        if (invalidCol.length) {
+            const invalidCols = Object.values(InvalidColumns).join(", ");
+            error = updateError(error, `Cannot use columns ${invalidCols} found in following:`, invalidCol);
+        }
+    }
+    return { tables: connector.tables, error };
 });

package/dist/api/controllers/deploy/index.js

@@ -13,6 +13,7 @@ const Deployment = require("./Deployment");
 const { Replication, getDeployedAppID } = require("@budibase/auth/db");
 const { DocumentTypes, getAutomationParams } = require("../../../db/utils");
 const { disableAllCrons, enableCronTrigger, } = require("../../../automations/utils");
+const { app: appCache } = require("@budibase/auth/cache");
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000;
 const DeploymentStatus = {
@@ -95,6 +96,7 @@ function deployApp(deployment) {
             appDoc.appId = productionAppId;
             appDoc.instance._id = productionAppId;
             yield db.put(appDoc);
+            yield appCache.invalidateAppMetadata(productionAppId);
             console.log("New app doc written successfully.");
             yield initDeployedApp(productionAppId);
             console.log("Deployed app initialised, setting deployment to successful");

package/dist/api/controllers/dev.js

@@ -16,6 +16,7 @@ const { request } = require("../../utilities/workerRequests");
 const { clearLock } = require("../../utilities/redis");
 const { Replication } = require("@budibase/auth").db;
 const { DocumentTypes } = require("../../db/utils");
+const { app: appCache } = require("@budibase/auth/cache");
 function redirect(ctx, method, path = "global") {
     return __awaiter(this, void 0, void 0, function* () {
         const { devPath } = ctx.params;
@@ -25,7 +26,8 @@ function redirect(ctx, method, path = "global") {
             body: ctx.request.body,
         }, true));
         if (response.status !== 200) {
-            ctx.throw(response.status, response.statusText);
+            const err = yield response.text();
+            ctx.throw(400, err);
         }
         const cookie = response.headers.get("set-cookie");
         if (cookie) {
@@ -102,6 +104,7 @@ exports.revert = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
         appDoc.appId = appId;
         appDoc.instance._id = appId;
         yield db.put(appDoc);
+        yield appCache.invalidateAppMetadata(appId);
         ctx.body = {
             message: "Reverted changes successfully.",
         };

package/dist/api/controllers/query.js

@@ -10,11 +10,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 const { processString } = require("@budibase/string-templates");
 const CouchDB = require("../../db");
-const { generateQueryID, getQueryParams } = require("../../db/utils");
-const { integrations } = require("../../integrations");
+const { generateQueryID, getQueryParams, isProdAppID, } = require("../../db/utils");
 const { BaseQueryVerbs } = require("../../constants");
-const env = require("../../environment");
-const ScriptRunner = require("../../utilities/scriptRunner");
+const { Thread, ThreadType } = require("../../threads");
+const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 });
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
     const wasArray = Array.isArray(input);
@@ -26,38 +25,6 @@ function enrichQueries(input) {
     }
     return wasArray ? queries : queries[0];
 }
-function formatResponse(resp) {
-    if (typeof resp === "string") {
-        try {
-            resp = JSON.parse(resp);
-        }
-        catch (err) {
-            resp = { response: resp };
-        }
-    }
-    return resp;
-}
-function runAndTransform(integration, queryVerb, enrichedQuery, transformer) {
-    return __awaiter(this, void 0, void 0, function* () {
-        let rows = formatResponse(yield integration[queryVerb](enrichedQuery));
-        // transform as required
-        if (transformer) {
-            const runner = new ScriptRunner(transformer, { data: rows });
-            rows = runner.execute();
-        }
-        // needs to an array for next step
-        if (!Array.isArray(rows)) {
-            rows = [rows];
-        }
-        // map into JSON if just raw primitive here
-        if (rows.find(row => typeof row !== "object")) {
-            rows = rows.map(value => ({ value }));
-        }
-        // get all the potential fields in the schema
-        let keys = rows.flatMap(Object.keys);
-        return { rows, keys };
-    });
-}
 exports.fetch = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = new CouchDB(ctx.appId);
@@ -94,7 +61,9 @@ function enrichQueryFields(fields, parameters = {}) {
             }
             else if (typeof fields[key] === "string") {
                 // enrich string value as normal
-                enrichedQuery[key] = yield processString(fields[key], parameters);
+                enrichedQuery[key] = yield processString(fields[key], parameters, {
+                    noHelpers: true,
+                });
             }
             else {
                 enrichedQuery[key] = fields[key];
@@ -121,10 +90,9 @@ exports.find = function (ctx) {
         const db = new CouchDB(ctx.appId);
         const query = enrichQueries(yield db.get(ctx.params.queryId));
         // remove properties that could be dangerous in real app
-        if (env.isProd()) {
+        if (isProdAppID(ctx.appId)) {
             delete query.fields;
             delete query.parameters;
-            delete query.schema;
         }
         ctx.body = query;
     });
@@ -133,21 +101,22 @@ exports.preview = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = new CouchDB(ctx.appId);
         const datasource = yield db.get(ctx.request.body.datasourceId);
-        const Integration = integrations[datasource.source];
-        if (!Integration) {
-            ctx.throw(400, "Integration type does not exist.");
-        }
         const { fields, parameters, queryVerb, transformer } = ctx.request.body;
         const enrichedQuery = yield enrichQueryFields(fields, parameters);
-        const integration = new Integration(datasource.config);
-        const { rows, keys } = yield runAndTransform(integration, queryVerb, enrichedQuery, transformer);
-        ctx.body = {
-            rows,
-            schemaFields: [...new Set(keys)],
-        };
-        // cleanup
-        if (integration.end) {
-            integration.end();
+        try {
+            const { rows, keys } = yield Runner.run({
+                datasource,
+                queryVerb,
+                query: enrichedQuery,
+                transformer,
+            });
+            ctx.body = {
+                rows,
+                schemaFields: [...new Set(keys)],
+            };
+        }
+        catch (err) {
+            ctx.throw(400, err);
         }
     });
 };
@@ -156,18 +125,19 @@ exports.execute = function (ctx) {
         const db = new CouchDB(ctx.appId);
         const query = yield db.get(ctx.params.queryId);
         const datasource = yield db.get(query.datasourceId);
-        const Integration = integrations[datasource.source];
-        if (!Integration) {
-            ctx.throw(400, "Integration type does not exist.");
-        }
         const enrichedQuery = yield enrichQueryFields(query.fields, ctx.request.body.parameters);
-        const integration = new Integration(datasource.config);
         // call the relevant CRUD method on the integration class
-        const { rows } = yield runAndTransform(integration, query.queryVerb, enrichedQuery, query.transformer);
-        ctx.body = rows;
-        // cleanup
-        if (integration.end) {
-            integration.end();
+        try {
+            const { rows } = yield Runner.run({
+                datasource,
+                queryVerb: query.queryVerb,
+                query: enrichedQuery,
+                transformer: query.transformer,
+            });
+            ctx.body = rows;
+        }
+        catch (err) {
+            ctx.throw(400, err);
         }
     });
 };