@budibase/server 0.9.184 → 0.9.185-alpha.11

package/dist/api/controllers/table/utils.js

@@ -14,7 +14,7 @@ const { getRowParams, generateRowID, InternalTables, } = require("../../../db/ut
 const { isEqual } = require("lodash/fp");
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants");
 const { inputProcessing } = require("../../../utilities/rowProcessor");
-const { USERS_TABLE_SCHEMA } = require("../../../constants");
+const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants");
 const { isExternalTable, breakExternalTableId, } = require("../../../integrations/utils");
 const { getViews, saveView } = require("../view/utils");
 const viewTemplate = require("../view/viewBuilder");
@@ -66,37 +66,38 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
     return tableToSave;
 };
 exports.handleDataImport = (appId, user, table, dataImport) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!dataImport || !dataImport.csvString) {
+        return table;
+    }
     const db = new CouchDB(appId);
-    if (dataImport && dataImport.csvString) {
-        // Populate the table with rows imported from CSV in a bulk update
-        const data = yield csvParser.transform(dataImport);
-        let finalData = [];
-        for (let i = 0; i < data.length; i++) {
-            let row = data[i];
-            row._id = generateRowID(table._id);
-            row.tableId = table._id;
-            const processed = inputProcessing(user, table, row, {
-                noAutoRelationships: true,
-            });
-            table = processed.table;
-            row = processed.row;
-            for (let [fieldName, schema] of Object.entries(table.schema)) {
-                // check whether the options need to be updated for inclusion as part of the data import
-                if (schema.type === FieldTypes.OPTIONS &&
-                    (!schema.constraints.inclusion ||
-                        schema.constraints.inclusion.indexOf(row[fieldName]) === -1)) {
-                    schema.constraints.inclusion = [
-                        ...schema.constraints.inclusion,
-                        row[fieldName],
-                    ];
-                }
+    // Populate the table with rows imported from CSV in a bulk update
+    const data = yield csvParser.transform(Object.assign(Object.assign({}, dataImport), { existingTable: table }));
+    let finalData = [];
+    for (let i = 0; i < data.length; i++) {
+        let row = data[i];
+        row._id = generateRowID(table._id);
+        row.tableId = table._id;
+        const processed = inputProcessing(user, table, row, {
+            noAutoRelationships: true,
+        });
+        table = processed.table;
+        row = processed.row;
+        for (let [fieldName, schema] of Object.entries(table.schema)) {
+            // check whether the options need to be updated for inclusion as part of the data import
+            if (schema.type === FieldTypes.OPTIONS &&
+                (!schema.constraints.inclusion ||
+                    schema.constraints.inclusion.indexOf(row[fieldName]) === -1)) {
+                schema.constraints.inclusion = [
+                    ...schema.constraints.inclusion,
+                    row[fieldName],
+                ];
             }
-            finalData.push(row);
         }
-        yield db.bulkDocs(finalData);
-        let response = yield db.put(table);
-        table._rev = response._rev;
+        finalData.push(row);
     }
+    yield db.bulkDocs(finalData);
+    let response = yield db.put(table);
+    table._rev = response._rev;
     return table;
 });
 exports.handleSearchIndexes = (appId, table) => __awaiter(void 0, void 0, void 0, function* () {
@@ -277,4 +278,37 @@ exports.checkForViewUpdates = (db, table, rename, deletedColumns) => __awaiter(v
         }
     }
 });
+exports.generateForeignKey = (column, relatedTable) => {
+    return `fk_${relatedTable.name}_${column.fieldName}`;
+};
+exports.generateJunctionTableName = (column, table, relatedTable) => {
+    return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`;
+};
+exports.foreignKeyStructure = (keyName, meta = null) => {
+    const structure = {
+        type: FieldTypes.NUMBER,
+        constraints: {},
+        name: keyName,
+    };
+    if (meta) {
+        structure.meta = meta;
+    }
+    return structure;
+};
+exports.hasTypeChanged = (table, oldTable) => {
+    if (!oldTable) {
+        return false;
+    }
+    for (let [key, field] of Object.entries(oldTable.schema)) {
+        const oldType = field.type;
+        if (!table.schema[key]) {
+            continue;
+        }
+        const newType = table.schema[key].type;
+        if (oldType !== newType && SwitchableTypes.indexOf(oldType) === -1) {
+            return true;
+        }
+    }
+    return false;
+};
 exports.TableSaveFunctions = TableSaveFunctions;

package/dist/api/controllers/user.js

@@ -11,22 +11,132 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 const CouchDB = require("../../db");
 const { generateUserMetadataID, getUserMetadataParams, } = require("../../db/utils");
 const { InternalTables } = require("../../db/utils");
-const { getGlobalUsers } = require("../../utilities/global");
+const { getGlobalUsers, getRawGlobalUser } = require("../../utilities/global");
 const { getFullUser } = require("../../utilities/users");
-function removeGlobalProps(user) {
-    // make sure to always remove some of the global user props
-    delete user.password;
-    delete user.roles;
-    delete user.builder;
-    return user;
+const { isEqual } = require("lodash");
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles");
+const { getDevelopmentAppID, getDeployedAppIDs } = require("@budibase/auth/db");
+const { doesDatabaseExist } = require("../../utilities");
+const { UserStatus } = require("@budibase/auth/constants");
+function rawMetadata(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return (yield db.allDocs(getUserMetadataParams(null, {
+            include_docs: true,
+        }))).rows.map(row => row.doc);
+    });
+}
+function combineMetadataAndUser(user, metadata) {
+    // skip users with no access
+    if (user.roleId === BUILTIN_ROLE_IDS.PUBLIC) {
+        return null;
+    }
+    delete user._rev;
+    const metadataId = generateUserMetadataID(user._id);
+    const newDoc = Object.assign(Object.assign({}, user), { _id: metadataId, tableId: InternalTables.USER_METADATA });
+    const found = Array.isArray(metadata)
+        ? metadata.find(doc => doc._id === metadataId)
+        : metadata;
+    // copy rev over for the purposes of equality check
+    if (found) {
+        newDoc._rev = found._rev;
+    }
+    if (found == null || !isEqual(newDoc, found)) {
+        return Object.assign(Object.assign({}, found), newDoc);
+    }
+    return null;
 }
+exports.syncGlobalUsers = (appId) => __awaiter(void 0, void 0, void 0, function* () {
+    // sync user metadata
+    const db = new CouchDB(appId);
+    const [users, metadata] = yield Promise.all([
+        getGlobalUsers(appId),
+        rawMetadata(db),
+    ]);
+    const toWrite = [];
+    for (let user of users) {
+        const combined = yield combineMetadataAndUser(user, metadata);
+        if (combined) {
+            toWrite.push(combined);
+        }
+    }
+    yield db.bulkDocs(toWrite);
+});
+exports.syncUser = function (ctx) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let deleting = false, user;
+        const userId = ctx.params.id;
+        try {
+            user = yield getRawGlobalUser(userId);
+        }
+        catch (err) {
+            if (err && err.status === 404) {
+                user = {};
+                deleting = true;
+            }
+            else {
+                throw err;
+            }
+        }
+        const roles = user.roles;
+        // remove props which aren't useful to metadata
+        delete user.password;
+        delete user.forceResetPassword;
+        delete user.roles;
+        // run through all production appIDs in the users roles
+        let prodAppIds;
+        // if they are a builder then get all production app IDs
+        if ((user.builder && user.builder.global) || deleting) {
+            prodAppIds = yield getDeployedAppIDs(CouchDB);
+        }
+        else {
+            prodAppIds = Object.entries(roles)
+                .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC)
+                .map(([appId]) => appId);
+        }
+        for (let prodAppId of prodAppIds) {
+            const roleId = roles[prodAppId];
+            const devAppId = getDevelopmentAppID(prodAppId);
+            for (let appId of [prodAppId, devAppId]) {
+                if (!(yield doesDatabaseExist(appId))) {
+                    continue;
+                }
+                const db = new CouchDB(appId);
+                const metadataId = generateUserMetadataID(userId);
+                let metadata;
+                try {
+                    metadata = yield db.get(metadataId);
+                }
+                catch (err) {
+                    if (deleting) {
+                        continue;
+                    }
+                    metadata = {
+                        tableId: InternalTables.USER_METADATA,
+                    };
+                }
+                // assign the roleId for the metadata doc
+                if (roleId) {
+                    metadata.roleId = roleId;
+                }
+                let combined = !deleting
+                    ? combineMetadataAndUser(user, metadata)
+                    : Object.assign(Object.assign({}, metadata), { status: UserStatus.INACTIVE, metadata: BUILTIN_ROLE_IDS.PUBLIC });
+                // if its null then there was no updates required
+                if (combined) {
+                    yield db.put(combined);
+                }
+            }
+        }
+        ctx.body = {
+            message: "User synced.",
+        };
+    });
+};
 exports.fetchMetadata = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const database = new CouchDB(ctx.appId);
         const global = yield getGlobalUsers(ctx.appId);
-        const metadata = (yield database.allDocs(getUserMetadataParams(null, {
-            include_docs: true,
-        }))).rows.map(row => row.doc);
+        const metadata = yield rawMetadata(database);
         const users = [];
         for (let user of global) {
             // find the metadata that matches up to the global ID
@@ -52,7 +162,9 @@ exports.updateMetadata = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const appId = ctx.appId;
         const db = new CouchDB(appId);
-        const user = removeGlobalProps(ctx.request.body);
+        const user = ctx.request.body;
+        // this isn't applicable to the user
+        delete user.roles;
         const metadata = Object.assign({ tableId: InternalTables.USER_METADATA }, user);
         ctx.body = yield db.put(metadata);
     });

package/dist/api/routes/query.js

@@ -41,7 +41,7 @@ router
     .get("/api/queries", authorized(BUILDER), queryController.fetch)
     .post("/api/queries", bodySubResource("datasourceId", "_id"), authorized(BUILDER), generateQueryValidation(), queryController.save)
     .post("/api/queries/preview", bodyResource("datasourceId"), authorized(BUILDER), generateQueryPreviewValidation(), queryController.preview)
-    .get("/api/queries/:queryId", authorized(PermissionTypes.QUERY, PermissionLevels.READ), queryController.find)
+    .get("/api/queries/:queryId", paramResource("queryId"), authorized(PermissionTypes.QUERY, PermissionLevels.READ), queryController.find)
     .post("/api/queries/:queryId", paramResource("queryId"), authorized(PermissionTypes.QUERY, PermissionLevels.WRITE), queryController.execute)
     .delete("/api/queries/:queryId/:revId", paramResource("queryId"), authorized(BUILDER), queryController.destroy);
 module.exports = router;

package/dist/api/routes/table.js

@@ -27,5 +27,11 @@ router
 // allows control over updating a table
 bodyResource("_id"), authorized(BUILDER), generateSaveValidator(), tableController.save)
     .post("/api/tables/csv/validate", authorized(BUILDER), tableController.validateCSVSchema)
-    .delete("/api/tables/:tableId/:revId", paramResource("tableId"), authorized(BUILDER), tableController.destroy);
+    .delete("/api/tables/:tableId/:revId", paramResource("tableId"), authorized(BUILDER), tableController.destroy)
+    // this is currently builder only, but in the future
+    // it could be carried out by an end user in app,
+    // however some thought will need to be had about
+    // implications for automations (triggers)
+    // new trigger type, bulk rows created
+    .post("/api/tables/:tableId/import", paramResource("tableId"), authorized(BUILDER), tableController.bulkImport);
 module.exports = router;

package/dist/api/routes/user.js

@@ -9,5 +9,6 @@ router
     .get("/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.READ), controller.findMetadata)
     .put("/api/users/metadata", authorized(PermissionTypes.USER, PermissionLevels.WRITE), controller.updateMetadata)
     .post("/api/users/metadata/self", authorized(PermissionTypes.USER, PermissionLevels.WRITE), controller.updateSelfMetadata)
-    .delete("/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), controller.destroyMetadata);
+    .delete("/api/users/metadata/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), controller.destroyMetadata)
+    .post("/api/users/metadata/sync/:id", authorized(PermissionTypes.USER, PermissionLevels.WRITE), controller.syncUser);
 module.exports = router;

package/dist/app.js

@@ -41,17 +41,8 @@ app.use(pino({
     level: env.LOG_LEVEL || "error",
 }));
 if (!env.isTest()) {
-    const bullApp = bullboard.init();
-    app.use((ctx, next) => __awaiter(void 0, void 0, void 0, function* () {
-        if (ctx.path.startsWith(bullboard.pathPrefix)) {
-            ctx.status = 200;
-            ctx.respond = false;
-            bullApp(ctx.req, ctx.res);
-        }
-        else {
-            yield next();
-        }
-    }));
+    const plugin = bullboard.init();
+    app.use(plugin);
 }
 app.context.eventEmitter = eventEmitter;
 app.context.auth = {};

package/dist/automations/bullboard.js

@@ -1,7 +1,16 @@
 "use strict";
-const { createBullBoard } = require("bull-board");
-const { BullAdapter } = require("bull-board/bullAdapter");
-const express = require("express");
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { createBullBoard } = require("@bull-board/api");
+const { BullAdapter } = require("@bull-board/api/bullAdapter");
+const { KoaAdapter } = require("@bull-board/koa");
 const env = require("../environment");
 const Queue = env.isTest()
     ? require("../utilities/queue/inMemoryQueue")
@@ -9,19 +18,37 @@ const Queue = env.isTest()
 const { JobQueues } = require("../constants");
 const { utils } = require("@budibase/auth/redis");
 const { opts, redisProtocolUrl } = utils.getRedisOptions();
-const redisConfig = redisProtocolUrl || { redis: opts };
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, redisConfig);
-exports.pathPrefix = "/bulladmin";
+const CLEANUP_PERIOD_MS = 60 * 1000;
+const queueConfig = redisProtocolUrl || { redis: opts };
+let cleanupInternal = null;
+let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig);
+function cleanup() {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield automationQueue.clean(CLEANUP_PERIOD_MS, "completed");
+    });
+}
+const PATH_PREFIX = "/bulladmin";
 exports.init = () => {
-    const expressApp = express();
+    // cleanup the events every 5 minutes
+    if (!cleanupInternal) {
+        cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS);
+        // fire off an initial cleanup
+        cleanup().catch(err => {
+            console.error(`Unable to cleanup automation queue initially - ${err}`);
+        });
+    }
     // Set up queues for bull board admin
     const queues = [automationQueue];
     const adapters = [];
+    const serverAdapter = new KoaAdapter();
     for (let queue of queues) {
         adapters.push(new BullAdapter(queue));
     }
-    const { router } = createBullBoard(adapters);
-    expressApp.use(exports.pathPrefix, router);
-    return expressApp;
+    createBullBoard({
+        queues: adapters,
+        serverAdapter,
+    });
+    serverAdapter.setBasePath(PATH_PREFIX);
+    return serverAdapter.registerPlugin();
 };
 exports.queue = automationQueue;

package/dist/automations/triggers.js

@@ -20,6 +20,10 @@ const { checkTestFlag } = require("../utilities/redis");
 const utils = require("./utils");
 const env = require("../environment");
 const TRIGGER_DEFINITIONS = definitions;
+const JOB_OPTS = {
+    removeOnComplete: true,
+    removeOnFail: true,
+};
 function queueRelevantRowAutomations(event, eventType) {
     return __awaiter(this, void 0, void 0, function* () {
         if (event.appId == null) {
@@ -47,7 +51,7 @@ function queueRelevantRowAutomations(event, eventType) {
             }
             if (automationTrigger.inputs &&
                 automationTrigger.inputs.tableId === event.row.tableId) {
-                yield queue.add({ automation, event });
+                yield queue.add({ automation, event }, JOB_OPTS);
             }
         }
     });
@@ -85,7 +89,7 @@ exports.externalTrigger = function (automation, params, { getResponses } = {}) {
             automation.definition.trigger != null &&
             automation.definition.trigger.stepId === definitions.APP.stepId &&
             automation.definition.trigger.stepId === "APP" &&
-            !checkTestFlag(automation._id)) {
+            !(yield checkTestFlag(automation._id))) {
             // values are likely to be submitted as strings, so we shall convert to correct type
             const coercedFields = {};
             const fields = automation.definition.trigger.inputs.fields;
@@ -99,7 +103,7 @@ exports.externalTrigger = function (automation, params, { getResponses } = {}) {
             return utils.processEvent({ data });
         }
         else {
-            return queue.add(data);
+            return queue.add(data, JOB_OPTS);
         }
     });
 };

package/dist/automations/utils.js

@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const runner = require("./thread");
+const { Thread, ThreadType } = require("../threads");
 const { definitions } = require("./triggerInfo");
 const webhooks = require("../api/controllers/webhook");
 const CouchDB = require("../db");
@@ -19,10 +19,11 @@ const { MetadataTypes } = require("../constants");
 const { getDeployedAppID } = require("@budibase/auth/db");
 const WH_STEP_ID = definitions.WEBHOOK.stepId;
 const CRON_STEP_ID = definitions.CRON.stepId;
+const Runner = new Thread(ThreadType.AUTOMATION);
 exports.processEvent = (job) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         // need to actually await these so that an error can be captured properly
-        return yield runner(job);
+        return yield Runner.run(job);
     }
     catch (err) {
         console.error(`${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`);
@@ -146,3 +147,11 @@ exports.checkForWebhooks = ({ appId, oldAuto, newAuto }) => __awaiter(void 0, vo
     }
     return newAuto;
 });
+/**
+ * When removing an app/unpublishing it need to make sure automations are cleaned up (cron).
+ * @param appId {string} the app that is being removed.
+ * @return {Promise<void>} clean is complete if this succeeds.
+ */
+exports.cleanupAutomations = (appId) => __awaiter(void 0, void 0, void 0, function* () {
+    yield exports.disableAllCrons(appId);
+});

package/dist/constants/index.js

@@ -41,6 +41,12 @@ exports.FieldTypes = {
     JSON: "json",
     INTERNAL: "internal",
 };
+exports.SwitchableTypes = [
+    exports.FieldTypes.STRING,
+    exports.FieldTypes.OPTIONS,
+    exports.FieldTypes.NUMBER,
+    exports.FieldTypes.BOOLEAN,
+];
 exports.RelationshipTypes = {
     ONE_TO_MANY: "one-to-many",
     MANY_TO_ONE: "many-to-one",
@@ -56,6 +62,10 @@ exports.DataSourceOperation = {
     READ: "READ",
     UPDATE: "UPDATE",
     DELETE: "DELETE",
+    BULK_CREATE: "BULK_CREATE",
+    CREATE_TABLE: "CREATE_TABLE",
+    UPDATE_TABLE: "UPDATE_TABLE",
+    DELETE_TABLE: "DELETE_TABLE",
 };
 exports.SortDirection = {
     ASCENDING: "ASCENDING",
@@ -140,5 +150,8 @@ exports.MetadataTypes = {
     AUTOMATION_TEST_INPUT: "automationTestInput",
     AUTOMATION_TEST_HISTORY: "automationTestHistory",
 };
+exports.BuildSchemaErrors = {
+    NO_KEY: "no_key",
+};
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets;

package/dist/definitions/datasource.js

@@ -7,6 +7,10 @@ var Operation;
     Operation["READ"] = "READ";
     Operation["UPDATE"] = "UPDATE";
     Operation["DELETE"] = "DELETE";
+    Operation["BULK_CREATE"] = "BULK_CREATE";
+    Operation["CREATE_TABLE"] = "CREATE_TABLE";
+    Operation["UPDATE_TABLE"] = "UPDATE_TABLE";
+    Operation["DELETE_TABLE"] = "DELETE_TABLE";
 })(Operation = exports.Operation || (exports.Operation = {}));
 var SortDirection;
 (function (SortDirection) {

package/dist/integrations/base/IntegrationBase.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/integrations/base/datasourcePlus.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });