@budibase/server 0.9.185-alpha.0 → 0.9.185-alpha.12

package/dist/integrations/base/sql.js

@@ -17,19 +17,46 @@ const datasource_1 = require("../../definitions/datasource");
 const utils_1 = require("../utils");
 const sqlTable_1 = __importDefault(require("./sqlTable"));
 const BASE_LIMIT = 5000;
+// these are invalid dates sent by the client, need to convert them to a real max date
+const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z";
+const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z";
+function parse(input) {
+    if (Array.isArray(input)) {
+        return JSON.stringify(input);
+    }
+    if (typeof input !== "string") {
+        return input;
+    }
+    if (input === MAX_ISO_DATE) {
+        return new Date(8640000000000000);
+    }
+    if (input === MIN_ISO_DATE) {
+        return new Date(-8640000000000000);
+    }
+    if (utils_1.isIsoDateString(input)) {
+        return new Date(input);
+    }
+    return input;
+}
 function parseBody(body) {
     for (let [key, value] of Object.entries(body)) {
-        if (Array.isArray(value)) {
-            body[key] = JSON.stringify(value);
-        }
-        if (typeof value !== "string") {
-            continue;
+        body[key] = parse(value);
+    }
+    return body;
+}
+function parseFilters(filters) {
+    for (let [key, value] of Object.entries(filters)) {
+        let parsed;
+        if (typeof value === "object") {
+            parsed = parseFilters(value);
         }
-        if (utils_1.isIsoDateString(value)) {
-            body[key] = new Date(value);
+        else {
+            parsed = parse(value);
         }
+        // @ts-ignore
+        filters[key] = parsed;
     }
-    return body;
+    return filters;
 }
 class InternalBuilder {
     constructor(client) {
@@ -45,6 +72,7 @@ class InternalBuilder {
         if (!filters) {
             return query;
         }
+        filters = parseFilters(filters);
         // if all or specified in filters, then everything is an or
         const allOr = filters.allOr;
         if (filters.oneOf) {
@@ -156,6 +184,15 @@ class InternalBuilder {
             return query.insert(parsedBody).returning("*");
         }
     }
+    bulkCreate(knex, json) {
+        const { endpoint, body } = json;
+        let query = knex(endpoint.entityId);
+        if (!Array.isArray(body)) {
+            return query;
+        }
+        const parsedBody = body.map(row => parseBody(row));
+        return query.insert(parsedBody);
+    }
     read(knex, json, limit) {
         var _a, _b;
         let { endpoint, resource, filters, sort, paginate, relationships } = json;
@@ -264,6 +301,9 @@ class SqlQueryBuilder extends sqlTable_1.default {
             case datasource_1.Operation.DELETE:
                 query = builder.delete(client, json, opts);
                 break;
+            case datasource_1.Operation.BULK_CREATE:
+                query = builder.bulkCreate(client, json);
+                break;
             case datasource_1.Operation.CREATE_TABLE:
             case datasource_1.Operation.UPDATE_TABLE:
             case datasource_1.Operation.DELETE_TABLE:

package/dist/integrations/base/sqlTable.js

@@ -22,8 +22,7 @@ function generateSchema(schema, table, tables, oldTable = null) {
     for (let [key, column] of Object.entries(table.schema)) {
         // skip things that are already correct
         const oldColumn = oldTable ? oldTable.schema[key] : null;
-        if ((oldColumn && oldColumn.type) ||
-            (primaryKey === key && !isJunction)) {
+        if ((oldColumn && oldColumn.type) || (primaryKey === key && !isJunction)) {
             continue;
         }
         switch (column.type) {

package/dist/integrations/mysql.js

@@ -147,7 +147,9 @@ var MySQLModule;
             });
         }
         read(query) {
-            return internalQuery(this.client, utils_1.getSqlQuery(query));
+            return __awaiter(this, void 0, void 0, function* () {
+                return internalQuery(this.client, utils_1.getSqlQuery(query));
+            });
         }
         update(query) {
             return __awaiter(this, void 0, void 0, function* () {

package/dist/integrations/postgres.js

@@ -135,9 +135,9 @@ var PostgresModule;
         }
         setSchema() {
             if (!this.config.schema) {
-                this.config.schema = 'public';
+                this.config.schema = "public";
             }
-            this.client.on('connect', (client) => {
+            this.client.on("connect", (client) => {
                 client.query(`SET search_path TO ${this.config.schema}`);
             });
             this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`;

package/dist/middleware/builder.js

@@ -13,6 +13,7 @@ const { doesUserHaveLock, updateLock, checkDebounce, setDebounce, } = require(".
 const CouchDB = require("../db");
 const { DocumentTypes } = require("../db/utils");
 const { PermissionTypes } = require("@budibase/auth/permissions");
+const { app: appCache } = require("@budibase/auth/cache");
 const DEBOUNCE_TIME_SEC = 30;
 /************************************************** *
  * This middleware has been broken out of the       *
@@ -51,7 +52,9 @@ function updateAppUpdatedAt(ctx) {
         const db = new CouchDB(appId);
         const metadata = yield db.get(DocumentTypes.APP_METADATA);
         metadata.updatedAt = new Date().toISOString();
-        yield db.put(metadata);
+        const response = yield db.put(metadata);
+        metadata._rev = response.rev;
+        yield appCache.invalidateAppMetadata(appId, metadata);
         // set a new debounce record with a short TTL
         yield setDebounce(appId, DEBOUNCE_TIME_SEC);
     });

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "0.9.184",
+    "version": "0.9.185-alpha.11",
     "description": "Budibase Web Server",
     "main": "src/index.js",
     "repository": {
@@ -68,9 +68,11 @@
     "author": "Budibase",
     "license": "AGPL-3.0-or-later",
     "dependencies": {
-        "@budibase/auth": "^0.9.184",
-        "@budibase/client": "^0.9.184",
-        "@budibase/string-templates": "^0.9.184",
+        "@budibase/auth": "^0.9.185-alpha.11",
+        "@budibase/client": "^0.9.185-alpha.11",
+        "@budibase/string-templates": "^0.9.185-alpha.11",
+        "@bull-board/api": "^3.7.0",
+        "@bull-board/koa": "^3.7.0",
         "@elastic/elasticsearch": "7.10.0",
         "@koa/router": "8.0.0",
         "@sendgrid/mail": "7.1.1",
@@ -80,7 +82,6 @@
         "aws-sdk": "^2.767.0",
         "bcryptjs": "2.4.3",
         "bull": "^3.22.4",
-        "bull-board": "^2.0.1",
         "chmodr": "1.2.0",
         "csvtojson": "2.0.10",
         "dotenv": "8.2.0",
@@ -120,6 +121,7 @@
         "uuid": "3.3.2",
         "validate.js": "0.13.1",
         "vm2": "^3.9.3",
+        "worker-farm": "^1.7.0",
         "yargs": "13.2.4",
         "zlib": "1.0.5"
     },
@@ -138,7 +140,6 @@
         "copyfiles": "^2.4.1",
         "docker-compose": "^0.23.6",
         "eslint": "^6.8.0",
-        "express": "^4.17.1",
         "jest": "^27.0.5",
         "nodemon": "^2.0.4",
         "prettier": "^2.3.1",

package/dist/{automations/thread.js → threads/automation.js}

@@ -8,8 +8,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const actions = require("./actions");
-const automationUtils = require("./automationUtils");
+const actions = require("../automations/actions");
+const automationUtils = require("../automations/automationUtils");
 const AutomationEmitter = require("../events/AutomationEmitter");
 const { processObject } = require("@budibase/string-templates");
 const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants;
@@ -18,7 +18,9 @@ const { DocumentTypes, isDevAppID } = require("../db/utils");
 const { doInTenant } = require("@budibase/auth/tenancy");
 const env = require("../environment");
 const usage = require("../utilities/usageQuota");
+const { definitions: triggerDefs } = require("../automations/triggerInfo");
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId;
+const CRON_STEP_ID = triggerDefs.CRON.stepId;
 const STOPPED_STATUS = { success: false, status: "STOPPED" };
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -31,6 +33,8 @@ class Orchestrator {
         this._chainCount = this._metadata ? this._metadata.automationChainCount : 0;
         this._appId = triggerOutput.appId;
         this._app = null;
+        const triggerStepId = automation.definition.trigger.stepId;
+        triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput);
         // remove from context
         delete triggerOutput.appId;
         delete triggerOutput.metadata;
@@ -42,10 +46,15 @@ class Orchestrator {
         this._emitter = new AutomationEmitter(this._chainCount + 1);
         this.executionOutput = { trigger: {}, steps: [] };
         // setup the execution output
-        const triggerStepId = automation.definition.trigger.stepId;
         const triggerId = automation.definition.trigger.id;
         this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput);
     }
+    cleanupTriggerOutputs(stepId, triggerOutput) {
+        if (stepId === CRON_STEP_ID) {
+            triggerOutput.timestamp = Date.now();
+        }
+        return triggerOutput;
+    }
     getStepFunctionality(stepId) {
         return __awaiter(this, void 0, void 0, function* () {
             let step = yield actions.getAction(stepId);
@@ -122,7 +131,14 @@ class Orchestrator {
         });
     }
 }
-module.exports = (job) => __awaiter(void 0, void 0, void 0, function* () {
-    const automationOrchestrator = new Orchestrator(job.data.automation, job.data.event);
-    return automationOrchestrator.execute();
-});
+module.exports = (input, callback) => {
+    const automationOrchestrator = new Orchestrator(input.data.automation, input.data.event);
+    automationOrchestrator
+        .execute()
+        .then(response => {
+        callback(null, response);
+    })
+        .catch(err => {
+        callback(err);
+    });
+};

package/dist/threads/index.js

@@ -0,0 +1,58 @@
+"use strict";
+const workerFarm = require("worker-farm");
+const env = require("../environment");
+const ThreadType = {
+    QUERY: "query",
+    AUTOMATION: "automation",
+};
+function typeToFile(type) {
+    let filename = null;
+    switch (type) {
+        case ThreadType.QUERY:
+            filename = "./query";
+            break;
+        case ThreadType.AUTOMATION:
+            filename = "./automation";
+            break;
+        default:
+            throw "Unknown thread type";
+    }
+    return require.resolve(filename);
+}
+class Thread {
+    constructor(type, opts = { timeoutMs: null, count: 1 }) {
+        this.type = type;
+        if (!env.isTest()) {
+            const workerOpts = {
+                autoStart: true,
+                maxConcurrentWorkers: opts.count ? opts.count : 1,
+            };
+            if (opts.timeoutMs) {
+                workerOpts.maxCallTime = opts.timeoutMs;
+            }
+            this.workers = workerFarm(workerOpts, typeToFile(type));
+        }
+    }
+    run(data) {
+        return new Promise((resolve, reject) => {
+            let fncToCall;
+            // if in test then don't use threading
+            if (env.isTest()) {
+                fncToCall = require(typeToFile(this.type));
+            }
+            else {
+                fncToCall = this.workers;
+            }
+            fncToCall(data, (err, response) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve(response);
+                }
+            });
+        });
+    }
+}
+module.exports.Thread = Thread;
+module.exports.ThreadType = ThreadType;

package/dist/threads/query.js

@@ -0,0 +1,61 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const ScriptRunner = require("../utilities/scriptRunner");
+const { integrations } = require("../integrations");
+function formatResponse(resp) {
+    if (typeof resp === "string") {
+        try {
+            resp = JSON.parse(resp);
+        }
+        catch (err) {
+            resp = { response: resp };
+        }
+    }
+    return resp;
+}
+function runAndTransform(datasource, queryVerb, query, transformer) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const Integration = integrations[datasource.source];
+        if (!Integration) {
+            throw "Integration type does not exist.";
+        }
+        const integration = new Integration(datasource.config);
+        let rows = formatResponse(yield integration[queryVerb](query));
+        // transform as required
+        if (transformer) {
+            const runner = new ScriptRunner(transformer, { data: rows });
+            rows = runner.execute();
+        }
+        // needs to an array for next step
+        if (!Array.isArray(rows)) {
+            rows = [rows];
+        }
+        // map into JSON if just raw primitive here
+        if (rows.find(row => typeof row !== "object")) {
+            rows = rows.map(value => ({ value }));
+        }
+        // get all the potential fields in the schema
+        let keys = rows.flatMap(Object.keys);
+        if (integration.end) {
+            integration.end();
+        }
+        return { rows, keys };
+    });
+}
+module.exports = (input, callback) => {
+    runAndTransform(input.datasource, input.queryVerb, input.query, input.transformer)
+        .then(response => {
+        callback(null, response);
+    })
+        .catch(err => {
+        callback(err);
+    });
+};