@budibase/server 1.0.35 → 1.0.39

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "1.0.34",
+    "version": "1.0.38",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -69,9 +69,10 @@
     "author": "Budibase",
     "license": "GPL-3.0",
     "dependencies": {
-        "@budibase/auth": "^1.0.34",
-        "@budibase/client": "^1.0.34",
-        "@budibase/string-templates": "^1.0.34",
+        "@apidevtools/swagger-parser": "^10.0.3",
+        "@budibase/backend-core": "^1.0.38",
+        "@budibase/client": "^1.0.38",
+        "@budibase/string-templates": "^1.0.38",
         "@bull-board/api": "^3.7.0",
         "@bull-board/koa": "^3.7.0",
         "@elastic/elasticsearch": "7.10.0",
@@ -85,12 +86,15 @@
         "bull": "^3.22.4",
         "chmodr": "1.2.0",
         "csvtojson": "2.0.10",
+        "curlconverter": "^3.21.0",
         "dotenv": "8.2.0",
         "download": "8.0.0",
         "fix-path": "3.0.0",
+        "form-data": "^4.0.0",
         "fs-extra": "8.1.0",
         "jimp": "0.16.1",
         "joi": "17.2.1",
+        "js-yaml": "^4.1.0",
         "jsonschema": "1.4.0",
         "knex": "^0.95.6",
         "koa": "2.7.0",
@@ -118,11 +122,13 @@
         "pouchdb-replication-stream": "1.2.9",
         "server-destroy": "1.0.1",
         "svelte": "^3.38.2",
+        "swagger-parser": "^10.0.3",
         "to-json-schema": "0.2.5",
         "uuid": "3.3.2",
         "validate.js": "0.13.1",
         "vm2": "^3.9.3",
         "worker-farm": "^1.7.0",
+        "xml2js": "^0.4.23",
         "yargs": "13.2.4",
         "zlib": "1.0.5"
     },
@@ -146,6 +152,7 @@
         "eslint": "^6.8.0",
         "jest": "^27.0.5",
         "nodemon": "^2.0.4",
+        "openapi-types": "^9.3.1",
         "path-to-regexp": "^6.2.0",
         "prettier": "^2.3.1",
         "rimraf": "^3.0.2",

package/dist/tests/utilities/TestConfiguration.js

@@ -8,20 +8,20 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles");
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles");
 const env = require("../../environment");
 const { basicTable, basicRow, basicRole, basicAutomation, basicDatasource, basicQuery, basicScreen, basicLayout, basicWebhook, TENANT_ID, } = require("./structures");
 const controllers = require("./controllers");
 const supertest = require("supertest");
 const { cleanup } = require("../../utilities/fileSystem");
-const { Cookies, Headers } = require("@budibase/auth").constants;
-const { jwt } = require("@budibase/auth").auth;
-const auth = require("@budibase/auth");
-const { getGlobalDB } = require("@budibase/auth/tenancy");
-const { createASession } = require("@budibase/auth/sessions");
-const { user: userCache } = require("@budibase/auth/cache");
+const { Cookies, Headers } = require("@budibase/backend-core/constants");
+const { jwt } = require("@budibase/backend-core/auth");
+const core = require("@budibase/backend-core");
+const { getGlobalDB } = require("@budibase/backend-core/tenancy");
+const { createASession } = require("@budibase/backend-core/sessions");
+const { user: userCache } = require("@budibase/backend-core/cache");
 const CouchDB = require("../../db");
-auth.init(CouchDB);
+core.init(CouchDB);
 const GLOBAL_USER_ID = "us_uuid1";
 const EMAIL = "babs@babs.com";
 class TestConfiguration {
@@ -294,6 +294,53 @@ class TestConfiguration {
             return this.datasource;
         });
     }
+    updateDatasource(datasource) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this._req(datasource, { datasourceId: datasource._id }, controllers.datasource.update);
+            this.datasource = response.datasource;
+            return this.datasource;
+        });
+    }
+    restDatasource(cfg) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.createDatasource({
+                datasource: Object.assign(Object.assign({}, basicDatasource().datasource), { source: "REST", config: cfg || {} }),
+            });
+        });
+    }
+    dynamicVariableDatasource() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let datasource = yield this.restDatasource();
+            const basedOnQuery = yield this.createQuery(Object.assign(Object.assign({}, basicQuery(datasource._id)), { fields: {
+                    path: "www.google.com",
+                } }));
+            datasource = yield this.updateDatasource(Object.assign(Object.assign({}, datasource), { config: {
+                    dynamicVariables: [
+                        {
+                            queryId: basedOnQuery._id,
+                            name: "variable3",
+                            value: "{{ data.0.[value] }}",
+                        },
+                    ],
+                } }));
+            return { datasource, query: basedOnQuery };
+        });
+    }
+    previewQuery(request, config, datasource, fields) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return request
+                .post(`/api/queries/preview`)
+                .send({
+                datasourceId: datasource._id,
+                parameters: {},
+                fields,
+                queryVerb: "read",
+            })
+                .set(config.defaultHeaders())
+                .expect("Content-Type", /json/)
+                .expect(200);
+        });
+    }
     createQuery(config = null) {
         return __awaiter(this, void 0, void 0, function* () {
             if (!this.datasource && !config) {

package/dist/tests/utilities/structures.js

@@ -1,6 +1,6 @@
 "use strict";
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles");
-const { BUILTIN_PERMISSION_IDS } = require("@budibase/auth/permissions");
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles");
+const { BUILTIN_PERMISSION_IDS } = require("@budibase/backend-core/permissions");
 const { createHomeScreen } = require("../../constants/screens");
 const { EMPTY_LAYOUT } = require("../../constants/layouts");
 const { cloneDeep } = require("lodash/fp");

package/dist/threads/automation.js

@@ -14,10 +14,10 @@ const actions = require("../automations/actions");
 const automationUtils = require("../automations/automationUtils");
 const AutomationEmitter = require("../events/AutomationEmitter");
 const { processObject } = require("@budibase/string-templates");
-const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants;
+const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants");
 const CouchDB = require("../db");
 const { DocumentTypes, isDevAppID } = require("../db/utils");
-const { doInTenant } = require("@budibase/auth/tenancy");
+const { doInTenant } = require("@budibase/backend-core/tenancy");
 const usage = require("../utilities/usageQuota");
 const { definitions: triggerDefs } = require("../automations/triggerInfo");
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId;

package/dist/threads/query.js

@@ -8,51 +8,208 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-require("./utils").threadSetup();
+const threadUtils = require("./utils");
+threadUtils.threadSetup();
 const ScriptRunner = require("../utilities/scriptRunner");
 const { integrations } = require("../integrations");
-function formatResponse(resp) {
-    if (typeof resp === "string") {
-        try {
-            resp = JSON.parse(resp);
-        }
-        catch (err) {
-            resp = { response: resp };
-        }
+const { processStringSync } = require("@budibase/string-templates");
+const CouchDB = require("../db");
+const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/);
+const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/);
+class QueryRunner {
+    constructor(input, flags = { noRecursiveQuery: false }) {
+        this.appId = input.appId;
+        this.datasource = input.datasource;
+        this.queryVerb = input.queryVerb;
+        this.fields = input.fields;
+        this.parameters = input.parameters;
+        this.pagination = input.pagination;
+        this.transformer = input.transformer;
+        this.queryId = input.queryId;
+        this.noRecursiveQuery = flags.noRecursiveQuery;
+        this.cachedVariables = [];
+        // allows the response from a query to be stored throughout this
+        // execution so that if it needs to be re-used for another variable
+        // it can be
+        this.queryResponse = {};
+        this.hasRerun = false;
     }
-    return resp;
-}
-function runAndTransform(datasource, queryVerb, query, transformer) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const Integration = integrations[datasource.source];
-        if (!Integration) {
-            throw "Integration type does not exist.";
-        }
-        const integration = new Integration(datasource.config);
-        let rows = formatResponse(yield integration[queryVerb](query));
-        // transform as required
-        if (transformer) {
-            const runner = new ScriptRunner(transformer, { data: rows });
-            rows = runner.execute();
-        }
-        // needs to an array for next step
-        if (!Array.isArray(rows)) {
-            rows = [rows];
-        }
-        // map into JSON if just raw primitive here
-        if (rows.find(row => typeof row !== "object")) {
-            rows = rows.map(value => ({ value }));
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { datasource, fields, queryVerb, transformer } = this;
+            // pre-query, make sure datasource variables are added to parameters
+            const parameters = yield this.addDatasourceVariables();
+            let query = this.enrichQueryFields(fields, parameters);
+            // Add pagination values for REST queries
+            if (this.pagination) {
+                query.paginationValues = this.pagination;
+            }
+            const Integration = integrations[datasource.source];
+            if (!Integration) {
+                throw "Integration type does not exist.";
+            }
+            const integration = new Integration(datasource.config);
+            let output = threadUtils.formatResponse(yield integration[queryVerb](query));
+            let rows = output, info = undefined, extra = undefined, pagination = undefined;
+            if (threadUtils.hasExtraData(output)) {
+                rows = output.data;
+                info = output.info;
+                extra = output.extra;
+                pagination = output.pagination;
+            }
+            // transform as required
+            if (transformer) {
+                const runner = new ScriptRunner(transformer, {
+                    data: rows,
+                    params: parameters,
+                });
+                rows = runner.execute();
+            }
+            // if the request fails we retry once, invalidating the cached value
+            if (info &&
+                info.code >= 400 &&
+                this.cachedVariables.length > 0 &&
+                !this.hasRerun) {
+                this.hasRerun = true;
+                // invalidate the cache value
+                yield threadUtils.invalidateDynamicVariables(this.cachedVariables);
+                return this.execute();
+            }
+            // needs to an array for next step
+            if (!Array.isArray(rows)) {
+                rows = [rows];
+            }
+            // map into JSON if just raw primitive here
+            if (rows.find(row => typeof row !== "object")) {
+                rows = rows.map(value => ({ value }));
+            }
+            // get all the potential fields in the schema
+            let keys = rows.flatMap(Object.keys);
+            if (integration.end) {
+                integration.end();
+            }
+            return { rows, keys, info, extra, pagination };
+        });
+    }
+    runAnotherQuery(queryId, parameters) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const db = new CouchDB(this.appId);
+            const query = yield db.get(queryId);
+            const datasource = yield db.get(query.datasourceId);
+            return new QueryRunner({
+                appId: this.appId,
+                datasource,
+                queryVerb: query.queryVerb,
+                fields: query.fields,
+                parameters,
+                transformer: query.transformer,
+            }, { noRecursiveQuery: true }).execute();
+        });
+    }
+    getDynamicVariable(variable) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { parameters } = this;
+            const queryId = variable.queryId, name = variable.name;
+            let value = yield threadUtils.checkCacheForDynamicVariable(queryId, name);
+            if (!value) {
+                value = this.queryResponse[queryId]
+                    ? this.queryResponse[queryId]
+                    : yield this.runAnotherQuery(queryId, parameters);
+                // store incase this query is to be called again
+                this.queryResponse[queryId] = value;
+                yield threadUtils.storeDynamicVariable(queryId, name, value);
+            }
+            else {
+                this.cachedVariables.push({ queryId, name });
+            }
+            return value;
+        });
+    }
+    addDatasourceVariables() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { datasource, parameters, fields } = this;
+            if (!datasource || !datasource.config) {
+                return parameters;
+            }
+            const staticVars = datasource.config.staticVariables || {};
+            const dynamicVars = datasource.config.dynamicVariables || [];
+            for (let [key, value] of Object.entries(staticVars)) {
+                if (!parameters[key]) {
+                    parameters[key] = value;
+                }
+            }
+            if (!this.noRecursiveQuery) {
+                // need to see if this uses any variables
+                const stringFields = JSON.stringify(fields);
+                const foundVars = dynamicVars.filter(variable => {
+                    // don't allow a query to use its own dynamic variable (loop)
+                    if (variable.queryId === this.queryId) {
+                        return false;
+                    }
+                    // look for {{ variable }} but allow spaces between handlebars
+                    const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`);
+                    return regex.test(stringFields);
+                });
+                const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar));
+                const responses = yield Promise.all(dynamics);
+                for (let i = 0; i < foundVars.length; i++) {
+                    const variable = foundVars[i];
+                    parameters[variable.name] = processStringSync(variable.value, {
+                        data: responses[i].rows,
+                        info: responses[i].extra,
+                    });
+                    // make sure its known that this uses dynamic variables in case it fails
+                    this.hasDynamicVariables = true;
+                }
+            }
+            return parameters;
+        });
+    }
+    enrichQueryFields(fields, parameters = {}) {
+        const enrichedQuery = {};
+        // enrich the fields with dynamic parameters
+        for (let key of Object.keys(fields)) {
+            if (fields[key] == null) {
+                continue;
+            }
+            if (typeof fields[key] === "object") {
+                // enrich nested fields object
+                enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters);
+            }
+            else if (typeof fields[key] === "string") {
+                // enrich string value as normal
+                let value = fields[key];
+                // add triple brace to avoid escaping e.g. '=' in cookie header
+                if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
+                    value = `{${value}}`;
+                }
+                enrichedQuery[key] = processStringSync(value, parameters, {
+                    noHelpers: true,
+                });
+            }
+            else {
+                enrichedQuery[key] = fields[key];
+            }
         }
-        // get all the potential fields in the schema
-        let keys = rows.flatMap(Object.keys);
-        if (integration.end) {
-            integration.end();
+        if (enrichedQuery.json ||
+            enrichedQuery.customData ||
+            enrichedQuery.requestBody) {
+            try {
+                enrichedQuery.json = JSON.parse(enrichedQuery.json ||
+                    enrichedQuery.customData ||
+                    enrichedQuery.requestBody);
+            }
+            catch (err) {
+                // no json found, ignore
+            }
+            delete enrichedQuery.customData;
         }
-        return { rows, keys };
-    });
+        return enrichedQuery;
+    }
 }
 module.exports = (input, callback) => {
-    runAndTransform(input.datasource, input.queryVerb, input.query, input.transformer)
+    const Runner = new QueryRunner(input);
+    Runner.execute()
         .then(response => {
         callback(null, response);
     })

package/dist/threads/utils.js

@@ -1,7 +1,32 @@
 "use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 const env = require("../environment");
 const CouchDB = require("../db");
-const { init } = require("@budibase/auth");
+const { init } = require("@budibase/backend-core");
+const redis = require("@budibase/backend-core/redis");
+const { SEPARATOR } = require("@budibase/backend-core/db");
+const VARIABLE_TTL_SECONDS = 3600;
+let client;
+function getClient() {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!client) {
+            client = yield new redis.Client(redis.utils.Databases.QUERY_VARS).init();
+        }
+        return client;
+    });
+}
+process.on("exit", () => __awaiter(void 0, void 0, void 0, function* () {
+    if (client)
+        yield client.finish();
+}));
 exports.threadSetup = () => {
     // don't run this if not threading
     if (env.isTest() || env.DISABLE_THREADING) {
@@ -11,3 +36,39 @@ exports.threadSetup = () => {
     env.setInThread();
     init(CouchDB);
 };
+function makeVariableKey(queryId, variable) {
+    return `${queryId}${SEPARATOR}${variable}`;
+}
+exports.checkCacheForDynamicVariable = (queryId, variable) => __awaiter(void 0, void 0, void 0, function* () {
+    const cache = yield getClient();
+    return cache.get(makeVariableKey(queryId, variable));
+});
+exports.invalidateDynamicVariables = (cachedVars) => __awaiter(void 0, void 0, void 0, function* () {
+    const cache = yield getClient();
+    let promises = [];
+    for (let variable of cachedVars) {
+        promises.push(cache.delete(makeVariableKey(variable.queryId, variable.name)));
+    }
+    yield Promise.all(promises);
+});
+exports.storeDynamicVariable = (queryId, variable, value) => __awaiter(void 0, void 0, void 0, function* () {
+    const cache = yield getClient();
+    yield cache.store(makeVariableKey(queryId, variable), value, VARIABLE_TTL_SECONDS);
+});
+exports.formatResponse = resp => {
+    if (typeof resp === "string") {
+        try {
+            resp = JSON.parse(resp);
+        }
+        catch (err) {
+            resp = { response: resp };
+        }
+    }
+    return resp;
+};
+exports.hasExtraData = response => {
+    return (typeof response === "object" &&
+        !Array.isArray(response) &&
+        response.data != null &&
+        response.info != null);
+};