@budibase/server 1.0.27-alpha.3 → 1.0.27-alpha.7

package/dist/api/controllers/query/import/sources/base/index.js

@@ -11,12 +11,13 @@ var MethodToVerb;
 })(MethodToVerb || (MethodToVerb = {}));
 class ImportSource {
     constructor() {
-        this.constructQuery = (datasourceId, name, method, path, queryString, headers = {}, parameters = [], body = undefined) => {
+        this.constructQuery = (datasourceId, name, method, path, url, queryString, headers = {}, parameters = [], body = undefined) => {
             const readable = true;
             const queryVerb = this.verbFromMethod(method);
             const transformer = "return data";
             const schema = {};
             path = this.processPath(path);
+            path = `${url.origin}/${path}`;
             queryString = this.processQuery(queryString);
             const requestBody = JSON.stringify(body, null, 2);
             const query = {

package/dist/api/controllers/query/import/sources/curl.js

@@ -26,8 +26,12 @@ const parseBody = (curl) => {
     if (curl.data) {
         const keys = Object.keys(curl.data);
         if (keys.length) {
-            const key = keys[0];
+            let key = keys[0];
             try {
+                // filter out the dollar syntax used by curl for shell support
+                if (key.startsWith("$")) {
+                    key = key.substring(1);
+                }
                 return JSON.parse(key);
             }
             catch (e) {
@@ -63,15 +67,17 @@ class Curl extends base_1.ImportSource {
             }
             return true;
         });
+        this.getUrl = () => {
+            return new url_1.URL(this.curl.raw_url);
+        };
         this.getInfo = () => __awaiter(this, void 0, void 0, function* () {
-            const url = new url_1.URL(this.curl.url);
+            const url = this.getUrl();
             return {
-                url: url.origin,
                 name: url.hostname,
             };
         });
         this.getQueries = (datasourceId) => __awaiter(this, void 0, void 0, function* () {
-            const url = new url_1.URL(this.curl.raw_url);
+            const url = this.getUrl();
             const name = url.pathname;
             const path = url.pathname;
             const method = this.curl.method;
@@ -82,7 +88,7 @@ class Curl extends base_1.ImportSource {
             if (cookieHeader) {
                 headers["Cookie"] = cookieHeader;
             }
-            const query = this.constructQuery(datasourceId, name, method, path, queryString, headers, [], requestBody);
+            const query = this.constructQuery(datasourceId, name, method, path, url, queryString, headers, [], requestBody);
             return [query];
         });
     }

package/dist/api/controllers/query/import/sources/openapi2.js

@@ -12,6 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenAPI2 = void 0;
 const openapi_types_1 = require("openapi-types");
 const openapi_1 = require("./base/openapi");
+const url_1 = require("url");
 const parameterNotRef = (param) => {
     // all refs are deferenced by parser library
     return true;
@@ -53,19 +54,21 @@ class OpenAPI2 extends openapi_1.OpenAPISource {
                 return false;
             }
         });
-        this.getInfo = () => __awaiter(this, void 0, void 0, function* () {
+        this.getUrl = () => {
             var _a;
             const scheme = ((_a = this.document.schemes) === null || _a === void 0 ? void 0 : _a.includes("https")) ? "https" : "http";
             const basePath = this.document.basePath || "";
             const host = this.document.host || "<host>";
-            const url = `${scheme}://${host}${basePath}`;
+            return new url_1.URL(`${scheme}://${host}${basePath}`);
+        };
+        this.getInfo = () => __awaiter(this, void 0, void 0, function* () {
             const name = this.document.info.title || "Swagger Import";
             return {
-                url: url,
-                name: name,
+                name,
             };
         });
         this.getQueries = (datasourceId) => __awaiter(this, void 0, void 0, function* () {
+            const url = this.getUrl();
             const queries = [];
             for (let [path, pathItem] of Object.entries(this.document.paths)) {
                 // parameters that apply to every operation in the path
@@ -128,7 +131,7 @@ class OpenAPI2 extends openapi_1.OpenAPISource {
                             }
                         }
                     }
-                    const query = this.constructQuery(datasourceId, name, methodName, path, queryString, headers, parameters, requestBody);
+                    const query = this.constructQuery(datasourceId, name, methodName, path, url, queryString, headers, parameters, requestBody);
                     queries.push(query);
                 }
             }

package/dist/api/controllers/query/index.js

@@ -8,13 +8,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const { processString } = require("@budibase/string-templates");
 const CouchDB = require("../../../db");
 const { generateQueryID, getQueryParams, isProdAppID, } = require("../../../db/utils");
 const { BaseQueryVerbs } = require("../../../constants");
 const { Thread, ThreadType } = require("../../../threads");
 const { save: saveDatasource } = require("../datasource");
 const { RestImporter } = require("./import");
+const { invalidateDynamicVariables } = require("../../../threads/utils");
 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 });
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -81,44 +81,6 @@ exports.save = function (ctx) {
         ctx.message = `Query ${query.name} saved successfully.`;
     });
 };
-function enrichQueryFields(fields, parameters = {}) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const enrichedQuery = {};
-        // enrich the fields with dynamic parameters
-        for (let key of Object.keys(fields)) {
-            if (fields[key] == null) {
-                continue;
-            }
-            if (typeof fields[key] === "object") {
-                // enrich nested fields object
-                enrichedQuery[key] = yield enrichQueryFields(fields[key], parameters);
-            }
-            else if (typeof fields[key] === "string") {
-                // enrich string value as normal
-                enrichedQuery[key] = yield processString(fields[key], parameters, {
-                    noHelpers: true,
-                });
-            }
-            else {
-                enrichedQuery[key] = fields[key];
-            }
-        }
-        if (enrichedQuery.json ||
-            enrichedQuery.customData ||
-            enrichedQuery.requestBody) {
-            try {
-                enrichedQuery.json = JSON.parse(enrichedQuery.json ||
-                    enrichedQuery.customData ||
-                    enrichedQuery.requestBody);
-            }
-            catch (err) {
-                // no json found, ignore
-            }
-            delete enrichedQuery.customData;
-        }
-        return enrichedQuery;
-    });
-}
 exports.find = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = new CouchDB(ctx.appId);
@@ -135,14 +97,18 @@ exports.preview = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = new CouchDB(ctx.appId);
         const datasource = yield db.get(ctx.request.body.datasourceId);
-        const { fields, parameters, queryVerb, transformer } = ctx.request.body;
-        const enrichedQuery = yield enrichQueryFields(fields, parameters);
+        // preview may not have a queryId as it hasn't been saved, but if it does
+        // this stops dynamic variables from calling the same query
+        const { fields, parameters, queryVerb, transformer, queryId } = ctx.request.body;
         try {
             const { rows, keys, info, extra } = yield Runner.run({
+                appId: ctx.appId,
                 datasource,
                 queryVerb,
-                query: enrichedQuery,
+                fields,
+                parameters,
                 transformer,
+                queryId,
             });
             ctx.body = {
                 rows,
@@ -156,30 +122,62 @@ exports.preview = function (ctx) {
         }
     });
 };
-exports.execute = function (ctx) {
+function execute(ctx, opts = { rowsOnly: false }) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = new CouchDB(ctx.appId);
         const query = yield db.get(ctx.params.queryId);
         const datasource = yield db.get(query.datasourceId);
-        const enrichedQuery = yield enrichQueryFields(query.fields, ctx.request.body.parameters);
         // call the relevant CRUD method on the integration class
         try {
-            const { rows } = yield Runner.run({
+            const { rows, extra } = yield Runner.run({
+                appId: ctx.appId,
                 datasource,
                 queryVerb: query.queryVerb,
-                query: enrichedQuery,
+                fields: query.fields,
+                parameters: ctx.request.body.parameters,
                 transformer: query.transformer,
+                queryId: ctx.params.queryId,
             });
-            ctx.body = rows;
+            if (opts && opts.rowsOnly) {
+                ctx.body = rows;
+            }
+            else {
+                ctx.body = Object.assign({ data: rows }, extra);
+            }
         }
         catch (err) {
             ctx.throw(400, err);
         }
     });
+}
+exports.executeV1 = function (ctx) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return execute(ctx, { rowsOnly: true });
+    });
 };
+exports.executeV2 = function (ctx) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return execute(ctx, { rowsOnly: false });
+    });
+};
+const removeDynamicVariables = (db, queryId) => __awaiter(void 0, void 0, void 0, function* () {
+    const query = yield db.get(queryId);
+    const datasource = yield db.get(query.datasourceId);
+    const dynamicVariables = datasource.config.dynamicVariables;
+    if (dynamicVariables) {
+        // delete dynamic variables from the datasource
+        const newVariables = dynamicVariables.filter(dv => dv.queryId !== queryId);
+        datasource.config.dynamicVariables = newVariables;
+        yield db.put(datasource);
+        // invalidate the deleted variables
+        const variablesToDelete = dynamicVariables.filter(dv => dv.queryId === queryId);
+        yield invalidateDynamicVariables(variablesToDelete);
+    }
+});
 exports.destroy = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = new CouchDB(ctx.appId);
+        yield removeDynamicVariables(db, ctx.params.queryId);
         yield db.remove(ctx.params.queryId, ctx.params.revId);
         ctx.message = `Query deleted.`;
         ctx.status = 200;

package/dist/api/controllers/query/validation.js

@@ -32,6 +32,7 @@ exports.generateQueryPreviewValidation = () => {
         extra: Joi.object().optional(),
         datasourceId: Joi.string().required(),
         transformer: Joi.string().optional(),
-        parameters: Joi.object({}).required().unknown(true)
+        parameters: Joi.object({}).required().unknown(true),
+        queryId: Joi.string().optional(),
     }));
 };

package/dist/api/routes/query.js

@@ -12,6 +12,8 @@ router
     .post("/api/queries/import", authorized(BUILDER), queryController.import)
     .post("/api/queries/preview", bodyResource("datasourceId"), authorized(BUILDER), generateQueryPreviewValidation(), queryController.preview)
     .get("/api/queries/:queryId", paramResource("queryId"), authorized(PermissionTypes.QUERY, PermissionLevels.READ), queryController.find)
-    .post("/api/queries/:queryId", paramResource("queryId"), authorized(PermissionTypes.QUERY, PermissionLevels.WRITE), queryController.execute)
+    // DEPRECATED - use new query endpoint for future work
+    .post("/api/queries/:queryId", paramResource("queryId"), authorized(PermissionTypes.QUERY, PermissionLevels.WRITE), queryController.executeV1)
+    .post("/api/v2/queries/:queryId", paramResource("queryId"), authorized(PermissionTypes.QUERY, PermissionLevels.WRITE), queryController.executeV2)
     .delete("/api/queries/:queryId/:revId", paramResource("queryId"), authorized(BUILDER), queryController.destroy);
 module.exports = router;

package/dist/automations/steps/executeQuery.js

@@ -55,6 +55,10 @@ exports.definition = {
                     type: "object",
                     description: "The response from the datasource execution",
                 },
+                info: {
+                    type: "object",
+                    description: "Some query types may return extra data, like headers from a REST query",
+                },
                 success: {
                     type: "boolean",
                     description: "Whether the action was successful",
@@ -85,14 +89,17 @@ exports.run = function ({ inputs, appId, emitter }) {
         });
         try {
             yield queryController.execute(ctx);
+            const _b = ctx.body, { data } = _b, rest = __rest(_b, ["data"]);
             return {
-                response: ctx.body,
+                response: data,
+                info: rest,
                 success: true,
             };
         }
         catch (err) {
             return {
                 success: false,
+                info: {},
                 response: automationUtils.getError(err),
             };
         }

package/dist/integrations/rest.js

@@ -47,7 +47,7 @@ var RestModule;
     const { performance } = require("perf_hooks");
     const FormData = require("form-data");
     const { URLSearchParams } = require("url");
-    const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js");
+    const { parseStringPromise: xmlParser, Builder: XmlBuilder, } = require("xml2js");
     const SCHEMA = {
         docs: "https://github.com/node-fetch/node-fetch",
         description: "With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
@@ -159,7 +159,7 @@ var RestModule;
         getUrl(path, queryString) {
             const main = `${path}?${queryString}`;
             let complete = main;
-            if (this.config.url && !main.startsWith(this.config.url)) {
+            if (this.config.url && !main.startsWith("http")) {
                 complete = !this.config.url ? main : `${this.config.url}/${main}`;
             }
             if (!complete.startsWith("http")) {
@@ -202,7 +202,7 @@ var RestModule;
                     break;
                 case BodyTypes.XML:
                     if (object != null) {
-                        string = (new XmlBuilder()).buildObject(object);
+                        string = new XmlBuilder().buildObject(object);
                     }
                     input.body = string;
                     input.headers["Content-Type"] = "application/xml";

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "1.0.27-alpha.2",
+    "version": "1.0.27-alpha.6",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -70,9 +70,9 @@
     "license": "GPL-3.0",
     "dependencies": {
         "@apidevtools/swagger-parser": "^10.0.3",
-        "@budibase/auth": "^1.0.27-alpha.2",
-        "@budibase/client": "^1.0.27-alpha.2",
-        "@budibase/string-templates": "^1.0.27-alpha.2",
+        "@budibase/auth": "^1.0.27-alpha.6",
+        "@budibase/client": "^1.0.27-alpha.6",
+        "@budibase/string-templates": "^1.0.27-alpha.6",
         "@bull-board/api": "^3.7.0",
         "@bull-board/koa": "^3.7.0",
         "@elastic/elasticsearch": "7.10.0",
@@ -122,6 +122,7 @@
         "pouchdb-replication-stream": "1.2.9",
         "server-destroy": "1.0.1",
         "svelte": "^3.38.2",
+        "swagger-parser": "^10.0.3",
         "to-json-schema": "0.2.5",
         "uuid": "3.3.2",
         "validate.js": "0.13.1",

package/dist/tests/utilities/TestConfiguration.js

@@ -294,6 +294,13 @@ class TestConfiguration {
             return this.datasource;
         });
     }
+    updateDatasource(datasource) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this._req(datasource, { datasourceId: datasource._id }, controllers.datasource.update);
+            this.datasource = response.datasource;
+            return this.datasource;
+        });
+    }
     createQuery(config = null) {
         return __awaiter(this, void 0, void 0, function* () {
             if (!this.datasource && !config) {

package/dist/threads/query.js

@@ -8,63 +8,156 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-require("./utils").threadSetup();
+const threadUtils = require("./utils");
+threadUtils.threadSetup();
 const ScriptRunner = require("../utilities/scriptRunner");
 const { integrations } = require("../integrations");
-function formatResponse(resp) {
-    if (typeof resp === "string") {
-        try {
-            resp = JSON.parse(resp);
-        }
-        catch (err) {
-            resp = { response: resp };
-        }
+const { processStringSync } = require("@budibase/string-templates");
+const CouchDB = require("../db");
+class QueryRunner {
+    constructor(input, flags = { noRecursiveQuery: false }) {
+        this.appId = input.appId;
+        this.datasource = input.datasource;
+        this.queryVerb = input.queryVerb;
+        this.fields = input.fields;
+        this.parameters = input.parameters;
+        this.transformer = input.transformer;
+        this.queryId = input.queryId;
+        this.noRecursiveQuery = flags.noRecursiveQuery;
+        this.cachedVariables = [];
+        // allows the response from a query to be stored throughout this
+        // execution so that if it needs to be re-used for another variable
+        // it can be
+        this.queryResponse = {};
+        this.hasRerun = false;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { datasource, fields, queryVerb, transformer } = this;
+            // pre-query, make sure datasource variables are added to parameters
+            const parameters = yield this.addDatasourceVariables();
+            const query = threadUtils.enrichQueryFields(fields, parameters);
+            const Integration = integrations[datasource.source];
+            if (!Integration) {
+                throw "Integration type does not exist.";
+            }
+            const integration = new Integration(datasource.config);
+            let output = threadUtils.formatResponse(yield integration[queryVerb](query));
+            let rows = output, info = undefined, extra = undefined;
+            if (threadUtils.hasExtraData(output)) {
+                rows = output.data;
+                info = output.info;
+                extra = output.extra;
+            }
+            // transform as required
+            if (transformer) {
+                const runner = new ScriptRunner(transformer, { data: rows });
+                rows = runner.execute();
+            }
+            // if the request fails we retry once, invalidating the cached value
+            if (info &&
+                info.code >= 400 &&
+                this.cachedVariables.length > 0 &&
+                !this.hasRerun) {
+                this.hasRerun = true;
+                // invalidate the cache value
+                yield threadUtils.invalidateDynamicVariables(this.cachedVariables);
+                return this.execute();
+            }
+            // needs to an array for next step
+            if (!Array.isArray(rows)) {
+                rows = [rows];
+            }
+            // map into JSON if just raw primitive here
+            if (rows.find(row => typeof row !== "object")) {
+                rows = rows.map(value => ({ value }));
+            }
+            // get all the potential fields in the schema
+            let keys = rows.flatMap(Object.keys);
+            if (integration.end) {
+                integration.end();
+            }
+            return { rows, keys, info, extra };
+        });
+    }
+    runAnotherQuery(queryId, parameters) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const db = new CouchDB(this.appId);
+            const query = yield db.get(queryId);
+            const datasource = yield db.get(query.datasourceId);
+            return new QueryRunner({
+                appId: this.appId,
+                datasource,
+                queryVerb: query.queryVerb,
+                fields: query.fields,
+                parameters,
+                transformer: query.transformer,
+            }, { noRecursiveQuery: true }).execute();
+        });
+    }
+    getDynamicVariable(variable) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { parameters } = this;
+            const queryId = variable.queryId, name = variable.name;
+            let value = yield threadUtils.checkCacheForDynamicVariable(queryId, name);
+            if (!value) {
+                value = this.queryResponse[queryId]
+                    ? this.queryResponse[queryId]
+                    : yield this.runAnotherQuery(queryId, parameters);
+                // store incase this query is to be called again
+                this.queryResponse[queryId] = value;
+                yield threadUtils.storeDynamicVariable(queryId, name, value);
+            }
+            else {
+                this.cachedVariables.push({ queryId, name });
+            }
+            return value;
+        });
+    }
+    addDatasourceVariables() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { datasource, parameters, fields } = this;
+            if (!datasource || !datasource.config) {
+                return parameters;
+            }
+            const staticVars = datasource.config.staticVariables || {};
+            const dynamicVars = datasource.config.dynamicVariables || [];
+            for (let [key, value] of Object.entries(staticVars)) {
+                if (!parameters[key]) {
+                    parameters[key] = value;
+                }
+            }
+            if (!this.noRecursiveQuery) {
+                // need to see if this uses any variables
+                const stringFields = JSON.stringify(fields);
+                const foundVars = dynamicVars.filter(variable => {
+                    // don't allow a query to use its own dynamic variable (loop)
+                    if (variable.queryId === this.queryId) {
+                        return false;
+                    }
+                    // look for {{ variable }} but allow spaces between handlebars
+                    const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`);
+                    return regex.test(stringFields);
+                });
+                const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar));
+                const responses = yield Promise.all(dynamics);
+                for (let i = 0; i < foundVars.length; i++) {
+                    const variable = foundVars[i];
+                    parameters[variable.name] = processStringSync(variable.value, {
+                        data: responses[i].rows,
+                        info: responses[i].extra,
+                    });
+                    // make sure its known that this uses dynamic variables in case it fails
+                    this.hasDynamicVariables = true;
+                }
+            }
+            return parameters;
+        });
     }
-    return resp;
-}
-function hasExtraData(response) {
-    return (typeof response === "object" &&
-        !Array.isArray(response) &&
-        response.data != null &&
-        response.info != null);
-}
-function runAndTransform(datasource, queryVerb, query, transformer) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const Integration = integrations[datasource.source];
-        if (!Integration) {
-            throw "Integration type does not exist.";
-        }
-        const integration = new Integration(datasource.config);
-        let output = formatResponse(yield integration[queryVerb](query));
-        let rows = output, info = undefined, extra = undefined;
-        if (hasExtraData(output)) {
-            rows = output.data;
-            info = output.info;
-            extra = output.extra;
-        }
-        // transform as required
-        if (transformer) {
-            const runner = new ScriptRunner(transformer, { data: rows });
-            rows = runner.execute();
-        }
-        // needs to an array for next step
-        if (!Array.isArray(rows)) {
-            rows = [rows];
-        }
-        // map into JSON if just raw primitive here
-        if (rows.find(row => typeof row !== "object")) {
-            rows = rows.map(value => ({ value }));
-        }
-        // get all the potential fields in the schema
-        let keys = rows.flatMap(Object.keys);
-        if (integration.end) {
-            integration.end();
-        }
-        return { rows, keys, info, extra };
-    });
 }
 module.exports = (input, callback) => {
-    runAndTransform(input.datasource, input.queryVerb, input.query, input.transformer)
+    const Runner = new QueryRunner(input);
+    Runner.execute()
         .then(response => {
         callback(null, response);
     })