@budibase/server 1.0.25 → 1.0.27-alpha.10

package/dist/integrations/rest.js

@@ -10,18 +10,54 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const datasource_1 = require("../definitions/datasource");
+const BodyTypes = {
+    NONE: "none",
+    FORM_DATA: "form",
+    XML: "xml",
+    ENCODED: "encoded",
+    JSON: "json",
+    TEXT: "text",
+};
+const coreFields = {
+    path: {
+        type: datasource_1.DatasourceFieldTypes.STRING,
+        display: "URL",
+    },
+    queryString: {
+        type: datasource_1.DatasourceFieldTypes.STRING,
+    },
+    headers: {
+        type: datasource_1.DatasourceFieldTypes.OBJECT,
+    },
+    enabledHeaders: {
+        type: datasource_1.DatasourceFieldTypes.OBJECT,
+    },
+    requestBody: {
+        type: datasource_1.DatasourceFieldTypes.JSON,
+    },
+    bodyType: {
+        type: datasource_1.DatasourceFieldTypes.STRING,
+        enum: Object.values(BodyTypes),
+    },
+};
 var RestModule;
 (function (RestModule) {
     const fetch = require("node-fetch");
+    const { formatBytes } = require("../utilities");
+    const { performance } = require("perf_hooks");
+    const FormData = require("form-data");
+    const { URLSearchParams } = require("url");
+    const { parseStringPromise: xmlParser, Builder: XmlBuilder, } = require("xml2js");
     const SCHEMA = {
         docs: "https://github.com/node-fetch/node-fetch",
-        description: "Representational state transfer (REST) is a de-facto standard for a software architecture for interactive applications that typically use multiple Web services. ",
+        description: "With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
         friendlyName: "REST API",
         datasource: {
             url: {
                 type: datasource_1.DatasourceFieldTypes.STRING,
-                default: "localhost",
-                required: true,
+                default: "",
+                required: false,
+                deprecated: true,
             },
             defaultHeaders: {
                 type: datasource_1.DatasourceFieldTypes.OBJECT,
@@ -34,174 +70,228 @@ var RestModule;
                 readable: true,
                 displayName: "POST",
                 type: datasource_1.QueryTypes.FIELDS,
-                urlDisplay: true,
-                fields: {
-                    path: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    queryString: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    headers: {
-                        type: datasource_1.DatasourceFieldTypes.OBJECT,
-                    },
-                    requestBody: {
-                        type: datasource_1.DatasourceFieldTypes.JSON,
-                    },
-                },
+                fields: coreFields,
             },
             read: {
                 displayName: "GET",
                 readable: true,
                 type: datasource_1.QueryTypes.FIELDS,
-                urlDisplay: true,
-                fields: {
-                    path: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    queryString: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    headers: {
-                        type: datasource_1.DatasourceFieldTypes.OBJECT,
-                    },
-                },
+                fields: coreFields,
             },
             update: {
                 displayName: "PUT",
                 readable: true,
                 type: datasource_1.QueryTypes.FIELDS,
-                urlDisplay: true,
-                fields: {
-                    path: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    queryString: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    headers: {
-                        type: datasource_1.DatasourceFieldTypes.OBJECT,
-                    },
-                    requestBody: {
-                        type: datasource_1.DatasourceFieldTypes.JSON,
-                    },
-                },
+                fields: coreFields,
             },
             patch: {
                 displayName: "PATCH",
                 readable: true,
                 type: datasource_1.QueryTypes.FIELDS,
-                urlDisplay: true,
-                fields: {
-                    path: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    queryString: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    headers: {
-                        type: datasource_1.DatasourceFieldTypes.OBJECT,
-                    },
-                    requestBody: {
-                        type: datasource_1.DatasourceFieldTypes.JSON,
-                    },
-                },
+                fields: coreFields,
             },
             delete: {
                 displayName: "DELETE",
                 type: datasource_1.QueryTypes.FIELDS,
-                urlDisplay: true,
-                fields: {
-                    path: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    queryString: {
-                        type: datasource_1.DatasourceFieldTypes.STRING,
-                    },
-                    headers: {
-                        type: datasource_1.DatasourceFieldTypes.OBJECT,
-                    },
-                    requestBody: {
-                        type: datasource_1.DatasourceFieldTypes.JSON,
-                    },
-                },
+                fields: coreFields,
             },
         },
     };
     class RestIntegration {
         constructor(config) {
             this.headers = {};
+            this.startTimeMs = performance.now();
             this.config = config;
         }
         parseResponse(response) {
             return __awaiter(this, void 0, void 0, function* () {
-                const contentType = response.headers.get("content-type");
-                if (contentType && contentType.indexOf("application/json") !== -1) {
-                    return yield response.json();
+                let data, raw, headers;
+                const contentType = response.headers.get("content-type") || "";
+                try {
+                    if (contentType.includes("application/json")) {
+                        data = yield response.json();
+                        raw = JSON.stringify(data);
+                    }
+                    else if (contentType.includes("text/xml") ||
+                        contentType.includes("application/xml")) {
+                        const rawXml = yield response.text();
+                        data =
+                            (yield xmlParser(rawXml, {
+                                explicitArray: false,
+                                trim: true,
+                                explicitRoot: false,
+                            })) || {};
+                        // there is only one structure, its an array, return the array so it appears as rows
+                        const keys = Object.keys(data);
+                        if (keys.length === 1 && Array.isArray(data[keys[0]])) {
+                            data = data[keys[0]];
+                        }
+                        raw = rawXml;
+                    }
+                    else {
+                        data = yield response.text();
+                        raw = data;
+                    }
                 }
-                else {
-                    return yield response.text();
+                catch (err) {
+                    throw "Failed to parse response body.";
                 }
+                const size = formatBytes(response.headers.get("content-length") || Buffer.byteLength(raw, "utf8"));
+                const time = `${Math.round(performance.now() - this.startTimeMs)}ms`;
+                headers = response.headers.raw();
+                for (let [key, value] of Object.entries(headers)) {
+                    headers[key] = Array.isArray(value) ? value[0] : value;
+                }
+                return {
+                    data,
+                    info: {
+                        code: response.status,
+                        size,
+                        time,
+                    },
+                    extra: {
+                        raw,
+                        headers,
+                    },
+                };
             });
         }
         getUrl(path, queryString) {
-            return `${this.config.url}/${path}?${queryString}`;
+            const main = `${path}?${queryString}`;
+            let complete = main;
+            if (this.config.url && !main.startsWith("http")) {
+                complete = !this.config.url ? main : `${this.config.url}/${main}`;
+            }
+            if (!complete.startsWith("http")) {
+                complete = `http://${complete}`;
+            }
+            return complete;
+        }
+        addBody(bodyType, body, input) {
+            let error, object, string;
+            try {
+                string = typeof body !== "string" ? JSON.stringify(body) : body;
+                object = typeof body === "object" ? body : JSON.parse(body);
+            }
+            catch (err) {
+                error = err;
+            }
+            if (!input.headers) {
+                input.headers = {};
+            }
+            switch (bodyType) {
+                case BodyTypes.NONE:
+                    break;
+                case BodyTypes.TEXT:
+                    // content type defaults to plaintext
+                    input.body = string;
+                    break;
+                case BodyTypes.ENCODED:
+                    const params = new URLSearchParams();
+                    for (let [key, value] of Object.entries(object)) {
+                        params.append(key, value);
+                    }
+                    input.body = params;
+                    break;
+                case BodyTypes.FORM_DATA:
+                    const form = new FormData();
+                    for (let [key, value] of Object.entries(object)) {
+                        form.append(key, value);
+                    }
+                    input.body = form;
+                    break;
+                case BodyTypes.XML:
+                    if (object != null) {
+                        string = new XmlBuilder().buildObject(object);
+                    }
+                    input.body = string;
+                    input.headers["Content-Type"] = "application/xml";
+                    break;
+                default:
+                case BodyTypes.JSON:
+                    // if JSON error, throw it
+                    if (error) {
+                        throw "Invalid JSON for request body";
+                    }
+                    input.body = string;
+                    input.headers["Content-Type"] = "application/json";
+                    break;
+            }
+            return input;
+        }
+        getAuthHeaders(authConfigId) {
+            let headers = {};
+            if (this.config.authConfigs && authConfigId) {
+                const authConfig = this.config.authConfigs.filter(c => c._id === authConfigId)[0];
+                // check the config still exists before proceeding
+                // if not - do nothing
+                if (authConfig) {
+                    let config;
+                    switch (authConfig.type) {
+                        case datasource_1.AuthType.BASIC:
+                            config = authConfig.config;
+                            headers.Authorization = `Basic ${Buffer.from(`${config.username}:${config.password}`).toString("base64")}`;
+                            break;
+                        case datasource_1.AuthType.BEARER:
+                            config = authConfig.config;
+                            headers.Authorization = `Bearer ${config.token}`;
+                            break;
+                    }
+                }
+            }
+            return headers;
         }
-        create({ path = "", queryString = "", headers = {}, json = {} }) {
+        _req(query) {
             return __awaiter(this, void 0, void 0, function* () {
-                this.headers = Object.assign(Object.assign({}, this.config.defaultHeaders), headers);
-                const response = yield fetch(this.getUrl(path, queryString), {
-                    method: "POST",
-                    headers: this.headers,
-                    body: JSON.stringify(json),
-                });
+                const { path = "", queryString = "", headers = {}, method = "GET", disabledHeaders, bodyType, requestBody, authConfigId, } = query;
+                const authHeaders = this.getAuthHeaders(authConfigId);
+                this.headers = Object.assign(Object.assign(Object.assign({}, this.config.defaultHeaders), headers), authHeaders);
+                if (disabledHeaders) {
+                    for (let headerKey of Object.keys(this.headers)) {
+                        if (disabledHeaders[headerKey]) {
+                            delete this.headers[headerKey];
+                        }
+                    }
+                }
+                let input = { method, headers: this.headers };
+                if (requestBody) {
+                    input = this.addBody(bodyType, requestBody, input);
+                }
+                this.startTimeMs = performance.now();
+                const url = this.getUrl(path, queryString);
+                const response = yield fetch(url, input);
                 return yield this.parseResponse(response);
             });
         }
-        read({ path = "", queryString = "", headers = {} }) {
+        create(opts) {
             return __awaiter(this, void 0, void 0, function* () {
-                this.headers = Object.assign(Object.assign({}, this.config.defaultHeaders), headers);
-                const response = yield fetch(this.getUrl(path, queryString), {
-                    headers: this.headers,
-                });
-                return yield this.parseResponse(response);
+                return this._req(Object.assign(Object.assign({}, opts), { method: "POST" }));
             });
         }
-        update({ path = "", queryString = "", headers = {}, json = {} }) {
+        read(opts) {
             return __awaiter(this, void 0, void 0, function* () {
-                this.headers = Object.assign(Object.assign({}, this.config.defaultHeaders), headers);
-                const response = yield fetch(this.getUrl(path, queryString), {
-                    method: "PUT",
-                    headers: this.headers,
-                    body: JSON.stringify(json),
-                });
-                return yield this.parseResponse(response);
+                return this._req(Object.assign(Object.assign({}, opts), { method: "GET" }));
             });
         }
-        patch({ path = "", queryString = "", headers = {}, json = {} }) {
+        update(opts) {
             return __awaiter(this, void 0, void 0, function* () {
-                this.headers = Object.assign(Object.assign({}, this.config.defaultHeaders), headers);
-                const response = yield fetch(this.getUrl(path, queryString), {
-                    method: "PATCH",
-                    headers: this.headers,
-                    body: JSON.stringify(json),
-                });
-                return yield this.parseResponse(response);
+                return this._req(Object.assign(Object.assign({}, opts), { method: "PUT" }));
             });
         }
-        delete({ path = "", queryString = "", headers = {} }) {
+        patch(opts) {
             return __awaiter(this, void 0, void 0, function* () {
-                this.headers = Object.assign(Object.assign({}, this.config.defaultHeaders), headers);
-                const response = yield fetch(this.getUrl(path, queryString), {
-                    method: "DELETE",
-                    headers: this.headers,
-                });
-                return yield this.parseResponse(response);
+                return this._req(Object.assign(Object.assign({}, opts), { method: "PATCH" }));
+            });
+        }
+        delete(opts) {
+            return __awaiter(this, void 0, void 0, function* () {
+                return this._req(Object.assign(Object.assign({}, opts), { method: "DELETE" }));
             });
         }
     }
     module.exports = {
         schema: SCHEMA,
         integration: RestIntegration,
+        AuthType: datasource_1.AuthType,
     };
 })(RestModule || (RestModule = {}));

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "1.0.24",
+    "version": "1.0.27-alpha.9",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -69,9 +69,10 @@
     "author": "Budibase",
     "license": "GPL-3.0",
     "dependencies": {
-        "@budibase/auth": "^1.0.24",
-        "@budibase/client": "^1.0.24",
-        "@budibase/string-templates": "^1.0.24",
+        "@apidevtools/swagger-parser": "^10.0.3",
+        "@budibase/auth": "^1.0.27-alpha.9",
+        "@budibase/client": "^1.0.27-alpha.9",
+        "@budibase/string-templates": "^1.0.27-alpha.9",
         "@bull-board/api": "^3.7.0",
         "@bull-board/koa": "^3.7.0",
         "@elastic/elasticsearch": "7.10.0",
@@ -85,12 +86,15 @@
         "bull": "^3.22.4",
         "chmodr": "1.2.0",
         "csvtojson": "2.0.10",
+        "curlconverter": "^3.21.0",
         "dotenv": "8.2.0",
         "download": "8.0.0",
         "fix-path": "3.0.0",
+        "form-data": "^4.0.0",
         "fs-extra": "8.1.0",
         "jimp": "0.16.1",
         "joi": "17.2.1",
+        "js-yaml": "^4.1.0",
         "jsonschema": "1.4.0",
         "knex": "^0.95.6",
         "koa": "2.7.0",
@@ -118,11 +122,13 @@
         "pouchdb-replication-stream": "1.2.9",
         "server-destroy": "1.0.1",
         "svelte": "^3.38.2",
+        "swagger-parser": "^10.0.3",
         "to-json-schema": "0.2.5",
         "uuid": "3.3.2",
         "validate.js": "0.13.1",
         "vm2": "^3.9.3",
         "worker-farm": "^1.7.0",
+        "xml2js": "^0.4.23",
         "yargs": "13.2.4",
         "zlib": "1.0.5"
     },
@@ -146,6 +152,7 @@
         "eslint": "^6.8.0",
         "jest": "^27.0.5",
         "nodemon": "^2.0.4",
+        "openapi-types": "^9.3.1",
         "path-to-regexp": "^6.2.0",
         "prettier": "^2.3.1",
         "rimraf": "^3.0.2",

package/dist/tests/utilities/TestConfiguration.js

@@ -294,6 +294,13 @@ class TestConfiguration {
             return this.datasource;
         });
     }
+    updateDatasource(datasource) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this._req(datasource, { datasourceId: datasource._id }, controllers.datasource.update);
+            this.datasource = response.datasource;
+            return this.datasource;
+        });
+    }
     createQuery(config = null) {
         return __awaiter(this, void 0, void 0, function* () {
             if (!this.datasource && !config) {

package/dist/threads/query.js

@@ -8,51 +8,159 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-require("./utils").threadSetup();
+const threadUtils = require("./utils");
+threadUtils.threadSetup();
 const ScriptRunner = require("../utilities/scriptRunner");
 const { integrations } = require("../integrations");
-function formatResponse(resp) {
-    if (typeof resp === "string") {
-        try {
-            resp = JSON.parse(resp);
-        }
-        catch (err) {
-            resp = { response: resp };
-        }
+const { processStringSync } = require("@budibase/string-templates");
+const CouchDB = require("../db");
+class QueryRunner {
+    constructor(input, flags = { noRecursiveQuery: false }) {
+        this.appId = input.appId;
+        this.datasource = input.datasource;
+        this.queryVerb = input.queryVerb;
+        this.fields = input.fields;
+        this.parameters = input.parameters;
+        this.transformer = input.transformer;
+        this.queryId = input.queryId;
+        this.noRecursiveQuery = flags.noRecursiveQuery;
+        this.cachedVariables = [];
+        // allows the response from a query to be stored throughout this
+        // execution so that if it needs to be re-used for another variable
+        // it can be
+        this.queryResponse = {};
+        this.hasRerun = false;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { datasource, fields, queryVerb, transformer } = this;
+            // pre-query, make sure datasource variables are added to parameters
+            const parameters = yield this.addDatasourceVariables();
+            const query = threadUtils.enrichQueryFields(fields, parameters);
+            const Integration = integrations[datasource.source];
+            if (!Integration) {
+                throw "Integration type does not exist.";
+            }
+            const integration = new Integration(datasource.config);
+            let output = threadUtils.formatResponse(yield integration[queryVerb](query));
+            let rows = output, info = undefined, extra = undefined;
+            if (threadUtils.hasExtraData(output)) {
+                rows = output.data;
+                info = output.info;
+                extra = output.extra;
+            }
+            // transform as required
+            if (transformer) {
+                const runner = new ScriptRunner(transformer, {
+                    data: rows,
+                    params: parameters,
+                });
+                rows = runner.execute();
+            }
+            // if the request fails we retry once, invalidating the cached value
+            if (info &&
+                info.code >= 400 &&
+                this.cachedVariables.length > 0 &&
+                !this.hasRerun) {
+                this.hasRerun = true;
+                // invalidate the cache value
+                yield threadUtils.invalidateDynamicVariables(this.cachedVariables);
+                return this.execute();
+            }
+            // needs to an array for next step
+            if (!Array.isArray(rows)) {
+                rows = [rows];
+            }
+            // map into JSON if just raw primitive here
+            if (rows.find(row => typeof row !== "object")) {
+                rows = rows.map(value => ({ value }));
+            }
+            // get all the potential fields in the schema
+            let keys = rows.flatMap(Object.keys);
+            if (integration.end) {
+                integration.end();
+            }
+            return { rows, keys, info, extra };
+        });
+    }
+    runAnotherQuery(queryId, parameters) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const db = new CouchDB(this.appId);
+            const query = yield db.get(queryId);
+            const datasource = yield db.get(query.datasourceId);
+            return new QueryRunner({
+                appId: this.appId,
+                datasource,
+                queryVerb: query.queryVerb,
+                fields: query.fields,
+                parameters,
+                transformer: query.transformer,
+            }, { noRecursiveQuery: true }).execute();
+        });
+    }
+    getDynamicVariable(variable) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { parameters } = this;
+            const queryId = variable.queryId, name = variable.name;
+            let value = yield threadUtils.checkCacheForDynamicVariable(queryId, name);
+            if (!value) {
+                value = this.queryResponse[queryId]
+                    ? this.queryResponse[queryId]
+                    : yield this.runAnotherQuery(queryId, parameters);
+                // store incase this query is to be called again
+                this.queryResponse[queryId] = value;
+                yield threadUtils.storeDynamicVariable(queryId, name, value);
+            }
+            else {
+                this.cachedVariables.push({ queryId, name });
+            }
+            return value;
+        });
+    }
+    addDatasourceVariables() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { datasource, parameters, fields } = this;
+            if (!datasource || !datasource.config) {
+                return parameters;
+            }
+            const staticVars = datasource.config.staticVariables || {};
+            const dynamicVars = datasource.config.dynamicVariables || [];
+            for (let [key, value] of Object.entries(staticVars)) {
+                if (!parameters[key]) {
+                    parameters[key] = value;
+                }
+            }
+            if (!this.noRecursiveQuery) {
+                // need to see if this uses any variables
+                const stringFields = JSON.stringify(fields);
+                const foundVars = dynamicVars.filter(variable => {
+                    // don't allow a query to use its own dynamic variable (loop)
+                    if (variable.queryId === this.queryId) {
+                        return false;
+                    }
+                    // look for {{ variable }} but allow spaces between handlebars
+                    const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`);
+                    return regex.test(stringFields);
+                });
+                const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar));
+                const responses = yield Promise.all(dynamics);
+                for (let i = 0; i < foundVars.length; i++) {
+                    const variable = foundVars[i];
+                    parameters[variable.name] = processStringSync(variable.value, {
+                        data: responses[i].rows,
+                        info: responses[i].extra,
+                    });
+                    // make sure its known that this uses dynamic variables in case it fails
+                    this.hasDynamicVariables = true;
+                }
+            }
+            return parameters;
+        });
     }
-    return resp;
-}
-function runAndTransform(datasource, queryVerb, query, transformer) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const Integration = integrations[datasource.source];
-        if (!Integration) {
-            throw "Integration type does not exist.";
-        }
-        const integration = new Integration(datasource.config);
-        let rows = formatResponse(yield integration[queryVerb](query));
-        // transform as required
-        if (transformer) {
-            const runner = new ScriptRunner(transformer, { data: rows });
-            rows = runner.execute();
-        }
-        // needs to an array for next step
-        if (!Array.isArray(rows)) {
-            rows = [rows];
-        }
-        // map into JSON if just raw primitive here
-        if (rows.find(row => typeof row !== "object")) {
-            rows = rows.map(value => ({ value }));
-        }
-        // get all the potential fields in the schema
-        let keys = rows.flatMap(Object.keys);
-        if (integration.end) {
-            integration.end();
-        }
-        return { rows, keys };
-    });
 }
 module.exports = (input, callback) => {
-    runAndTransform(input.datasource, input.queryVerb, input.query, input.transformer)
+    const Runner = new QueryRunner(input);
+    Runner.execute()
         .then(response => {
         callback(null, response);
     })