@budibase/server 0.0.32 → 0.0.999-alpha.30

package/dist/threads/query.js

@@ -0,0 +1,265 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.execute = void 0;
+const utils_1 = __importDefault(require("./utils"));
+utils_1.default.threadSetup();
+const scriptRunner_1 = __importDefault(require("../utilities/scriptRunner"));
+const integrations_1 = require("../integrations");
+const string_templates_1 = require("@budibase/string-templates");
+const backend_core_1 = require("@budibase/backend-core");
+const utils_2 = require("../db/utils");
+const sdk_1 = __importDefault(require("../sdk"));
+const fp_1 = require("lodash/fp");
+const utils_3 = require("../integrations/utils");
+const sql_1 = require("../integrations/queries/sql");
+class QueryRunner {
+    constructor(input, flags = { noRecursiveQuery: false }) {
+        this.datasource = input.datasource;
+        this.queryVerb = input.queryVerb;
+        this.fields = input.fields;
+        this.parameters = input.parameters;
+        this.pagination = input.pagination;
+        this.transformer = input.transformer;
+        this.queryId = input.queryId;
+        this.noRecursiveQuery = flags.noRecursiveQuery;
+        this.cachedVariables = [];
+        // Additional context items for enrichment
+        this.ctx = input.ctx;
+        // allows the response from a query to be stored throughout this
+        // execution so that if it needs to be re-used for another variable
+        // it can be
+        this.queryResponse = {};
+        this.hasRerun = false;
+        this.hasRefreshedOAuth = false;
+        this.hasDynamicVariables = false;
+    }
+    execute() {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            let { datasource, fields, queryVerb, transformer } = this;
+            let datasourceClone = (0, fp_1.cloneDeep)(datasource);
+            let fieldsClone = (0, fp_1.cloneDeep)(fields);
+            const Integration = yield (0, integrations_1.getIntegration)(datasourceClone.source);
+            if (!Integration) {
+                throw "Integration type does not exist.";
+            }
+            if (datasourceClone.config.authConfigs) {
+                const updatedConfigs = [];
+                for (let config of datasourceClone.config.authConfigs) {
+                    updatedConfigs.push(yield sdk_1.default.queries.enrichContext(config, this.ctx));
+                }
+                datasourceClone.config.authConfigs = updatedConfigs;
+            }
+            const integration = new Integration(datasourceClone.config);
+            // pre-query, make sure datasource variables are added to parameters
+            const parameters = yield this.addDatasourceVariables();
+            // Enrich the parameters with the addition context items.
+            // 'user' is now a reserved variable key in mapping parameters
+            const enrichedParameters = yield sdk_1.default.queries.enrichContext(parameters, this.ctx);
+            const enrichedContext = Object.assign(Object.assign({}, enrichedParameters), this.ctx);
+            // Parse global headers
+            if (datasourceClone.config.defaultHeaders) {
+                datasourceClone.config.defaultHeaders = yield sdk_1.default.queries.enrichContext(datasourceClone.config.defaultHeaders, enrichedContext);
+            }
+            let query;
+            // handle SQL injections by interpolating the variables
+            if ((0, utils_3.isSQL)(datasourceClone)) {
+                query = yield (0, sql_1.interpolateSQL)(fieldsClone, enrichedContext, integration);
+            }
+            else {
+                query = yield sdk_1.default.queries.enrichContext(fieldsClone, enrichedContext);
+            }
+            // Add pagination values for REST queries
+            if (this.pagination) {
+                query.paginationValues = this.pagination;
+            }
+            let output = utils_1.default.formatResponse(yield integration[queryVerb](query));
+            let rows = output, info = undefined, extra = undefined, pagination = undefined;
+            if (utils_1.default.hasExtraData(output)) {
+                rows = output.data;
+                info = output.info;
+                extra = output.extra;
+                pagination = output.pagination;
+            }
+            // transform as required
+            if (transformer) {
+                const runner = new scriptRunner_1.default(transformer, {
+                    data: rows,
+                    params: enrichedParameters,
+                });
+                rows = runner.execute();
+            }
+            // if the request fails we retry once, invalidating the cached value
+            if (info && info.code >= 400 && !this.hasRerun) {
+                if (((_a = this.ctx.user) === null || _a === void 0 ? void 0 : _a.provider) &&
+                    info.code === 401 &&
+                    !this.hasRefreshedOAuth) {
+                    yield this.refreshOAuth2(this.ctx);
+                    // Attempt to refresh the access token from the provider
+                    this.hasRefreshedOAuth = true;
+                }
+                else {
+                    this.hasRerun = true;
+                }
+                yield utils_1.default.invalidateDynamicVariables(this.cachedVariables);
+                return this.execute();
+            }
+            // check for undefined response
+            if (!rows) {
+                rows = [];
+            }
+            // needs to an array for next step
+            if (!Array.isArray(rows)) {
+                rows = [rows];
+            }
+            // map into JSON if just raw primitive here
+            if (rows.find((row) => typeof row !== "object")) {
+                rows = rows.map((value) => ({ value }));
+            }
+            // get all the potential fields in the schema
+            let keys = rows.flatMap(Object.keys);
+            if (integration.end) {
+                integration.end();
+            }
+            return { rows, keys, info, extra, pagination };
+        });
+    }
+    runAnotherQuery(queryId, parameters) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const db = backend_core_1.context.getAppDB();
+            const query = yield db.get(queryId);
+            const datasource = yield sdk_1.default.datasources.get(query.datasourceId, {
+                enriched: true,
+            });
+            return new QueryRunner({
+                datasource,
+                queryVerb: query.queryVerb,
+                fields: query.fields,
+                parameters,
+                transformer: query.transformer,
+                queryId,
+                ctx: this.ctx,
+            }, { noRecursiveQuery: true }).execute();
+        });
+    }
+    refreshOAuth2(ctx) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { oauth2, providerType, _id } = ctx.user;
+            const { configId } = ctx.auth;
+            if (!providerType || !(oauth2 === null || oauth2 === void 0 ? void 0 : oauth2.refreshToken)) {
+                throw new Error("No refresh token found for authenticated user");
+            }
+            const resp = yield backend_core_1.auth.refreshOAuthToken(oauth2.refreshToken, providerType, configId);
+            // Refresh session flow. Should be in same location as refreshOAuthToken
+            // There are several other properties available in 'resp'
+            if (!resp.err) {
+                const globalUserId = (0, utils_2.getGlobalIDFromUserMetadataID)(_id);
+                yield backend_core_1.auth.updateUserOAuth(globalUserId, resp);
+                this.ctx.user = yield backend_core_1.cache.user.getUser(globalUserId);
+            }
+            else {
+                // In this event the user may have oAuth issues that
+                // could require re-authenticating with their provider.
+                let errorMessage = resp.err.data ? resp.err.data : resp.err.toString();
+                throw new Error("OAuth2 access token could not be refreshed: " + errorMessage);
+            }
+            return resp;
+        });
+    }
+    getDynamicVariable(variable) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { parameters } = this;
+            const queryId = variable.queryId, name = variable.name;
+            let value = yield utils_1.default.checkCacheForDynamicVariable(queryId, name);
+            if (!value) {
+                value = this.queryResponse[queryId]
+                    ? this.queryResponse[queryId]
+                    : yield this.runAnotherQuery(queryId, parameters);
+                // store incase this query is to be called again
+                this.queryResponse[queryId] = value;
+                yield utils_1.default.storeDynamicVariable(queryId, name, value);
+            }
+            else {
+                this.cachedVariables.push({ queryId, name });
+            }
+            return value;
+        });
+    }
+    addDatasourceVariables() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let { datasource, parameters, fields } = this;
+            if (!datasource || !datasource.config) {
+                return parameters;
+            }
+            const staticVars = datasource.config.staticVariables || {};
+            const dynamicVars = datasource.config.dynamicVariables || [];
+            for (let [key, value] of Object.entries(staticVars)) {
+                if (!parameters[key]) {
+                    parameters[key] = value;
+                }
+            }
+            if (!this.noRecursiveQuery) {
+                // need to see if this uses any variables
+                const stringFields = JSON.stringify(fields);
+                const foundVars = dynamicVars.filter((variable) => {
+                    // don't allow a query to use its own dynamic variable (loop)
+                    if (variable.queryId === this.queryId) {
+                        return false;
+                    }
+                    // look for {{ variable }} but allow spaces between handlebars
+                    const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`);
+                    return regex.test(stringFields);
+                });
+                const dynamics = foundVars.map((dynVar) => this.getDynamicVariable(dynVar));
+                const responses = yield Promise.all(dynamics);
+                for (let i = 0; i < foundVars.length; i++) {
+                    const variable = foundVars[i];
+                    parameters[variable.name] = (0, string_templates_1.processStringSync)(variable.value, {
+                        data: responses[i].rows,
+                        info: responses[i].extra,
+                    }, {
+                        escapeNewlines: true,
+                    });
+                    // make sure its known that this uses dynamic variables in case it fails
+                    this.hasDynamicVariables = true;
+                }
+            }
+            return parameters;
+        });
+    }
+}
+function execute(input, callback) {
+    const run = () => __awaiter(this, void 0, void 0, function* () {
+        const Runner = new QueryRunner(input);
+        try {
+            const response = yield Runner.execute();
+            callback(null, response);
+        }
+        catch (err) {
+            callback(err);
+        }
+    });
+    backend_core_1.context.doInAppContext(input.appId, () => __awaiter(this, void 0, void 0, function* () {
+        if (input.environmentVariables) {
+            return backend_core_1.context.doInEnvironmentContext(input.environmentVariables, () => {
+                return run();
+            });
+        }
+        else {
+            return run();
+        }
+    }));
+}
+exports.execute = execute;

package/dist/threads/utils.js

@@ -0,0 +1,122 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasExtraData = exports.formatResponse = exports.storeDynamicVariable = exports.invalidateDynamicVariables = exports.checkCacheForDynamicVariable = exports.threadSetup = void 0;
+const environment_1 = __importDefault(require("../environment"));
+const db = __importStar(require("../db"));
+const backend_core_1 = require("@budibase/backend-core");
+const VARIABLE_TTL_SECONDS = 3600;
+let client;
+function getClient() {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!client) {
+            client = yield new backend_core_1.redis.Client(backend_core_1.redis.utils.Databases.QUERY_VARS).init();
+        }
+        return client;
+    });
+}
+process.on("exit", () => __awaiter(void 0, void 0, void 0, function* () {
+    if (client) {
+        yield client.finish();
+    }
+}));
+function makeVariableKey(queryId, variable) {
+    return `${queryId}${backend_core_1.db.SEPARATOR}${variable}`;
+}
+function threadSetup() {
+    // don't run this if not threading
+    if (environment_1.default.isTest() || environment_1.default.DISABLE_THREADING) {
+        return;
+    }
+    // when thread starts, make sure it is recorded
+    environment_1.default.setInThread();
+    db.init();
+}
+exports.threadSetup = threadSetup;
+function checkCacheForDynamicVariable(queryId, variable) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const cache = yield getClient();
+        return cache.get(makeVariableKey(queryId, variable));
+    });
+}
+exports.checkCacheForDynamicVariable = checkCacheForDynamicVariable;
+function invalidateDynamicVariables(cachedVars) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const cache = yield getClient();
+        let promises = [];
+        for (let variable of cachedVars) {
+            promises.push(cache.delete(makeVariableKey(variable.queryId, variable.name)));
+        }
+        yield Promise.all(promises);
+    });
+}
+exports.invalidateDynamicVariables = invalidateDynamicVariables;
+function storeDynamicVariable(queryId, variable, value) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const cache = yield getClient();
+        yield cache.store(makeVariableKey(queryId, variable), value, VARIABLE_TTL_SECONDS);
+    });
+}
+exports.storeDynamicVariable = storeDynamicVariable;
+function formatResponse(resp) {
+    if (typeof resp === "string") {
+        try {
+            resp = JSON.parse(resp);
+        }
+        catch (err) {
+            resp = { response: resp };
+        }
+    }
+    return resp;
+}
+exports.formatResponse = formatResponse;
+function hasExtraData(response) {
+    return (typeof response === "object" &&
+        !Array.isArray(response) &&
+        response &&
+        response.data != null &&
+        response.info != null);
+}
+exports.hasExtraData = hasExtraData;
+exports.default = {
+    hasExtraData,
+    formatResponse,
+    storeDynamicVariable,
+    invalidateDynamicVariables,
+    checkCacheForDynamicVariable,
+    threadSetup,
+};