@budibase/server 1.0.27-alpha.13 → 1.0.27-alpha.17

package/dist/environment.js

@@ -57,6 +57,7 @@ module.exports = {
     DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
     ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
     DISABLE_THREADING: process.env.DISABLE_THREADING,
+    QUERY_THREAD_TIMEOUT: process.env.QUERY_THREAD_TIMEOUT,
     _set(key, value) {
         process.env[key] = value;
         module.exports[key] = value;

package/dist/integrations/dynamodb.js

@@ -77,6 +77,17 @@ var DynamoModule;
                     },
                 },
             },
+            describe: {
+                type: datasource_1.QueryTypes.FIELDS,
+                customisable: true,
+                readable: true,
+                fields: {
+                    table: {
+                        type: datasource_1.DatasourceFieldTypes.STRING,
+                        required: true,
+                    },
+                },
+            },
             get: {
                 type: datasource_1.QueryTypes.FIELDS,
                 customisable: true,
@@ -161,6 +172,14 @@ var DynamoModule;
                 return response;
             });
         }
+        describe(query) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const params = {
+                    TableName: query.table,
+                };
+                return new AWS.DynamoDB().describeTable(params).promise();
+            });
+        }
         get(query) {
             return __awaiter(this, void 0, void 0, function* () {
                 const params = Object.assign({ TableName: query.table }, query.json);

package/dist/integrations/rest.js

@@ -39,6 +39,9 @@ const coreFields = {
         type: datasource_1.DatasourceFieldTypes.STRING,
         enum: Object.values(BodyTypes),
     },
+    pagination: {
+        type: datasource_1.DatasourceFieldTypes.OBJECT,
+    },
 };
 var RestModule;
 (function (RestModule) {
@@ -103,7 +106,7 @@ var RestModule;
             this.startTimeMs = performance.now();
             this.config = config;
         }
-        parseResponse(response) {
+        parseResponse(response, pagination) {
             return __awaiter(this, void 0, void 0, function* () {
                 let data, raw, headers;
                 const contentType = response.headers.get("content-type") || "";
@@ -142,6 +145,11 @@ var RestModule;
                 for (let [key, value] of Object.entries(headers)) {
                     headers[key] = Array.isArray(value) ? value[0] : value;
                 }
+                // Check if a pagination cursor exists in the response
+                let nextCursor = null;
+                if (pagination === null || pagination === void 0 ? void 0 : pagination.responseParam) {
+                    nextCursor = data === null || data === void 0 ? void 0 : data[pagination.responseParam];
+                }
                 return {
                     data,
                     info: {
@@ -153,10 +161,31 @@ var RestModule;
                         raw,
                         headers,
                     },
+                    pagination: {
+                        cursor: nextCursor,
+                    },
                 };
             });
         }
-        getUrl(path, queryString) {
+        getUrl(path, queryString, pagination, paginationValues) {
+            // Add pagination params to query string if required
+            if ((pagination === null || pagination === void 0 ? void 0 : pagination.location) === "query" && paginationValues) {
+                const { pageParam, sizeParam } = pagination;
+                const params = new URLSearchParams();
+                // Append page number or cursor param if configured
+                if (pageParam && paginationValues.page != null) {
+                    params.append(pageParam, paginationValues.page);
+                }
+                // Append page size param if configured
+                if (sizeParam && paginationValues.limit != null) {
+                    params.append(sizeParam, paginationValues.limit);
+                }
+                // Prepend query string with pagination params
+                let paginationString = params.toString();
+                if (paginationString) {
+                    queryString = `${paginationString}&${queryString}`;
+                }
+            }
             const main = `${path}?${queryString}`;
             let complete = main;
             if (this.config.url && !main.startsWith("http")) {
@@ -167,21 +196,35 @@ var RestModule;
             }
             return complete;
         }
-        addBody(bodyType, body, input) {
-            let error, object, string;
+        addBody(bodyType, body, input, pagination, paginationValues) {
+            if (!input.headers) {
+                input.headers = {};
+            }
+            if (bodyType === BodyTypes.NONE) {
+                return input;
+            }
+            let error, object = {}, string = "";
             try {
-                string = typeof body !== "string" ? JSON.stringify(body) : body;
-                object = typeof body === "object" ? body : JSON.parse(body);
+                if (body) {
+                    string = typeof body !== "string" ? JSON.stringify(body) : body;
+                    object = typeof body === "object" ? body : JSON.parse(body);
+                }
             }
             catch (err) {
                 error = err;
             }
-            if (!input.headers) {
-                input.headers = {};
-            }
+            // Util to add pagination values to a certain body type
+            const addPaginationToBody = (insertFn) => {
+                if ((pagination === null || pagination === void 0 ? void 0 : pagination.location) === "body") {
+                    if ((pagination === null || pagination === void 0 ? void 0 : pagination.pageParam) && (paginationValues === null || paginationValues === void 0 ? void 0 : paginationValues.page) != null) {
+                        insertFn(pagination.pageParam, paginationValues.page);
+                    }
+                    if ((pagination === null || pagination === void 0 ? void 0 : pagination.sizeParam) && (paginationValues === null || paginationValues === void 0 ? void 0 : paginationValues.limit) != null) {
+                        insertFn(pagination.sizeParam, paginationValues.limit);
+                    }
+                }
+            };
             switch (bodyType) {
-                case BodyTypes.NONE:
-                    break;
                 case BodyTypes.TEXT:
                     // content type defaults to plaintext
                     input.body = string;
@@ -191,6 +234,9 @@ var RestModule;
                     for (let [key, value] of Object.entries(object)) {
                         params.append(key, value);
                     }
+                    addPaginationToBody((key, value) => {
+                        params.append(key, value);
+                    });
                     input.body = params;
                     break;
                 case BodyTypes.FORM_DATA:
@@ -198,6 +244,9 @@ var RestModule;
                     for (let [key, value] of Object.entries(object)) {
                         form.append(key, value);
                     }
+                    addPaginationToBody((key, value) => {
+                        form.append(key, value);
+                    });
                     input.body = form;
                     break;
                 case BodyTypes.XML:
@@ -207,13 +256,15 @@ var RestModule;
                     input.body = string;
                     input.headers["Content-Type"] = "application/xml";
                     break;
-                default:
                 case BodyTypes.JSON:
                     // if JSON error, throw it
                     if (error) {
                         throw "Invalid JSON for request body";
                     }
-                    input.body = string;
+                    addPaginationToBody((key, value) => {
+                        object[key] = value;
+                    });
+                    input.body = JSON.stringify(object);
                     input.headers["Content-Type"] = "application/json";
                     break;
             }
@@ -243,7 +294,7 @@ var RestModule;
         }
         _req(query) {
             return __awaiter(this, void 0, void 0, function* () {
-                const { path = "", queryString = "", headers = {}, method = "GET", disabledHeaders, bodyType, requestBody, authConfigId, } = query;
+                const { path = "", queryString = "", headers = {}, method = "GET", disabledHeaders, bodyType, requestBody, authConfigId, pagination, paginationValues, } = query;
                 const authHeaders = this.getAuthHeaders(authConfigId);
                 this.headers = Object.assign(Object.assign(Object.assign({}, this.config.defaultHeaders), headers), authHeaders);
                 if (disabledHeaders) {
@@ -254,13 +305,11 @@ var RestModule;
                     }
                 }
                 let input = { method, headers: this.headers };
-                if (requestBody) {
-                    input = this.addBody(bodyType, requestBody, input);
-                }
+                input = this.addBody(bodyType, requestBody, input, pagination, paginationValues);
                 this.startTimeMs = performance.now();
-                const url = this.getUrl(path, queryString);
+                const url = this.getUrl(path, queryString, pagination, paginationValues);
                 const response = yield fetch(url, input);
-                return yield this.parseResponse(response);
+                return yield this.parseResponse(response, pagination);
             });
         }
         create(opts) {

package/dist/integrations/s3.js

@@ -20,7 +20,7 @@ var S3Module;
         datasource: {
             region: {
                 type: "string",
-                required: true,
+                required: false,
                 default: "us-east-1",
             },
             accessKeyId: {
@@ -31,6 +31,15 @@ var S3Module;
                 type: "password",
                 required: true,
             },
+            endpoint: {
+                type: "string",
+                required: false,
+            },
+            signatureVersion: {
+                type: "string",
+                required: false,
+                default: "v4"
+            },
         },
         query: {
             read: {
@@ -47,13 +56,13 @@ var S3Module;
     class S3Integration {
         constructor(config) {
             this.config = config;
-            this.connectionPromise = this.connect();
-            this.client = new AWS.S3();
-        }
-        connect() {
-            return __awaiter(this, void 0, void 0, function* () {
-                AWS.config.update(this.config);
-            });
+            if (this.config.endpoint) {
+                this.config.s3ForcePathStyle = true;
+            }
+            else {
+                delete this.config.endpoint;
+            }
+            this.client = new AWS.S3(this.config);
         }
         read(query) {
             return __awaiter(this, void 0, void 0, function* () {

package/dist/integrations/utils.js

@@ -2,39 +2,39 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.finaliseExternalTables = exports.isIsoDateString = exports.isSQL = exports.getSqlQuery = exports.convertSqlType = exports.breakRowIdField = exports.convertRowId = exports.isRowId = exports.generateRowIdField = exports.breakExternalTableId = exports.buildExternalTableId = exports.isExternalTable = exports.SqlClients = void 0;
 const datasource_1 = require("../definitions/datasource");
-const { DocumentTypes, SEPARATOR } = require("../db/utils");
-const { FieldTypes, BuildSchemaErrors, InvalidColumns, } = require("../constants");
-const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`;
+const utils_1 = require("../db/utils");
+const constants_1 = require("../constants");
+const DOUBLE_SEPARATOR = `${utils_1.SEPARATOR}${utils_1.SEPARATOR}`;
 const ROW_ID_REGEX = /^\[.*]$/g;
 const SQL_TYPE_MAP = {
-    text: FieldTypes.LONGFORM,
-    varchar: FieldTypes.STRING,
-    integer: FieldTypes.NUMBER,
-    bigint: FieldTypes.NUMBER,
-    decimal: FieldTypes.NUMBER,
-    smallint: FieldTypes.NUMBER,
-    real: FieldTypes.NUMBER,
-    "double precision": FieldTypes.NUMBER,
-    timestamp: FieldTypes.DATETIME,
-    time: FieldTypes.DATETIME,
-    boolean: FieldTypes.BOOLEAN,
-    json: FieldTypes.JSON,
-    date: FieldTypes.DATETIME,
-    blob: FieldTypes.LONGFORM,
-    enum: FieldTypes.STRING,
-    float: FieldTypes.NUMBER,
-    int: FieldTypes.NUMBER,
-    numeric: FieldTypes.NUMBER,
-    mediumint: FieldTypes.NUMBER,
-    dec: FieldTypes.NUMBER,
-    double: FieldTypes.NUMBER,
-    fixed: FieldTypes.NUMBER,
-    datetime: FieldTypes.DATETIME,
-    tinyint: FieldTypes.BOOLEAN,
-    long: FieldTypes.LONGFORM,
-    number: FieldTypes.NUMBER,
-    binary_float: FieldTypes.NUMBER,
-    binary_double: FieldTypes.NUMBER,
+    text: constants_1.FieldTypes.LONGFORM,
+    varchar: constants_1.FieldTypes.STRING,
+    integer: constants_1.FieldTypes.NUMBER,
+    bigint: constants_1.FieldTypes.NUMBER,
+    decimal: constants_1.FieldTypes.NUMBER,
+    smallint: constants_1.FieldTypes.NUMBER,
+    real: constants_1.FieldTypes.NUMBER,
+    "double precision": constants_1.FieldTypes.NUMBER,
+    timestamp: constants_1.FieldTypes.DATETIME,
+    time: constants_1.FieldTypes.DATETIME,
+    boolean: constants_1.FieldTypes.BOOLEAN,
+    json: constants_1.FieldTypes.JSON,
+    date: constants_1.FieldTypes.DATETIME,
+    blob: constants_1.FieldTypes.LONGFORM,
+    enum: constants_1.FieldTypes.STRING,
+    float: constants_1.FieldTypes.NUMBER,
+    int: constants_1.FieldTypes.NUMBER,
+    numeric: constants_1.FieldTypes.NUMBER,
+    mediumint: constants_1.FieldTypes.NUMBER,
+    dec: constants_1.FieldTypes.NUMBER,
+    double: constants_1.FieldTypes.NUMBER,
+    fixed: constants_1.FieldTypes.NUMBER,
+    datetime: constants_1.FieldTypes.DATETIME,
+    tinyint: constants_1.FieldTypes.BOOLEAN,
+    long: constants_1.FieldTypes.LONGFORM,
+    number: constants_1.FieldTypes.NUMBER,
+    binary_float: constants_1.FieldTypes.NUMBER,
+    binary_double: constants_1.FieldTypes.NUMBER,
 };
 var SqlClients;
 (function (SqlClients) {
@@ -44,7 +44,7 @@ var SqlClients;
     SqlClients["ORACLE"] = "oracledb";
 })(SqlClients = exports.SqlClients || (exports.SqlClients = {}));
 function isExternalTable(tableId) {
-    return tableId.includes(DocumentTypes.DATASOURCE);
+    return tableId.includes(utils_1.DocumentTypes.DATASOURCE);
 }
 exports.isExternalTable = isExternalTable;
 function buildExternalTableId(datasourceId, tableName) {
@@ -109,7 +109,7 @@ function convertSqlType(type) {
             return internal;
         }
     }
-    return FieldTypes.STRING;
+    return constants_1.FieldTypes.STRING;
 }
 exports.convertSqlType = convertSqlType;
 function getSqlQuery(query) {
@@ -148,7 +148,10 @@ function copyExistingPropsOver(tableName, table, entities) {
             if (!existingTableSchema.hasOwnProperty(key)) {
                 continue;
             }
-            if (existingTableSchema[key].type === "link") {
+            if (existingTableSchema[key].type === constants_1.FieldTypes.LINK ||
+                existingTableSchema[key].type === constants_1.FieldTypes.OPTIONS ||
+                ((!table.schema[key] || table.schema[key].type === constants_1.FieldTypes.NUMBER) &&
+                    existingTableSchema[key].type === constants_1.FieldTypes.BOOLEAN)) {
                 table.schema[key] = existingTableSchema[key];
             }
         }
@@ -156,18 +159,18 @@ function copyExistingPropsOver(tableName, table, entities) {
     return table;
 }
 function finaliseExternalTables(tables, entities) {
-    const invalidColumns = Object.values(InvalidColumns);
+    const invalidColumns = Object.values(constants_1.InvalidColumns);
     let finalTables = {};
     const errors = {};
     for (let [name, table] of Object.entries(tables)) {
         const schemaFields = Object.keys(table.schema);
         // make sure every table has a key
         if (table.primary == null || table.primary.length === 0) {
-            errors[name] = BuildSchemaErrors.NO_KEY;
+            errors[name] = constants_1.BuildSchemaErrors.NO_KEY;
             continue;
         }
         else if (schemaFields.find(field => invalidColumns.includes(field))) {
-            errors[name] = BuildSchemaErrors.INVALID_COLUMN;
+            errors[name] = constants_1.BuildSchemaErrors.INVALID_COLUMN;
             continue;
         }
         // make sure all previous props have been added back

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "1.0.27-alpha.12",
+    "version": "1.0.27-alpha.16",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -70,9 +70,9 @@
     "license": "GPL-3.0",
     "dependencies": {
         "@apidevtools/swagger-parser": "^10.0.3",
-        "@budibase/backend-core": "^1.0.27-alpha.12",
-        "@budibase/client": "^1.0.27-alpha.12",
-        "@budibase/string-templates": "^1.0.27-alpha.12",
+        "@budibase/backend-core": "^1.0.27-alpha.16",
+        "@budibase/client": "^1.0.27-alpha.16",
+        "@budibase/string-templates": "^1.0.27-alpha.16",
         "@bull-board/api": "^3.7.0",
         "@bull-board/koa": "^3.7.0",
         "@elastic/elasticsearch": "7.10.0",

package/dist/threads/query.js

@@ -14,6 +14,8 @@ const ScriptRunner = require("../utilities/scriptRunner");
 const { integrations } = require("../integrations");
 const { processStringSync } = require("@budibase/string-templates");
 const CouchDB = require("../db");
+const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/);
+const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/);
 class QueryRunner {
     constructor(input, flags = { noRecursiveQuery: false }) {
         this.appId = input.appId;
@@ -21,6 +23,7 @@ class QueryRunner {
         this.queryVerb = input.queryVerb;
         this.fields = input.fields;
         this.parameters = input.parameters;
+        this.pagination = input.pagination;
         this.transformer = input.transformer;
         this.queryId = input.queryId;
         this.noRecursiveQuery = flags.noRecursiveQuery;
@@ -36,18 +39,23 @@ class QueryRunner {
             let { datasource, fields, queryVerb, transformer } = this;
             // pre-query, make sure datasource variables are added to parameters
             const parameters = yield this.addDatasourceVariables();
-            const query = threadUtils.enrichQueryFields(fields, parameters);
+            let query = this.enrichQueryFields(fields, parameters);
+            // Add pagination values for REST queries
+            if (this.pagination) {
+                query.paginationValues = this.pagination;
+            }
             const Integration = integrations[datasource.source];
             if (!Integration) {
                 throw "Integration type does not exist.";
             }
             const integration = new Integration(datasource.config);
             let output = threadUtils.formatResponse(yield integration[queryVerb](query));
-            let rows = output, info = undefined, extra = undefined;
+            let rows = output, info = undefined, extra = undefined, pagination = undefined;
             if (threadUtils.hasExtraData(output)) {
                 rows = output.data;
                 info = output.info;
                 extra = output.extra;
+                pagination = output.pagination;
             }
             // transform as required
             if (transformer) {
@@ -80,7 +88,7 @@ class QueryRunner {
             if (integration.end) {
                 integration.end();
             }
-            return { rows, keys, info, extra };
+            return { rows, keys, info, extra, pagination };
         });
     }
     runAnotherQuery(queryId, parameters) {
@@ -157,6 +165,47 @@ class QueryRunner {
             return parameters;
         });
     }
+    enrichQueryFields(fields, parameters = {}) {
+        const enrichedQuery = {};
+        // enrich the fields with dynamic parameters
+        for (let key of Object.keys(fields)) {
+            if (fields[key] == null) {
+                continue;
+            }
+            if (typeof fields[key] === "object") {
+                // enrich nested fields object
+                enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters);
+            }
+            else if (typeof fields[key] === "string") {
+                // enrich string value as normal
+                let value = fields[key];
+                // add triple brace to avoid escaping e.g. '=' in cookie header
+                if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
+                    value = `{${value}}`;
+                }
+                enrichedQuery[key] = processStringSync(value, parameters, {
+                    noHelpers: true,
+                });
+            }
+            else {
+                enrichedQuery[key] = fields[key];
+            }
+        }
+        if (enrichedQuery.json ||
+            enrichedQuery.customData ||
+            enrichedQuery.requestBody) {
+            try {
+                enrichedQuery.json = JSON.parse(enrichedQuery.json ||
+                    enrichedQuery.customData ||
+                    enrichedQuery.requestBody);
+            }
+            catch (err) {
+                // no json found, ignore
+            }
+            delete enrichedQuery.customData;
+        }
+        return enrichedQuery;
+    }
 }
 module.exports = (input, callback) => {
     const Runner = new QueryRunner(input);

package/dist/threads/utils.js

@@ -13,11 +13,8 @@ const CouchDB = require("../db");
 const { init } = require("@budibase/backend-core");
 const redis = require("@budibase/backend-core/redis");
 const { SEPARATOR } = require("@budibase/backend-core/db");
-const { processStringSync } = require("@budibase/string-templates");
 const VARIABLE_TTL_SECONDS = 3600;
 let client;
-const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/);
-const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/);
 function getClient() {
     return __awaiter(this, void 0, void 0, function* () {
         if (!client) {
@@ -75,44 +72,3 @@ exports.hasExtraData = response => {
         response.data != null &&
         response.info != null);
 };
-exports.enrichQueryFields = (fields, parameters = {}) => {
-    const enrichedQuery = {};
-    // enrich the fields with dynamic parameters
-    for (let key of Object.keys(fields)) {
-        if (fields[key] == null) {
-            continue;
-        }
-        if (typeof fields[key] === "object") {
-            // enrich nested fields object
-            enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters);
-        }
-        else if (typeof fields[key] === "string") {
-            // enrich string value as normal
-            let value = fields[key];
-            // add triple brace to avoid escaping e.g. '=' in cookie header
-            if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
-                value = `{${value}}`;
-            }
-            enrichedQuery[key] = processStringSync(value, parameters, {
-                noHelpers: true,
-            });
-        }
-        else {
-            enrichedQuery[key] = fields[key];
-        }
-    }
-    if (enrichedQuery.json ||
-        enrichedQuery.customData ||
-        enrichedQuery.requestBody) {
-        try {
-            enrichedQuery.json = JSON.parse(enrichedQuery.json ||
-                enrichedQuery.customData ||
-                enrichedQuery.requestBody);
-        }
-        catch (err) {
-            // no json found, ignore
-        }
-        delete enrichedQuery.customData;
-    }
-    return enrichedQuery;
-};