@budibase/server 1.0.27-alpha.13 → 1.0.27-alpha.14

package/dist/utilities/index.js

@@ -38,7 +38,7 @@ exports.checkSlashesInUrl = url => {
  * @return {string} The base URL of the object store (MinIO or S3).
  */
 exports.objectStoreUrl = () => {
-    if (env.SELF_HOSTED) {
+    if (env.SELF_HOSTED || env.MINIO_URL) {
         // can use a relative url for this as all goes through the proxy (this is hosted in minio)
         return OBJ_STORE_DIRECTORY;
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.27-alpha.13",
+  "version": "1.0.27-alpha.14",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.27-alpha.13",
-    "@budibase/client": "^1.0.27-alpha.13",
-    "@budibase/string-templates": "^1.0.27-alpha.13",
+    "@budibase/backend-core": "^1.0.27-alpha.14",
+    "@budibase/client": "^1.0.27-alpha.14",
+    "@budibase/string-templates": "^1.0.27-alpha.14",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -165,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "e48e21ffb846288ce46883674626b79755617445"
+  "gitHead": "f9cd961be007bc9c360e15c3c09ec6889ed47b51"
 }

package/src/api/controllers/query/index.js

@@ -140,11 +140,12 @@ async function execute(ctx, opts = { rowsOnly: false }) {
 
   // call the relevant CRUD method on the integration class
   try {
-    const { rows, extra } = await Runner.run({
+    const { rows, pagination, extra } = await Runner.run({
       appId: ctx.appId,
       datasource,
       queryVerb: query.queryVerb,
       fields: query.fields,
+      pagination: ctx.request.body.pagination,
       parameters: ctx.request.body.parameters,
       transformer: query.transformer,
       queryId: ctx.params.queryId,
@@ -152,7 +153,7 @@ async function execute(ctx, opts = { rowsOnly: false }) {
     if (opts && opts.rowsOnly) {
       ctx.body = rows
     } else {
-      ctx.body = { data: rows, ...extra }
+      ctx.body = { data: rows, pagination, ...extra }
     }
   } catch (err) {
     ctx.throw(400, err)

package/src/definitions/datasource.ts

@@ -232,6 +232,8 @@ export interface RestQueryFields {
   json: object
   method: string
   authConfigId: string
+  pagination: PaginationConfig | null
+  paginationValues: PaginationValues | null
 }
 
 export interface RestConfig {
@@ -252,6 +254,19 @@ export interface RestConfig {
   ]
 }
 
+export interface PaginationConfig {
+  type: string
+  location: string
+  pageParam: string
+  sizeParam: string | null
+  responseParam: string | null
+}
+
+export interface PaginationValues {
+  page: string | number | null
+  limit: number | null
+}
+
 export interface Query {
   _id?: string
   datasourceId: string

package/src/environment.js

@@ -65,6 +65,7 @@ module.exports = {
   DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
+  QUERY_THREAD_TIMEOUT: process.env.QUERY_THREAD_TIMEOUT,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

package/src/integrations/dynamodb.ts

@@ -80,6 +80,17 @@ module DynamoModule {
           },
         },
       },
+      describe: {
+        type: QueryTypes.FIELDS,
+        customisable: true,
+        readable: true,
+        fields: {
+          table: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
       get: {
         type: QueryTypes.FIELDS,
         customisable: true,
@@ -180,6 +191,13 @@ module DynamoModule {
       return response
     }
 
+    async describe(query: { table: string }) {
+      const params = {
+        TableName: query.table,
+      }
+      return new AWS.DynamoDB().describeTable(params).promise()
+    }
+
     async get(query: { table: string; json: object }) {
       const params = {
         TableName: query.table,

package/src/integrations/rest.ts

@@ -4,9 +4,11 @@ import {
   QueryTypes,
   RestConfig,
   RestQueryFields as RestQuery,
+  PaginationConfig,
   AuthType,
   BasicAuthConfig,
   BearerAuthConfig,
+  PaginationValues,
 } from "../definitions/datasource"
 import { IntegrationBase } from "./base/IntegrationBase"
 
@@ -40,6 +42,9 @@ const coreFields = {
     type: DatasourceFieldTypes.STRING,
     enum: Object.values(BodyTypes),
   },
+  pagination: {
+    type: DatasourceFieldTypes.OBJECT
+  }
 }
 
 module RestModule {
@@ -115,7 +120,7 @@ module RestModule {
       this.config = config
     }
 
-    async parseResponse(response: any) {
+    async parseResponse(response: any, pagination: PaginationConfig | null) {
       let data, raw, headers
       const contentType = response.headers.get("content-type") || ""
       try {
@@ -154,6 +159,13 @@ module RestModule {
       for (let [key, value] of Object.entries(headers)) {
         headers[key] = Array.isArray(value) ? value[0] : value
       }
+
+      // Check if a pagination cursor exists in the response
+      let nextCursor = null
+      if (pagination?.responseParam) {
+        nextCursor = data?.[pagination.responseParam]
+      }
+
       return {
         data,
         info: {
@@ -165,10 +177,35 @@ module RestModule {
           raw,
           headers,
         },
+        pagination: {
+          cursor: nextCursor
+        }
       }
     }
 
-    getUrl(path: string, queryString: string): string {
+    getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string {
+      // Add pagination params to query string if required
+      if (pagination?.location === "query" && paginationValues) {
+        const { pageParam, sizeParam } = pagination
+        const params = new URLSearchParams()
+
+        // Append page number or cursor param if configured
+        if (pageParam && paginationValues.page != null) {
+          params.append(pageParam, paginationValues.page)
+        }
+
+        // Append page size param if configured
+        if (sizeParam && paginationValues.limit != null) {
+          params.append(sizeParam, paginationValues.limit)
+        }
+
+        // Prepend query string with pagination params
+        let paginationString = params.toString()
+        if (paginationString) {
+          queryString = `${paginationString}&${queryString}`
+        }
+      }
+
       const main = `${path}?${queryString}`
       let complete = main
       if (this.config.url && !main.startsWith("http")) {
@@ -180,20 +217,36 @@ module RestModule {
       return complete
     }
 
-    addBody(bodyType: string, body: string | any, input: any) {
-      let error, object, string
+    addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) {
+      if (!input.headers) {
+        input.headers = {}
+      }
+      if (bodyType === BodyTypes.NONE) {
+        return input
+      }
+      let error, object: any = {}, string = ""
       try {
-        string = typeof body !== "string" ? JSON.stringify(body) : body
-        object = typeof body === "object" ? body : JSON.parse(body)
+        if (body) {
+          string = typeof body !== "string" ? JSON.stringify(body) : body
+          object = typeof body === "object" ? body : JSON.parse(body)
+        }
       } catch (err) {
         error = err
       }
-      if (!input.headers) {
-        input.headers = {}
+
+      // Util to add pagination values to a certain body type
+      const addPaginationToBody = (insertFn: Function) => {
+        if (pagination?.location === "body") {
+          if (pagination?.pageParam && paginationValues?.page != null) {
+            insertFn(pagination.pageParam, paginationValues.page)
+          }
+          if (pagination?.sizeParam && paginationValues?.limit != null) {
+            insertFn(pagination.sizeParam, paginationValues.limit)
+          }
+        }
       }
+
       switch (bodyType) {
-        case BodyTypes.NONE:
-          break
         case BodyTypes.TEXT:
           // content type defaults to plaintext
           input.body = string
@@ -203,6 +256,9 @@ module RestModule {
           for (let [key, value] of Object.entries(object)) {
             params.append(key, value)
           }
+          addPaginationToBody((key: string, value: any) => {
+            params.append(key, value)
+          })
           input.body = params
           break
         case BodyTypes.FORM_DATA:
@@ -210,6 +266,9 @@ module RestModule {
           for (let [key, value] of Object.entries(object)) {
             form.append(key, value)
           }
+          addPaginationToBody((key: string, value: any) => {
+            form.append(key, value)
+          })
           input.body = form
           break
         case BodyTypes.XML:
@@ -219,13 +278,15 @@ module RestModule {
           input.body = string
           input.headers["Content-Type"] = "application/xml"
           break
-        default:
         case BodyTypes.JSON:
           // if JSON error, throw it
           if (error) {
             throw "Invalid JSON for request body"
           }
-          input.body = string
+          addPaginationToBody((key: string, value: any) => {
+            object[key] = value
+          })
+          input.body = JSON.stringify(object)
           input.headers["Content-Type"] = "application/json"
           break
       }
@@ -271,6 +332,8 @@ module RestModule {
         bodyType,
         requestBody,
         authConfigId,
+        pagination,
+        paginationValues
       } = query
       const authHeaders = this.getAuthHeaders(authConfigId)
 
@@ -289,14 +352,12 @@ module RestModule {
       }
 
       let input: any = { method, headers: this.headers }
-      if (requestBody) {
-        input = this.addBody(bodyType, requestBody, input)
-      }
+      input = this.addBody(bodyType, requestBody, input, pagination, paginationValues)
 
       this.startTimeMs = performance.now()
-      const url = this.getUrl(path, queryString)
+      const url = this.getUrl(path, queryString, pagination, paginationValues)
       const response = await fetch(url, input)
-      return await this.parseResponse(response)
+      return await this.parseResponse(response, pagination)
     }
 
     async create(opts: RestQuery) {

package/src/integrations/s3.ts

@@ -8,6 +8,8 @@ module S3Module {
     region: string
     accessKeyId: string
     secretAccessKey: string
+    s3ForcePathStyle: boolean
+    endpoint?: string
   }
 
   const SCHEMA: Integration = {
@@ -18,7 +20,7 @@ module S3Module {
     datasource: {
       region: {
         type: "string",
-        required: true,
+        required: false,
         default: "us-east-1",
       },
       accessKeyId: {
@@ -29,6 +31,15 @@ module S3Module {
         type: "password",
         required: true,
       },
+      endpoint: {
+        type: "string",
+        required: false,
+      },
+      signatureVersion: {
+        type: "string",
+        required: false,
+        default: "v4"
+      },
     },
     query: {
       read: {
@@ -46,16 +57,16 @@ module S3Module {
   class S3Integration implements IntegrationBase {
     private readonly config: S3Config
     private client: any
-    private connectionPromise: Promise<any>
 
     constructor(config: S3Config) {
       this.config = config
-      this.connectionPromise = this.connect()
-      this.client = new AWS.S3()
-    }
+      if (this.config.endpoint) {
+        this.config.s3ForcePathStyle = true
+      } else {
+        delete this.config.endpoint
+      }
 
-    async connect() {
-      AWS.config.update(this.config)
+      this.client = new AWS.S3(this.config)
     }
 
     async read(query: { bucket: string }) {