@budibase/server 1.0.32 → 1.0.39

package/src/integrations/rest.ts

@@ -2,29 +2,73 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
+  RestConfig,
+  RestQueryFields as RestQuery,
+  PaginationConfig,
+  AuthType,
+  BasicAuthConfig,
+  BearerAuthConfig,
+  PaginationValues,
 } from "../definitions/datasource"
 import { IntegrationBase } from "./base/IntegrationBase"
 
+const BodyTypes = {
+  NONE: "none",
+  FORM_DATA: "form",
+  XML: "xml",
+  ENCODED: "encoded",
+  JSON: "json",
+  TEXT: "text",
+}
+
+const coreFields = {
+  path: {
+    type: DatasourceFieldTypes.STRING,
+    display: "URL",
+  },
+  queryString: {
+    type: DatasourceFieldTypes.STRING,
+  },
+  headers: {
+    type: DatasourceFieldTypes.OBJECT,
+  },
+  enabledHeaders: {
+    type: DatasourceFieldTypes.OBJECT,
+  },
+  requestBody: {
+    type: DatasourceFieldTypes.JSON,
+  },
+  bodyType: {
+    type: DatasourceFieldTypes.STRING,
+    enum: Object.values(BodyTypes),
+  },
+  pagination: {
+    type: DatasourceFieldTypes.OBJECT,
+  },
+}
+
 module RestModule {
   const fetch = require("node-fetch")
-
-  interface RestConfig {
-    url: string
-    defaultHeaders: {
-      [key: string]: any
-    }
-  }
+  const { formatBytes } = require("../utilities")
+  const { performance } = require("perf_hooks")
+  const FormData = require("form-data")
+  const { URLSearchParams } = require("url")
+  const {
+    parseStringPromise: xmlParser,
+    Builder: XmlBuilder,
+  } = require("xml2js")
 
   const SCHEMA: Integration = {
     docs: "https://github.com/node-fetch/node-fetch",
     description:
-      "Representational state transfer (REST) is a de-facto standard for a software architecture for interactive applications that typically use multiple Web services. ",
+      "With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
     friendlyName: "REST API",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,
-        default: "localhost",
-        required: true,
+        default: "",
+        required: false,
+        deprecated: true,
       },
       defaultHeaders: {
         type: DatasourceFieldTypes.OBJECT,
@@ -37,97 +81,30 @@ module RestModule {
         readable: true,
         displayName: "POST",
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-          requestBody: {
-            type: DatasourceFieldTypes.JSON,
-          },
-        },
+        fields: coreFields,
       },
       read: {
         displayName: "GET",
         readable: true,
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-        },
+        fields: coreFields,
       },
       update: {
         displayName: "PUT",
         readable: true,
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-          requestBody: {
-            type: DatasourceFieldTypes.JSON,
-          },
-        },
+        fields: coreFields,
       },
       patch: {
         displayName: "PATCH",
         readable: true,
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-          requestBody: {
-            type: DatasourceFieldTypes.JSON,
-          },
-        },
+        fields: coreFields,
       },
       delete: {
         displayName: "DELETE",
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-          requestBody: {
-            type: DatasourceFieldTypes.JSON,
-          },
-        },
+        fields: coreFields,
       },
     },
   }
@@ -137,99 +114,295 @@ module RestModule {
     private headers: {
       [key: string]: string
     } = {}
+    private startTimeMs: number = performance.now()
 
     constructor(config: RestConfig) {
       this.config = config
     }
 
-    async parseResponse(response: any) {
-      const contentType = response.headers.get("content-type")
-      if (contentType && contentType.indexOf("application/json") !== -1) {
-        return await response.json()
-      } else {
-        return await response.text()
+    async parseResponse(response: any, pagination: PaginationConfig | null) {
+      let data, raw, headers
+      const contentType = response.headers.get("content-type") || ""
+      try {
+        if (contentType.includes("application/json")) {
+          data = await response.json()
+          raw = JSON.stringify(data)
+        } else if (
+          contentType.includes("text/xml") ||
+          contentType.includes("application/xml")
+        ) {
+          const rawXml = await response.text()
+          data =
+            (await xmlParser(rawXml, {
+              explicitArray: false,
+              trim: true,
+              explicitRoot: false,
+            })) || {}
+          // there is only one structure, its an array, return the array so it appears as rows
+          const keys = Object.keys(data)
+          if (keys.length === 1 && Array.isArray(data[keys[0]])) {
+            data = data[keys[0]]
+          }
+          raw = rawXml
+        } else {
+          data = await response.text()
+          raw = data
+        }
+      } catch (err) {
+        throw "Failed to parse response body."
+      }
+      const size = formatBytes(
+        response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
+      )
+      const time = `${Math.round(performance.now() - this.startTimeMs)}ms`
+      headers = response.headers.raw()
+      for (let [key, value] of Object.entries(headers)) {
+        headers[key] = Array.isArray(value) ? value[0] : value
       }
-    }
 
-    getUrl(path: string, queryString: string): string {
-      return `${this.config.url}/${path}?${queryString}`
-    }
+      // Check if a pagination cursor exists in the response
+      let nextCursor = null
+      if (pagination?.responseParam) {
+        nextCursor = data?.[pagination.responseParam]
+      }
 
-    async create({ path = "", queryString = "", headers = {}, json = {} }) {
-      this.headers = {
-        ...this.config.defaultHeaders,
-        ...headers,
+      return {
+        data,
+        info: {
+          code: response.status,
+          size,
+          time,
+        },
+        extra: {
+          raw,
+          headers,
+        },
+        pagination: {
+          cursor: nextCursor,
+        },
       }
+    }
+
+    getUrl(
+      path: string,
+      queryString: string,
+      pagination: PaginationConfig | null,
+      paginationValues: PaginationValues | null
+    ): string {
+      // Add pagination params to query string if required
+      if (pagination?.location === "query" && paginationValues) {
+        const { pageParam, sizeParam } = pagination
+        const params = new URLSearchParams()
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        method: "POST",
-        headers: this.headers,
-        body: JSON.stringify(json),
-      })
+        // Append page number or cursor param if configured
+        if (pageParam && paginationValues.page != null) {
+          params.append(pageParam, paginationValues.page)
+        }
+
+        // Append page size param if configured
+        if (sizeParam && paginationValues.limit != null) {
+          params.append(sizeParam, paginationValues.limit)
+        }
+
+        // Prepend query string with pagination params
+        let paginationString = params.toString()
+        if (paginationString) {
+          queryString = `${paginationString}&${queryString}`
+        }
+      }
 
-      return await this.parseResponse(response)
+      const main = `${path}?${queryString}`
+      let complete = main
+      if (this.config.url && !main.startsWith("http")) {
+        complete = !this.config.url ? main : `${this.config.url}/${main}`
+      }
+      if (!complete.startsWith("http")) {
+        complete = `http://${complete}`
+      }
+      return complete
     }
 
-    async read({ path = "", queryString = "", headers = {} }) {
-      this.headers = {
-        ...this.config.defaultHeaders,
-        ...headers,
+    addBody(
+      bodyType: string,
+      body: string | any,
+      input: any,
+      pagination: PaginationConfig | null,
+      paginationValues: PaginationValues | null
+    ) {
+      if (!input.headers) {
+        input.headers = {}
+      }
+      if (bodyType === BodyTypes.NONE) {
+        return input
+      }
+      let error,
+        object: any = {},
+        string = ""
+      try {
+        if (body) {
+          string = typeof body !== "string" ? JSON.stringify(body) : body
+          object = typeof body === "object" ? body : JSON.parse(body)
+        }
+      } catch (err) {
+        error = err
       }
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        headers: this.headers,
-      })
+      // Util to add pagination values to a certain body type
+      const addPaginationToBody = (insertFn: Function) => {
+        if (pagination?.location === "body") {
+          if (pagination?.pageParam && paginationValues?.page != null) {
+            insertFn(pagination.pageParam, paginationValues.page)
+          }
+          if (pagination?.sizeParam && paginationValues?.limit != null) {
+            insertFn(pagination.sizeParam, paginationValues.limit)
+          }
+        }
+      }
 
-      return await this.parseResponse(response)
+      switch (bodyType) {
+        case BodyTypes.TEXT:
+          // content type defaults to plaintext
+          input.body = string
+          break
+        case BodyTypes.ENCODED:
+          const params = new URLSearchParams()
+          for (let [key, value] of Object.entries(object)) {
+            params.append(key, value)
+          }
+          addPaginationToBody((key: string, value: any) => {
+            params.append(key, value)
+          })
+          input.body = params
+          break
+        case BodyTypes.FORM_DATA:
+          const form = new FormData()
+          for (let [key, value] of Object.entries(object)) {
+            form.append(key, value)
+          }
+          addPaginationToBody((key: string, value: any) => {
+            form.append(key, value)
+          })
+          input.body = form
+          break
+        case BodyTypes.XML:
+          if (object != null) {
+            string = new XmlBuilder().buildObject(object)
+          }
+          input.body = string
+          input.headers["Content-Type"] = "application/xml"
+          break
+        case BodyTypes.JSON:
+          // if JSON error, throw it
+          if (error) {
+            throw "Invalid JSON for request body"
+          }
+          addPaginationToBody((key: string, value: any) => {
+            object[key] = value
+          })
+          input.body = JSON.stringify(object)
+          input.headers["Content-Type"] = "application/json"
+          break
+      }
+      return input
     }
 
-    async update({ path = "", queryString = "", headers = {}, json = {} }) {
-      this.headers = {
-        ...this.config.defaultHeaders,
-        ...headers,
-      }
+    getAuthHeaders(authConfigId: string): { [key: string]: any } {
+      let headers: any = {}
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        method: "PUT",
-        headers: this.headers,
-        body: JSON.stringify(json),
-      })
+      if (this.config.authConfigs && authConfigId) {
+        const authConfig = this.config.authConfigs.filter(
+          c => c._id === authConfigId
+        )[0]
+        // check the config still exists before proceeding
+        // if not - do nothing
+        if (authConfig) {
+          let config
+          switch (authConfig.type) {
+            case AuthType.BASIC:
+              config = authConfig.config as BasicAuthConfig
+              headers.Authorization = `Basic ${Buffer.from(
+                `${config.username}:${config.password}`
+              ).toString("base64")}`
+              break
+            case AuthType.BEARER:
+              config = authConfig.config as BearerAuthConfig
+              headers.Authorization = `Bearer ${config.token}`
+              break
+          }
+        }
+      }
 
-      return await this.parseResponse(response)
+      return headers
     }
 
-    async patch({ path = "", queryString = "", headers = {}, json = {} }) {
+    async _req(query: RestQuery) {
+      const {
+        path = "",
+        queryString = "",
+        headers = {},
+        method = "GET",
+        disabledHeaders,
+        bodyType,
+        requestBody,
+        authConfigId,
+        pagination,
+        paginationValues,
+      } = query
+      const authHeaders = this.getAuthHeaders(authConfigId)
+
       this.headers = {
         ...this.config.defaultHeaders,
         ...headers,
+        ...authHeaders,
+      }
+
+      if (disabledHeaders) {
+        for (let headerKey of Object.keys(this.headers)) {
+          if (disabledHeaders[headerKey]) {
+            delete this.headers[headerKey]
+          }
+        }
       }
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        method: "PATCH",
-        headers: this.headers,
-        body: JSON.stringify(json),
-      })
+      let input: any = { method, headers: this.headers }
+      input = this.addBody(
+        bodyType,
+        requestBody,
+        input,
+        pagination,
+        paginationValues
+      )
 
-      return await this.parseResponse(response)
+      this.startTimeMs = performance.now()
+      const url = this.getUrl(path, queryString, pagination, paginationValues)
+      const response = await fetch(url, input)
+      return await this.parseResponse(response, pagination)
     }
 
-    async delete({ path = "", queryString = "", headers = {} }) {
-      this.headers = {
-        ...this.config.defaultHeaders,
-        ...headers,
-      }
+    async create(opts: RestQuery) {
+      return this._req({ ...opts, method: "POST" })
+    }
+
+    async read(opts: RestQuery) {
+      return this._req({ ...opts, method: "GET" })
+    }
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        method: "DELETE",
-        headers: this.headers,
-      })
+    async update(opts: RestQuery) {
+      return this._req({ ...opts, method: "PUT" })
+    }
+
+    async patch(opts: RestQuery) {
+      return this._req({ ...opts, method: "PATCH" })
+    }
 
-      return await this.parseResponse(response)
+    async delete(opts: RestQuery) {
+      return this._req({ ...opts, method: "DELETE" })
     }
   }
 
   module.exports = {
     schema: SCHEMA,
     integration: RestIntegration,
+    AuthType,
   }
 }

package/src/integrations/s3.ts

@@ -8,6 +8,8 @@ module S3Module {
     region: string
     accessKeyId: string
     secretAccessKey: string
+    s3ForcePathStyle: boolean
+    endpoint?: string
   }
 
   const SCHEMA: Integration = {
@@ -18,7 +20,7 @@ module S3Module {
     datasource: {
       region: {
         type: "string",
-        required: true,
+        required: false,
         default: "us-east-1",
       },
       accessKeyId: {
@@ -29,6 +31,15 @@ module S3Module {
         type: "password",
         required: true,
       },
+      endpoint: {
+        type: "string",
+        required: false,
+      },
+      signatureVersion: {
+        type: "string",
+        required: false,
+        default: "v4"
+      },
     },
     query: {
       read: {
@@ -46,16 +57,16 @@ module S3Module {
   class S3Integration implements IntegrationBase {
     private readonly config: S3Config
     private client: any
-    private connectionPromise: Promise<any>
 
     constructor(config: S3Config) {
       this.config = config
-      this.connectionPromise = this.connect()
-      this.client = new AWS.S3()
-    }
+      if (this.config.endpoint) {
+        this.config.s3ForcePathStyle = true
+      } else {
+        delete this.config.endpoint
+      }
 
-    async connect() {
-      AWS.config.update(this.config)
+      this.client = new AWS.S3(this.config)
     }
 
     async read(query: { bucket: string }) {