@budibase/server 1.0.18 → 1.0.19-alpha.3

package/src/definitions/datasource.ts

@@ -1,4 +1,4 @@
-import { Row, Table } from "./common"
+import { Row, Table, Base } from "./common"
 
 export enum Operation {
   CREATE = "CREATE",
@@ -181,3 +181,75 @@ export interface SqlQuery {
 export interface QueryOptions {
   disableReturning?: boolean
 }
+
+export interface Datasource extends Base {
+  type: string
+  name: string
+  source: SourceNames
+  // the config is defined by the schema
+  config: {
+    [key: string]: string | number | boolean
+  }
+  plus: boolean
+  entities?: {
+    [key: string]: Table
+  }
+}
+
+export enum AuthType {
+  BASIC = "basic",
+  BEARER = "bearer",
+}
+
+interface AuthConfig {
+  _id: string
+  name: string
+  type: AuthType
+  config: BasicAuthConfig | BearerAuthConfig
+}
+
+export interface BasicAuthConfig {
+  username: string
+  password: string
+}
+
+export interface BearerAuthConfig {
+  token: string
+}
+
+export interface QueryParameter {
+  name: string
+  default: string
+}
+
+export interface RestQueryFields {
+  path: string
+  queryString?: string
+  headers: { [key: string]: any }
+  disabledHeaders: { [key: string]: any }
+  requestBody: any
+  bodyType: string
+  json: object
+  method: string
+  authConfigId: string
+}
+
+export interface RestConfig {
+  url: string
+  defaultHeaders: {
+    [key: string]: any
+  }
+  authConfigs: AuthConfig[]
+}
+
+export interface Query {
+  _id?: string
+  datasourceId: string
+  name: string
+  parameters: QueryParameter[]
+  fields: RestQueryFields | any
+  transformer: string | null
+  schema: any
+  readable: boolean
+  queryVerb: string
+}

package/src/integrations/base/IntegrationBase.ts

@@ -1,6 +1,6 @@
 export interface IntegrationBase {
-  create?(query: any): Promise<any[]>
-  read?(query: any): Promise<any[]>
-  update?(query: any): Promise<any[]>
-  delete?(query: any): Promise<any[]>
+  create?(query: any): Promise<any[] | any>
+  read?(query: any): Promise<any[] | any>
+  update?(query: any): Promise<any[] | any>
+  delete?(query: any): Promise<any[] | any>
 }

package/src/integrations/oracle.ts

@@ -450,7 +450,7 @@ module OracleModule {
             })
             return lastRow.rows
           } else {
-            return [{ [ operation.toLowerCase() ]: true }]
+            return [{ [operation.toLowerCase()]: true }]
           }
         }
       }

package/src/integrations/rest.ts

@@ -2,29 +2,61 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
+  RestConfig,
+  RestQueryFields as RestQuery,
+  AuthType,
+  BasicAuthConfig,
+  BearerAuthConfig
 } from "../definitions/datasource"
 import { IntegrationBase } from "./base/IntegrationBase"
 
+const BodyTypes = {
+  NONE: "none",
+  FORM_DATA: "form",
+  ENCODED: "encoded",
+  JSON: "json",
+  TEXT: "text",
+}
+
+const coreFields = {
+  path: {
+    type: DatasourceFieldTypes.STRING,
+    display: "URL",
+  },
+  queryString: {
+    type: DatasourceFieldTypes.STRING,
+  },
+  headers: {
+    type: DatasourceFieldTypes.OBJECT,
+  },
+  enabledHeaders: {
+    type: DatasourceFieldTypes.OBJECT,
+  },
+  requestBody: {
+    type: DatasourceFieldTypes.JSON,
+  },
+  bodyType: {
+    type: DatasourceFieldTypes.STRING,
+    enum: Object.values(BodyTypes),
+  },
+}
+
 module RestModule {
   const fetch = require("node-fetch")
-
-  interface RestConfig {
-    url: string
-    defaultHeaders: {
-      [key: string]: any
-    }
-  }
+  const { formatBytes } = require("../utilities")
+  const { performance } = require("perf_hooks")
 
   const SCHEMA: Integration = {
     docs: "https://github.com/node-fetch/node-fetch",
     description:
-      "Representational state transfer (REST) is a de-facto standard for a software architecture for interactive applications that typically use multiple Web services. ",
+      "With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
     friendlyName: "REST API",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,
-        default: "localhost",
-        required: true,
+        default: "",
+        required: false,
+        deprecated: true,
       },
       defaultHeaders: {
         type: DatasourceFieldTypes.OBJECT,
@@ -37,97 +69,30 @@ module RestModule {
         readable: true,
         displayName: "POST",
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-          requestBody: {
-            type: DatasourceFieldTypes.JSON,
-          },
-        },
+        fields: coreFields,
       },
       read: {
         displayName: "GET",
         readable: true,
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-        },
+        fields: coreFields,
       },
       update: {
         displayName: "PUT",
         readable: true,
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-          requestBody: {
-            type: DatasourceFieldTypes.JSON,
-          },
-        },
+        fields: coreFields,
       },
       patch: {
         displayName: "PATCH",
         readable: true,
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-          requestBody: {
-            type: DatasourceFieldTypes.JSON,
-          },
-        },
+        fields: coreFields,
       },
       delete: {
         displayName: "DELETE",
         type: QueryTypes.FIELDS,
-        urlDisplay: true,
-        fields: {
-          path: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          queryString: {
-            type: DatasourceFieldTypes.STRING,
-          },
-          headers: {
-            type: DatasourceFieldTypes.OBJECT,
-          },
-          requestBody: {
-            type: DatasourceFieldTypes.JSON,
-          },
-        },
+        fields: coreFields,
       },
     },
   }
@@ -137,99 +102,145 @@ module RestModule {
     private headers: {
       [key: string]: string
     } = {}
+    private startTimeMs: number = performance.now()
 
     constructor(config: RestConfig) {
       this.config = config
     }
 
     async parseResponse(response: any) {
+      let data, raw, headers
       const contentType = response.headers.get("content-type")
       if (contentType && contentType.indexOf("application/json") !== -1) {
-        return await response.json()
+        data = await response.json()
+        raw = JSON.stringify(data)
       } else {
-        return await response.text()
+        data = await response.text()
+        raw = data
+      }
+      const size = formatBytes(response.headers.get("content-length") || Buffer.byteLength(raw, "utf8"))
+      const time = `${Math.round(performance.now() - this.startTimeMs)}ms`
+      headers = response.headers.raw()
+      for (let [key, value] of Object.entries(headers)) {
+        headers[key] = Array.isArray(value) ? value[0] : value
+      }
+      return {
+        data,
+        info: {
+          code: response.status,
+          size,
+          time,
+        },
+        extra: {
+          raw,
+          headers,
+        },
       }
     }
 
     getUrl(path: string, queryString: string): string {
-      return `${this.config.url}/${path}?${queryString}`
+      const main = `${path}?${queryString}`
+      let complete = main
+      if (this.config.url && !main.startsWith(this.config.url)) {
+        complete = !this.config.url ? main : `${this.config.url}/${main}`
+      }
+      if (!complete.startsWith("http")) {
+        complete = `http://${complete}`
+      }
+      return complete
     }
 
-    async create({ path = "", queryString = "", headers = {}, json = {} }) {
-      this.headers = {
-        ...this.config.defaultHeaders,
-        ...headers,
+    getAuthHeaders(authConfigId: string): { [key: string]: any }{
+      let headers: any = {}
+
+      if (this.config.authConfigs && authConfigId) {
+        const authConfig = this.config.authConfigs.filter(
+          c => c._id === authConfigId
+        )[0]
+        // check the config still exists before proceeding
+        // if not - do nothing
+        if (authConfig) {
+          let config
+          switch (authConfig.type) {
+            case AuthType.BASIC:
+              config = authConfig.config as BasicAuthConfig
+              headers.Authorization = `Basic ${Buffer.from(
+                `${config.username}:${config.password}`
+              ).toString("base64")}`
+              break
+            case AuthType.BEARER:
+              config = authConfig.config as BearerAuthConfig
+              headers.Authorization = `Bearer ${config.token}`
+              break
+          }
+        }
       }
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        method: "POST",
-        headers: this.headers,
-        body: JSON.stringify(json),
-      })
-
-      return await this.parseResponse(response)
+      return headers
     }
 
-    async read({ path = "", queryString = "", headers = {} }) {
+    async _req(query: RestQuery) {
+      const { path = "", queryString = "", headers = {}, method = "GET", disabledHeaders, bodyType, requestBody, authConfigId } = query
+      const authHeaders = this.getAuthHeaders(authConfigId)
+
       this.headers = {
         ...this.config.defaultHeaders,
         ...headers,
+        ...authHeaders,
       }
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        headers: this.headers,
-      })
-
-      return await this.parseResponse(response)
-    }
+      if (disabledHeaders) {
+        for (let headerKey of Object.keys(this.headers)) {
+          if (disabledHeaders[headerKey]) {
+            delete this.headers[headerKey]
+          }
+        }
+      }
 
-    async update({ path = "", queryString = "", headers = {}, json = {} }) {
-      this.headers = {
-        ...this.config.defaultHeaders,
-        ...headers,
+      let json
+      if (bodyType === BodyTypes.JSON && requestBody) {
+        try {
+          json = JSON.parse(requestBody)
+        } catch (err) {
+          throw "Invalid JSON for request body"
+        }
       }
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        method: "PUT",
-        headers: this.headers,
-        body: JSON.stringify(json),
-      })
+      const input: any = { method, headers: this.headers }
+      if (json && typeof json === "object" && Object.keys(json).length > 0) {
+        input.body = JSON.stringify(json)
+      }
 
+      this.startTimeMs = performance.now()
+      const url = this.getUrl(path, queryString)
+      const response = await fetch(url, input)
       return await this.parseResponse(response)
     }
 
-    async patch({ path = "", queryString = "", headers = {}, json = {} }) {
-      this.headers = {
-        ...this.config.defaultHeaders,
-        ...headers,
-      }
-
-      const response = await fetch(this.getUrl(path, queryString), {
-        method: "PATCH",
-        headers: this.headers,
-        body: JSON.stringify(json),
-      })
+    async create(opts: RestQuery) {
+      return this._req({ ...opts, method: "POST" })
+    }
 
-      return await this.parseResponse(response)
+    async read(opts: RestQuery) {
+      return this._req({ ...opts, method: "GET" })
     }
 
-    async delete({ path = "", queryString = "", headers = {} }) {
-      this.headers = {
-        ...this.config.defaultHeaders,
-        ...headers,
-      }
+    async update(opts: RestQuery) {
+      return this._req({ ...opts, method: "PUT" })
+    }
 
-      const response = await fetch(this.getUrl(path, queryString), {
-        method: "DELETE",
-        headers: this.headers,
-      })
+    async patch(opts: RestQuery) {
+      return this._req({ ...opts, method: "PATCH" })
+    }
 
-      return await this.parseResponse(response)
+    async delete(opts: RestQuery) {
+      return this._req({ ...opts, method: "DELETE" })
     }
   }
 
   module.exports = {
     schema: SCHEMA,
     integration: RestIntegration,
+    AuthType,
   }
 }

package/src/integrations/tests/rest.spec.js

@@ -1,6 +1,9 @@
 jest.mock("node-fetch", () =>
   jest.fn(() => ({
     headers: {
+      raw: () => {
+        return { "content-type": ["application/json"] }
+      },
       get: () => ["application/json"]
     },
     json: jest.fn(), 
@@ -9,6 +12,7 @@ jest.mock("node-fetch", () =>
 )
 const fetch = require("node-fetch")
 const RestIntegration = require("../rest")
+const { AuthType } = require("../rest")
 
 class TestConfiguration {
   constructor(config = {}) {
@@ -24,6 +28,7 @@ describe("REST Integration", () => {
     config = new TestConfiguration({
       url: BASE_URL,
     })
+    jest.clearAllMocks()
   })
 
   it("calls the create method with the correct params", async () => {
@@ -33,9 +38,10 @@ describe("REST Integration", () => {
       headers: {
         Accept: "application/json",
       },
-      json: {
+      bodyType: "json",
+      requestBody: JSON.stringify({
         name: "test",
-      },
+      }),
     }
     const response = await config.integration.create(query)
     expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
@@ -60,6 +66,7 @@ describe("REST Integration", () => {
       headers: {
         Accept: "text/html",
       },
+      method: "GET",
     })
   })
 
@@ -70,13 +77,14 @@ describe("REST Integration", () => {
       headers: {
         Accept: "application/json",
       },
-      json: {
+      bodyType: "json",
+      requestBody: JSON.stringify({
         name: "test",
-      },
+      }),
     }
     const response = await config.integration.update(query)
     expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
-      method: "POST",
+      method: "PUT",
       body: '{"name":"test"}',
       headers: {
         Accept: "application/json",
@@ -91,9 +99,10 @@ describe("REST Integration", () => {
       headers: {
         Accept: "application/json",
       },
-      json: {
+      bodyType: "json",
+      requestBody: JSON.stringify({
         name: "test",
-      },
+      }),
     }
     const response = await config.integration.delete(query)
     expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
@@ -101,6 +110,61 @@ describe("REST Integration", () => {
       headers: {
         Accept: "application/json",
       },
+      body: '{"name":"test"}',
+    })
+  })
+
+  describe("authentication", () => {
+    const basicAuth = {
+      _id: "c59c14bd1898a43baa08da68959b24686",
+      name: "basic-1",
+      type : AuthType.BASIC,
+      config : {
+        username: "user",
+        password: "password"
+      }
+    }
+    
+    const bearerAuth = {
+      _id: "0d91d732f34e4befabeff50b392a8ff3",
+      name: "bearer-1",
+      type : AuthType.BEARER,
+      config : {
+        "token": "mytoken"
+      }
+    }
+
+    beforeEach(() => {
+      config = new TestConfiguration({
+        url: BASE_URL,
+        authConfigs : [basicAuth, bearerAuth]
+      })
+    })
+
+    it("adds basic auth", async () => {
+      const query = {
+        authConfigId: "c59c14bd1898a43baa08da68959b24686"
+      }
+      await config.integration.read(query)
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
+        method: "GET",
+        headers: {
+          Authorization: "Basic dXNlcjpwYXNzd29yZA=="
+        },
+      })
+    })
+
+    it("adds bearer auth", async () => {
+      const query = {
+        authConfigId: "0d91d732f34e4befabeff50b392a8ff3"
+      }
+      await config.integration.read(query)
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
+        method: "GET",
+        headers: {
+          Authorization: "Bearer mytoken"
+        },
+      })
     })
   })
 })

package/src/threads/query.js

@@ -15,6 +15,15 @@ function formatResponse(resp) {
   return resp
 }
 
+function hasExtraData(response) {
+  return (
+    typeof response === "object" &&
+    !Array.isArray(response) &&
+    response.data != null &&
+    response.info != null
+  )
+}
+
 async function runAndTransform(datasource, queryVerb, query, transformer) {
   const Integration = integrations[datasource.source]
   if (!Integration) {
@@ -22,7 +31,15 @@ async function runAndTransform(datasource, queryVerb, query, transformer) {
   }
   const integration = new Integration(datasource.config)
 
-  let rows = formatResponse(await integration[queryVerb](query))
+  let output = formatResponse(await integration[queryVerb](query))
+  let rows = output,
+    info = undefined,
+    extra = undefined
+  if (hasExtraData(output)) {
+    rows = output.data
+    info = output.info
+    extra = output.extra
+  }
 
   // transform as required
   if (transformer) {
@@ -47,7 +64,7 @@ async function runAndTransform(datasource, queryVerb, query, transformer) {
     integration.end()
   }
 
-  return { rows, keys }
+  return { rows, keys, info, extra }
 }
 
 module.exports = (input, callback) => {

package/src/utilities/index.js

@@ -150,3 +150,14 @@ exports.doesDatabaseExist = async dbName => {
     return false
   }
 }
+
+exports.formatBytes = bytes => {
+  const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
+  const byteIncrements = 1024
+  let unit = 0
+  let size = parseInt(bytes, 10) || 0
+  while (size >= byteIncrements && ++unit) {
+    size /= byteIncrements
+  }
+  return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)}${units[unit]}`
+}