@budibase/server 1.0.98 → 1.0.101

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.98",
+  "version": "1.0.101",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -68,12 +68,13 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.98",
-    "@budibase/client": "^1.0.98",
-    "@budibase/string-templates": "^1.0.98",
+    "@budibase/backend-core": "^1.0.101",
+    "@budibase/client": "^1.0.101",
+    "@budibase/string-templates": "^1.0.101",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
+    "@google-cloud/firestore": "^5.0.2",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
     "@sentry/node": "^6.0.0",
@@ -145,6 +146,7 @@
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
+    "@types/lodash": "4.14.180",
     "@types/node": "^15.12.4",
     "@types/oracledb": "^5.2.1",
     "@types/redis": "^4.0.11",
@@ -153,6 +155,7 @@
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
+    "is-wsl": "^2.2.0",
     "jest": "^27.0.5",
     "jest-openapi": "^0.14.2",
     "nodemon": "^2.0.4",
@@ -171,5 +174,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "c13a6db9c74b50786732c52d2827e018a3542047"
+  "gitHead": "3862854f5b44c63494d6e5fc060c16a5e1890d1a"
 }

package/scripts/dev/manage.js

@@ -2,10 +2,11 @@
 const compose = require("docker-compose")
 const path = require("path")
 const fs = require("fs")
+const isWsl = require("is-wsl")
 const { processStringSync } = require("@budibase/string-templates")
 
 function isLinux() {
-  return process.platform !== "darwin" && process.platform !== "win32"
+  return !isWsl && process.platform !== "darwin" && process.platform !== "win32"
 }
 
 // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.

package/specs/openapi.json

@@ -1260,10 +1260,30 @@
         ]
       },
       "executeQuery": {
-        "description": "The query body must contain the required parameters for the query, this depends on query type, setup and bindings.",
+        "description": "The parameters required for executing a query.",
         "type": "object",
-        "additionalProperties": {
-          "description": "Key value properties of any type, depending on the query output schema."
+        "properties": {
+          "parameters": {
+            "type": "object",
+            "description": "This contains the required parameters for the query, this depends on query type, setup and bindings.",
+            "additionalProperties": {
+              "description": "Key value properties of any type, depending on the query output schema."
+            }
+          },
+          "pagination": {
+            "type": "object",
+            "description": "For supported query types (currently on REST) pagination can be performed using these properties.",
+            "properties": {
+              "page": {
+                "type": "string",
+                "description": "The page which has been returned from a previous query."
+              },
+              "limit": {
+                "type": "number",
+                "description": "The number of rows to return per page."
+              }
+            }
+          }
         }
       },
       "executeQueryOutput": {

package/specs/openapi.yaml

@@ -951,11 +951,27 @@ components:
       required:
         - data
     executeQuery:
-      description: The query body must contain the required parameters for the query,
-        this depends on query type, setup and bindings.
+      description: The parameters required for executing a query.
       type: object
-      additionalProperties:
-        description: Key value properties of any type, depending on the query output schema.
+      properties:
+        parameters:
+          type: object
+          description: This contains the required parameters for the query, this depends
+            on query type, setup and bindings.
+          additionalProperties:
+            description: Key value properties of any type, depending on the query output
+              schema.
+        pagination:
+          type: object
+          description: For supported query types (currently on REST) pagination can be
+            performed using these properties.
+          properties:
+            page:
+              type: string
+              description: The page which has been returned from a previous query.
+            limit:
+              type: number
+              description: The number of rows to return per page.
     executeQueryOutput:
       type: object
       properties:

package/specs/resources/query.js

@@ -124,12 +124,34 @@ const querySchema = object(
 )
 
 const executeQuerySchema = {
-  description:
-    "The query body must contain the required parameters for the query, this depends on query type, setup and bindings.",
+  description: "The parameters required for executing a query.",
   type: "object",
-  additionalProperties: {
-    description:
-      "Key value properties of any type, depending on the query output schema.",
+  properties: {
+    parameters: {
+      type: "object",
+      description:
+        "This contains the required parameters for the query, this depends on query type, setup and bindings.",
+      additionalProperties: {
+        description:
+          "Key value properties of any type, depending on the query output schema.",
+      },
+    },
+    pagination: {
+      type: "object",
+      description:
+        "For supported query types (currently on REST) pagination can be performed using these properties.",
+      properties: {
+        page: {
+          type: "string",
+          description:
+            "The page which has been returned from a previous query.",
+        },
+        limit: {
+          type: "number",
+          description: "The number of rows to return per page.",
+        },
+      },
+    },
   },
 }
 

package/src/api/controllers/integration.js

@@ -1,8 +1,24 @@
+const { cloneDeep } = require("lodash")
 const { definitions } = require("../../integrations")
+const { getTenantId } = require("@budibase/backend-core/tenancy")
+const { SourceNames } = require("../../definitions/datasource")
+const googlesheets = require("../../integrations/googlesheets")
+const env = require("../../environment")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
-  ctx.body = definitions
+  const defs = cloneDeep(definitions)
+
+  // for google sheets integration google verification
+  if (env.EXCLUDE_QUOTAS_TENANTS) {
+    const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.split(",")
+    const tenantId = getTenantId()
+    if (excludedTenants.includes(tenantId)) {
+      defs[SourceNames.GOOGLE_SHEETS] = googlesheets.schema
+    }
+  }
+
+  ctx.body = defs
 }
 
 exports.find = async function (ctx) {

package/src/api/controllers/permission.js

@@ -4,6 +4,7 @@ const {
   getDBRoleID,
   getExternalRoleID,
   getBuiltinRoles,
+  checkForRoleResourceArray,
 } = require("@budibase/backend-core/roles")
 const { getRoleParams } = require("../../db/utils")
 const {
@@ -144,12 +145,11 @@ exports.getResourcePerms = async function (ctx) {
   for (let level of SUPPORTED_LEVELS) {
     // update the various roleIds in the resource permissions
     for (let role of roles) {
-      const rolePerms = role.permissions
+      const rolePerms = checkForRoleResourceArray(role.permissions, resourceId)
       if (
         rolePerms &&
         rolePerms[resourceId] &&
-        (rolePerms[resourceId] === level ||
-          rolePerms[resourceId].indexOf(level) !== -1)
+        rolePerms[resourceId].indexOf(level) !== -1
       ) {
         permissions[level] = getExternalRoleID(role._id)
       }

package/src/api/controllers/query/import/index.ts

@@ -2,6 +2,7 @@ import { queryValidation } from "../validation"
 import { generateQueryID } from "../../../../db/utils"
 import { ImportInfo, ImportSource } from "./sources/base"
 import { OpenAPI2 } from "./sources/openapi2"
+import { OpenAPI3 } from "./sources/openapi3"
 import { Query } from "./../../../../definitions/common"
 import { Curl } from "./sources/curl"
 // @ts-ignore
@@ -18,7 +19,7 @@ export class RestImporter {
 
   constructor(data: string) {
     this.data = data
-    this.sources = [new OpenAPI2(), new Curl()]
+    this.sources = [new OpenAPI2(), new OpenAPI3(), new Curl()]
   }
 
   init = async () => {

package/src/api/controllers/query/import/sources/base/index.ts

@@ -23,7 +23,7 @@ export abstract class ImportSource {
     name: string,
     method: string,
     path: string,
-    url: URL,
+    url: URL | string | undefined,
     queryString: string,
     headers: object = {},
     parameters: QueryParameter[] = [],
@@ -34,7 +34,17 @@ export abstract class ImportSource {
     const transformer = "return data"
     const schema = {}
     path = this.processPath(path)
-    path = `${url.origin}/${path}`
+    if (url) {
+      if (typeof url === "string") {
+        path = `${url}/${path}`
+      } else {
+        let href = url.href
+        if (href.endsWith("/")) {
+          href = href.slice(0, -1)
+        }
+        path = `${href}/${path}`
+      }
+    }
     queryString = this.processQuery(queryString)
     const requestBody = JSON.stringify(body, null, 2)
 

package/src/api/controllers/query/import/sources/curl.ts

@@ -74,7 +74,7 @@ export class Curl extends ImportSource {
   getQueries = async (datasourceId: string): Promise<Query[]> => {
     const url = this.getUrl()
     const name = url.pathname
-    const path = url.pathname
+    const path = url.origin + url.pathname
     const method = this.curl.method
     const queryString = url.search
     const headers = this.curl.headers
@@ -90,7 +90,7 @@ export class Curl extends ImportSource {
       name,
       method,
       path,
-      url,
+      undefined,
       queryString,
       headers,
       [],

package/src/api/controllers/query/import/sources/openapi3.ts

@@ -0,0 +1,205 @@
+import { ImportInfo } from "./base"
+import { Query, QueryParameter } from "../../../../../definitions/datasource"
+import { OpenAPIV3 } from "openapi-types"
+import { OpenAPISource } from "./base/openapi"
+import { URL } from "url"
+
+const parameterNotRef = (
+  param: OpenAPIV3.ParameterObject | OpenAPIV3.ReferenceObject
+): param is OpenAPIV3.ParameterObject => {
+  // all refs are deferenced by parser library
+  return true
+}
+
+const requestBodyNotRef = (
+  param: OpenAPIV3.RequestBodyObject | OpenAPIV3.ReferenceObject | undefined
+): param is OpenAPIV3.RequestBodyObject => {
+  // all refs are deferenced by parser library
+  return param !== undefined
+}
+
+const schemaNotRef = (
+  param: OpenAPIV3.SchemaObject | OpenAPIV3.ReferenceObject | undefined
+): param is OpenAPIV3.SchemaObject => {
+  // all refs are deferenced by parser library
+  return param !== undefined
+}
+
+const isOpenAPI3 = (document: any): document is OpenAPIV3.Document => {
+  return document.openapi.includes("3.0")
+}
+
+const methods: string[] = Object.values(OpenAPIV3.HttpMethods)
+
+const isOperation = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV3.OperationObject => {
+  return methods.includes(key)
+}
+
+const isParameter = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV3.ParameterObject => {
+  return !isOperation(key, pathItem)
+}
+
+const getRequestBody = (operation: OpenAPIV3.OperationObject) => {
+  if (requestBodyNotRef(operation.requestBody)) {
+    const request: OpenAPIV3.RequestBodyObject = operation.requestBody
+    const supportedMimeTypes = getMimeTypes(operation)
+    if (supportedMimeTypes.length > 0) {
+      const mimeType = supportedMimeTypes[0]
+
+      // try get example from request
+      const content = request.content[mimeType]
+      if (content.example) {
+        return content.example
+      }
+
+      // try get example from schema
+      if (schemaNotRef(content.schema)) {
+        const schema = content.schema
+        if (schema.example) {
+          return schema.example
+        }
+      }
+    }
+  }
+  return undefined
+}
+
+const getMimeTypes = (operation: OpenAPIV3.OperationObject): string[] => {
+  if (requestBodyNotRef(operation.requestBody)) {
+    const request: OpenAPIV3.RequestBodyObject = operation.requestBody
+    return Object.keys(request.content)
+  }
+  return []
+}
+
+/**
+ * OpenAPI Version 3.0
+ * https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md
+ */
+export class OpenAPI3 extends OpenAPISource {
+  document!: OpenAPIV3.Document
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const document: any = await this.parseData(data)
+      if (isOpenAPI3(document)) {
+        this.document = document
+        return true
+      } else {
+        return false
+      }
+    } catch (err) {
+      return false
+    }
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const name = this.document.info.title || "OpenAPI Import"
+    return {
+      name,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    let url: string | URL | undefined
+    if (this.document.servers?.length) {
+      url = this.document.servers[0].url
+      try {
+        url = new URL(url)
+      } catch (err) {
+        // unable to construct url, e.g. with variables
+        // proceed with string form of url
+      }
+    }
+
+    const queries: Query[] = []
+
+    for (let [path, pathItemObject] of Object.entries(this.document.paths)) {
+      // parameters that apply to every operation in the path
+      let pathParams: OpenAPIV3.ParameterObject[] = []
+
+      // pathItemObject can be undefined
+      if (!pathItemObject) {
+        continue
+      }
+
+      for (let [key, opOrParams] of Object.entries(pathItemObject)) {
+        if (isParameter(key, opOrParams)) {
+          const pathParameters = opOrParams as OpenAPIV3.ParameterObject[]
+          pathParams.push(...pathParameters)
+          continue
+        }
+        // can not be a parameter, must be an operation
+        const operation = opOrParams as OpenAPIV3.OperationObject
+
+        const methodName = key
+        const name = operation.operationId || path
+        let queryString = ""
+        const headers: any = {}
+        let requestBody = getRequestBody(operation)
+        const parameters: QueryParameter[] = []
+        const mimeTypes = getMimeTypes(operation)
+
+        if (mimeTypes.length > 0) {
+          headers["Content-Type"] = mimeTypes[0]
+        }
+
+        // combine the path parameters with the operation parameters
+        const operationParams = operation.parameters || []
+        const allParams = [...pathParams, ...operationParams]
+
+        for (let param of allParams) {
+          if (parameterNotRef(param)) {
+            switch (param.in) {
+              case "query":
+                let prefix = ""
+                if (queryString) {
+                  prefix = "&"
+                }
+                queryString = `${queryString}${prefix}${param.name}={{${param.name}}}`
+                break
+              case "header":
+                headers[param.name] = `{{${param.name}}}`
+                break
+              case "path":
+                // do nothing: param is already in the path
+                break
+              case "formData":
+                // future enhancement
+                break
+            }
+
+            // add the parameter if it can be bound in our config
+            if (["query", "header", "path"].includes(param.in)) {
+              parameters.push({
+                name: param.name,
+                default: "",
+              })
+            }
+          }
+        }
+
+        const query = this.constructQuery(
+          datasourceId,
+          name,
+          methodName,
+          path,
+          url,
+          queryString,
+          headers,
+          parameters,
+          requestBody
+        )
+        queries.push(query)
+      }
+    }
+
+    return queries
+  }
+}