@budibase/server 1.0.99 → 1.0.102

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.99",
+  "version": "1.0.102",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -68,12 +68,13 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.99",
-    "@budibase/client": "^1.0.99",
-    "@budibase/string-templates": "^1.0.99",
+    "@budibase/backend-core": "^1.0.102",
+    "@budibase/client": "^1.0.102",
+    "@budibase/string-templates": "^1.0.102",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
+    "@google-cloud/firestore": "^5.0.2",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
     "@sentry/node": "^6.0.0",
@@ -145,6 +146,7 @@
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
+    "@types/lodash": "4.14.180",
     "@types/node": "^15.12.4",
     "@types/oracledb": "^5.2.1",
     "@types/redis": "^4.0.11",
@@ -153,6 +155,7 @@
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
+    "is-wsl": "^2.2.0",
     "jest": "^27.0.5",
     "jest-openapi": "^0.14.2",
     "nodemon": "^2.0.4",
@@ -171,5 +174,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "c5880f9ec2a22c0a04388e3cd4727a19ad14bfd7"
+  "gitHead": "665f572421060b82166b89c2114ffaea72390102"
 }

package/scripts/dev/manage.js

@@ -2,10 +2,11 @@
 const compose = require("docker-compose")
 const path = require("path")
 const fs = require("fs")
+const isWsl = require("is-wsl")
 const { processStringSync } = require("@budibase/string-templates")
 
 function isLinux() {
-  return process.platform !== "darwin" && process.platform !== "win32"
+  return !isWsl && process.platform !== "darwin" && process.platform !== "win32"
 }
 
 // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.

package/src/api/controllers/integration.js

@@ -1,8 +1,24 @@
+const { cloneDeep } = require("lodash")
 const { definitions } = require("../../integrations")
+const { getTenantId } = require("@budibase/backend-core/tenancy")
+const { SourceNames } = require("../../definitions/datasource")
+const googlesheets = require("../../integrations/googlesheets")
+const env = require("../../environment")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
-  ctx.body = definitions
+  const defs = cloneDeep(definitions)
+
+  // for google sheets integration google verification
+  if (env.EXCLUDE_QUOTAS_TENANTS) {
+    const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.split(",")
+    const tenantId = getTenantId()
+    if (excludedTenants.includes(tenantId)) {
+      defs[SourceNames.GOOGLE_SHEETS] = googlesheets.schema
+    }
+  }
+
+  ctx.body = defs
 }
 
 exports.find = async function (ctx) {

package/src/api/controllers/query/import/index.ts

@@ -2,6 +2,7 @@ import { queryValidation } from "../validation"
 import { generateQueryID } from "../../../../db/utils"
 import { ImportInfo, ImportSource } from "./sources/base"
 import { OpenAPI2 } from "./sources/openapi2"
+import { OpenAPI3 } from "./sources/openapi3"
 import { Query } from "./../../../../definitions/common"
 import { Curl } from "./sources/curl"
 // @ts-ignore
@@ -18,7 +19,7 @@ export class RestImporter {
 
   constructor(data: string) {
     this.data = data
-    this.sources = [new OpenAPI2(), new Curl()]
+    this.sources = [new OpenAPI2(), new OpenAPI3(), new Curl()]
   }
 
   init = async () => {

package/src/api/controllers/query/import/sources/base/index.ts

@@ -23,7 +23,7 @@ export abstract class ImportSource {
     name: string,
     method: string,
     path: string,
-    url: URL,
+    url: URL | string | undefined,
     queryString: string,
     headers: object = {},
     parameters: QueryParameter[] = [],
@@ -34,7 +34,17 @@ export abstract class ImportSource {
     const transformer = "return data"
     const schema = {}
     path = this.processPath(path)
-    path = `${url.origin}/${path}`
+    if (url) {
+      if (typeof url === "string") {
+        path = `${url}/${path}`
+      } else {
+        let href = url.href
+        if (href.endsWith("/")) {
+          href = href.slice(0, -1)
+        }
+        path = `${href}/${path}`
+      }
+    }
     queryString = this.processQuery(queryString)
     const requestBody = JSON.stringify(body, null, 2)
 

package/src/api/controllers/query/import/sources/curl.ts

@@ -74,7 +74,7 @@ export class Curl extends ImportSource {
   getQueries = async (datasourceId: string): Promise<Query[]> => {
     const url = this.getUrl()
     const name = url.pathname
-    const path = url.pathname
+    const path = url.origin + url.pathname
     const method = this.curl.method
     const queryString = url.search
     const headers = this.curl.headers
@@ -90,7 +90,7 @@ export class Curl extends ImportSource {
       name,
       method,
       path,
-      url,
+      undefined,
       queryString,
       headers,
       [],

package/src/api/controllers/query/import/sources/openapi3.ts

@@ -0,0 +1,205 @@
+import { ImportInfo } from "./base"
+import { Query, QueryParameter } from "../../../../../definitions/datasource"
+import { OpenAPIV3 } from "openapi-types"
+import { OpenAPISource } from "./base/openapi"
+import { URL } from "url"
+
+const parameterNotRef = (
+  param: OpenAPIV3.ParameterObject | OpenAPIV3.ReferenceObject
+): param is OpenAPIV3.ParameterObject => {
+  // all refs are deferenced by parser library
+  return true
+}
+
+const requestBodyNotRef = (
+  param: OpenAPIV3.RequestBodyObject | OpenAPIV3.ReferenceObject | undefined
+): param is OpenAPIV3.RequestBodyObject => {
+  // all refs are deferenced by parser library
+  return param !== undefined
+}
+
+const schemaNotRef = (
+  param: OpenAPIV3.SchemaObject | OpenAPIV3.ReferenceObject | undefined
+): param is OpenAPIV3.SchemaObject => {
+  // all refs are deferenced by parser library
+  return param !== undefined
+}
+
+const isOpenAPI3 = (document: any): document is OpenAPIV3.Document => {
+  return document.openapi.includes("3.0")
+}
+
+const methods: string[] = Object.values(OpenAPIV3.HttpMethods)
+
+const isOperation = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV3.OperationObject => {
+  return methods.includes(key)
+}
+
+const isParameter = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV3.ParameterObject => {
+  return !isOperation(key, pathItem)
+}
+
+const getRequestBody = (operation: OpenAPIV3.OperationObject) => {
+  if (requestBodyNotRef(operation.requestBody)) {
+    const request: OpenAPIV3.RequestBodyObject = operation.requestBody
+    const supportedMimeTypes = getMimeTypes(operation)
+    if (supportedMimeTypes.length > 0) {
+      const mimeType = supportedMimeTypes[0]
+
+      // try get example from request
+      const content = request.content[mimeType]
+      if (content.example) {
+        return content.example
+      }
+
+      // try get example from schema
+      if (schemaNotRef(content.schema)) {
+        const schema = content.schema
+        if (schema.example) {
+          return schema.example
+        }
+      }
+    }
+  }
+  return undefined
+}
+
+const getMimeTypes = (operation: OpenAPIV3.OperationObject): string[] => {
+  if (requestBodyNotRef(operation.requestBody)) {
+    const request: OpenAPIV3.RequestBodyObject = operation.requestBody
+    return Object.keys(request.content)
+  }
+  return []
+}
+
+/**
+ * OpenAPI Version 3.0
+ * https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md
+ */
+export class OpenAPI3 extends OpenAPISource {
+  document!: OpenAPIV3.Document
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const document: any = await this.parseData(data)
+      if (isOpenAPI3(document)) {
+        this.document = document
+        return true
+      } else {
+        return false
+      }
+    } catch (err) {
+      return false
+    }
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const name = this.document.info.title || "OpenAPI Import"
+    return {
+      name,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    let url: string | URL | undefined
+    if (this.document.servers?.length) {
+      url = this.document.servers[0].url
+      try {
+        url = new URL(url)
+      } catch (err) {
+        // unable to construct url, e.g. with variables
+        // proceed with string form of url
+      }
+    }
+
+    const queries: Query[] = []
+
+    for (let [path, pathItemObject] of Object.entries(this.document.paths)) {
+      // parameters that apply to every operation in the path
+      let pathParams: OpenAPIV3.ParameterObject[] = []
+
+      // pathItemObject can be undefined
+      if (!pathItemObject) {
+        continue
+      }
+
+      for (let [key, opOrParams] of Object.entries(pathItemObject)) {
+        if (isParameter(key, opOrParams)) {
+          const pathParameters = opOrParams as OpenAPIV3.ParameterObject[]
+          pathParams.push(...pathParameters)
+          continue
+        }
+        // can not be a parameter, must be an operation
+        const operation = opOrParams as OpenAPIV3.OperationObject
+
+        const methodName = key
+        const name = operation.operationId || path
+        let queryString = ""
+        const headers: any = {}
+        let requestBody = getRequestBody(operation)
+        const parameters: QueryParameter[] = []
+        const mimeTypes = getMimeTypes(operation)
+
+        if (mimeTypes.length > 0) {
+          headers["Content-Type"] = mimeTypes[0]
+        }
+
+        // combine the path parameters with the operation parameters
+        const operationParams = operation.parameters || []
+        const allParams = [...pathParams, ...operationParams]
+
+        for (let param of allParams) {
+          if (parameterNotRef(param)) {
+            switch (param.in) {
+              case "query":
+                let prefix = ""
+                if (queryString) {
+                  prefix = "&"
+                }
+                queryString = `${queryString}${prefix}${param.name}={{${param.name}}}`
+                break
+              case "header":
+                headers[param.name] = `{{${param.name}}}`
+                break
+              case "path":
+                // do nothing: param is already in the path
+                break
+              case "formData":
+                // future enhancement
+                break
+            }
+
+            // add the parameter if it can be bound in our config
+            if (["query", "header", "path"].includes(param.in)) {
+              parameters.push({
+                name: param.name,
+                default: "",
+              })
+            }
+          }
+        }
+
+        const query = this.constructQuery(
+          datasourceId,
+          name,
+          methodName,
+          path,
+          url,
+          queryString,
+          headers,
+          parameters,
+          requestBody
+        )
+        queries.push(query)
+      }
+    }
+
+    return queries
+  }
+}

package/src/api/controllers/query/import/sources/tests/openapi3/data/crud/crud.json

@@ -0,0 +1,253 @@
+{
+  "openapi": "3.0.2",
+  "info": {
+    "description": "A basic swagger file",
+    "version": "1.0.0",
+    "title": "CRUD"
+  },
+  "servers": [
+    {
+      "url": "http://example.com"
+    }
+  ],
+  "tags": [
+    {
+      "name": "entity"
+    }
+  ],
+  "paths": {
+    "/entities": {
+      "post": {
+        "tags": [
+          "entity"
+        ],
+        "operationId": "createEntity",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/CreateEntity"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/Entity"
+                }
+              }
+            }
+          }
+        }
+      },
+      "get": {
+        "tags": [
+          "entity"
+        ],
+        "operationId": "getEntities",
+        "parameters": [
+          {
+            "$ref": "#/components/parameters/PageParameter"
+          },
+          {
+            "$ref": "#/components/parameters/SizeParameter"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/Entities"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "/entities/{entityId}": {
+      "parameters": [
+        {
+          "$ref": "#/components/parameters/EntityIdParameter"
+        }
+      ],
+      "get": {
+        "tags": [
+          "entity"
+        ],
+        "operationId": "getEntity",
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/Entity"
+                }
+              }
+            }
+          }
+        }
+      },
+      "put": {
+        "tags": [
+          "entity"
+        ],
+        "operationId": "updateEntity",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/Entity"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/Entity"
+                }
+              }
+            }
+          }
+        }
+      },
+      "patch": {
+        "tags": [
+          "entity"
+        ],
+        "operationId": "patchEntity",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/Entity"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/Entity"
+                }
+              }
+            }
+          }
+        }
+      },
+      "delete": {
+        "tags": [
+          "entity"
+        ],
+        "parameters": [
+          {
+            "$ref": "#/components/parameters/APIKeyParameter"
+          }
+        ],
+        "operationId": "deleteEntity",
+        "responses": {
+          "204": {
+            "description": "successful operation"
+          }
+        }
+      }
+    }
+  },
+  "components": {
+    "parameters": {
+      "EntityIdParameter": {
+        "schema": {
+          "type": "integer",
+          "format": "int64"
+        },
+        "name": "entityId",
+        "in": "path",
+        "required": true
+      },
+     "PageParameter": {
+       "schema": {
+         "type": "integer",
+         "format": "int32"
+        },
+        "name": "page",
+        "in": "query",
+        "required": false
+      },
+      "SizeParameter": {
+        "schema": {
+          "type": "integer",
+          "format": "int32"
+        },
+        "name": "size",
+        "in": "query",
+        "required": false
+      },
+      "APIKeyParameter": {
+        "schema": {
+          "type": "string"
+        },
+        "name": "x-api-key",
+        "in": "header",
+        "required": false
+      }
+    },
+    "schemas": {
+      "CreateEntity": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "type": {
+            "type": "string"
+          }
+        },
+        "example": {
+          "name": "name",
+          "type": "type"
+        }
+      },
+      "Entity": {
+        "allOf": [
+          {
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "integer",
+                "format": "int64"
+              }
+            }
+          },
+          {
+            "$ref": "#/components/schemas/CreateEntity"
+          }
+        ],
+        "example": {
+          "id": 1,
+          "name": "name",
+          "type": "type"
+        }
+      },
+      "Entities" : {
+        "type": "array",
+        "items": {
+          "$ref": "#/components/schemas/Entity"
+        }
+      }
+      }
+  }
+}