@budibase/server 1.0.35 → 1.0.39

package/src/api/controllers/query/import/sources/base/index.ts

@@ -0,0 +1,87 @@
+import { Query, QueryParameter } from "../../../../../../definitions/datasource"
+import { URL } from "url"
+
+export interface ImportInfo {
+  name: string
+}
+
+enum MethodToVerb {
+  get = "read",
+  post = "create",
+  put = "update",
+  patch = "patch",
+  delete = "delete",
+}
+
+export abstract class ImportSource {
+  abstract isSupported(data: string): Promise<boolean>
+  abstract getInfo(): Promise<ImportInfo>
+  abstract getQueries(datasourceId: string): Promise<Query[]>
+
+  constructQuery = (
+    datasourceId: string,
+    name: string,
+    method: string,
+    path: string,
+    url: URL,
+    queryString: string,
+    headers: object = {},
+    parameters: QueryParameter[] = [],
+    body: object | undefined = undefined
+  ): Query => {
+    const readable = true
+    const queryVerb = this.verbFromMethod(method)
+    const transformer = "return data"
+    const schema = {}
+    path = this.processPath(path)
+    path = `${url.origin}/${path}`
+    queryString = this.processQuery(queryString)
+    const requestBody = JSON.stringify(body, null, 2)
+
+    const query: Query = {
+      datasourceId,
+      name,
+      parameters,
+      fields: {
+        headers,
+        queryString,
+        path,
+        requestBody,
+      },
+      transformer,
+      schema,
+      readable,
+      queryVerb,
+    }
+
+    return query
+  }
+
+  verbFromMethod = (method: string) => {
+    const verb = (<any>MethodToVerb)[method]
+    if (!verb) {
+      throw new Error(`Unsupported method: ${method}`)
+    }
+    return verb
+  }
+
+  processPath = (path: string): string => {
+    if (path?.startsWith("/")) {
+      path = path.substring(1)
+    }
+
+    // add extra braces around params for binding
+    path = path.replace(/[{]/g, "{{")
+    path = path.replace(/[}]/g, "}}")
+
+    return path
+  }
+
+  processQuery = (queryString: string): string => {
+    if (queryString?.startsWith("?")) {
+      return queryString.substring(1)
+    }
+
+    return queryString
+  }
+}

package/src/api/controllers/query/import/sources/base/openapi.ts

@@ -0,0 +1,24 @@
+import { ImportSource } from "."
+import SwaggerParser from "@apidevtools/swagger-parser"
+import { OpenAPI } from "openapi-types"
+const yaml = require("js-yaml")
+
+export abstract class OpenAPISource extends ImportSource {
+  parseData = async (data: string): Promise<OpenAPI.Document> => {
+    let json: OpenAPI.Document
+    try {
+      json = JSON.parse(data)
+    } catch (jsonErr) {
+      // couldn't parse json
+      // try to convert yaml -> json
+      try {
+        json = yaml.load(data)
+      } catch (yamlErr) {
+        // couldn't parse yaml
+        throw new Error("Could not parse JSON or YAML")
+      }
+    }
+
+    return SwaggerParser.validate(json, {})
+  }
+}

package/src/api/controllers/query/import/sources/curl.ts

@@ -0,0 +1,102 @@
+import { ImportSource, ImportInfo } from "./base"
+import { Query } from "../../../../../definitions/common"
+import { URL } from "url"
+const curlconverter = require("curlconverter")
+
+const parseCurl = (data: string): any => {
+  const curlJson = curlconverter.toJsonString(data)
+  return JSON.parse(curlJson)
+}
+
+/**
+ * The curl converter parses the request body into the key field of an object
+ * e.g. --d '{"key":"val"}' produces an object { "{"key":"val"}" : "" }
+ * This is not what we want, so we need to parse out the key from the object
+ */
+const parseBody = (curl: any) => {
+  if (curl.data) {
+    const keys = Object.keys(curl.data)
+    if (keys.length) {
+      let key = keys[0]
+      try {
+        // filter out the dollar syntax used by curl for shell support
+        if (key.startsWith("$")) {
+          key = key.substring(1)
+        }
+        return JSON.parse(key)
+      } catch (e) {
+        // do nothing
+      }
+    }
+  }
+  return undefined
+}
+
+const parseCookie = (curl: any) => {
+  if (curl.cookies) {
+    return Object.entries(curl.cookies).reduce((acc, entry) => {
+      const [key, value] = entry
+      return acc + `${key}=${value}; `
+    }, "")
+  }
+
+  return null
+}
+
+/**
+ * Curl
+ * https://curl.se/docs/manpage.html
+ */
+export class Curl extends ImportSource {
+  curl: any
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const curl = parseCurl(data)
+      this.curl = curl
+    } catch (err) {
+      return false
+    }
+    return true
+  }
+
+  getUrl = (): URL => {
+    return new URL(this.curl.raw_url)
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const url = this.getUrl()
+    return {
+      name: url.hostname,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const url = this.getUrl()
+    const name = url.pathname
+    const path = url.pathname
+    const method = this.curl.method
+    const queryString = url.search
+    const headers = this.curl.headers
+    const requestBody = parseBody(this.curl)
+
+    const cookieHeader = parseCookie(this.curl)
+    if (cookieHeader) {
+      headers["Cookie"] = cookieHeader
+    }
+
+    const query = this.constructQuery(
+      datasourceId,
+      name,
+      method,
+      path,
+      url,
+      queryString,
+      headers,
+      [],
+      requestBody
+    )
+
+    return [query]
+  }
+}

package/src/api/controllers/query/import/sources/openapi2.ts

@@ -0,0 +1,163 @@
+import { ImportInfo } from "./base"
+import { Query, QueryParameter } from "../../../../../definitions/datasource"
+import { OpenAPIV2 } from "openapi-types"
+import { OpenAPISource } from "./base/openapi"
+import { URL } from "url"
+
+const parameterNotRef = (
+  param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject
+): param is OpenAPIV2.Parameter => {
+  // all refs are deferenced by parser library
+  return true
+}
+
+const isOpenAPI2 = (document: any): document is OpenAPIV2.Document => {
+  if (document.swagger === "2.0") {
+    return true
+  } else {
+    return false
+  }
+}
+
+const methods: string[] = Object.values(OpenAPIV2.HttpMethods)
+
+const isOperation = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV2.OperationObject => {
+  return methods.includes(key)
+}
+
+const isParameter = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV2.Parameter => {
+  return !isOperation(key, pathItem)
+}
+
+/**
+ * OpenAPI Version 2.0 - aka "Swagger"
+ * https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md
+ */
+export class OpenAPI2 extends OpenAPISource {
+  document!: OpenAPIV2.Document
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const document: any = await this.parseData(data)
+      if (isOpenAPI2(document)) {
+        this.document = document
+        return true
+      } else {
+        return false
+      }
+    } catch (err) {
+      return false
+    }
+  }
+
+  getUrl = (): URL => {
+    const scheme = this.document.schemes?.includes("https") ? "https" : "http"
+    const basePath = this.document.basePath || ""
+    const host = this.document.host || "<host>"
+    return new URL(`${scheme}://${host}${basePath}`)
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const name = this.document.info.title || "Swagger Import"
+    return {
+      name,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const url = this.getUrl()
+    const queries = []
+
+    for (let [path, pathItem] of Object.entries(this.document.paths)) {
+      // parameters that apply to every operation in the path
+      let pathParams: OpenAPIV2.Parameter[] = []
+
+      for (let [key, opOrParams] of Object.entries(pathItem)) {
+        if (isParameter(key, opOrParams)) {
+          const pathParameters = opOrParams as OpenAPIV2.Parameter[]
+          pathParams.push(...pathParameters)
+          continue
+        }
+        // can not be a parameter, must be an operation
+        const operation = opOrParams as OpenAPIV2.OperationObject
+
+        const methodName = key
+        const name = operation.operationId || path
+        let queryString = ""
+        const headers: any = {}
+        let requestBody = undefined
+        const parameters: QueryParameter[] = []
+
+        if (operation.consumes) {
+          headers["Content-Type"] = operation.consumes[0]
+        }
+
+        // combine the path parameters with the operation parameters
+        const operationParams = operation.parameters || []
+        const allParams = [...pathParams, ...operationParams]
+
+        for (let param of allParams) {
+          if (parameterNotRef(param)) {
+            switch (param.in) {
+              case "query":
+                let prefix = ""
+                if (queryString) {
+                  prefix = "&"
+                }
+                queryString = `${queryString}${prefix}${param.name}={{${param.name}}}`
+                break
+              case "header":
+                headers[param.name] = `{{${param.name}}}`
+                break
+              case "path":
+                // do nothing: param is already in the path
+                break
+              case "formData":
+                // future enhancement
+                break
+              case "body":
+                // set the request body to the example provided
+                // future enhancement: generate an example from the schema
+                let bodyParam: OpenAPIV2.InBodyParameterObject =
+                  param as OpenAPIV2.InBodyParameterObject
+                if (param.schema.example) {
+                  const schema = bodyParam.schema as OpenAPIV2.SchemaObject
+                  requestBody = schema.example
+                }
+                break
+            }
+
+            // add the parameter if it can be bound in our config
+            if (["query", "header", "path"].includes(param.in)) {
+              parameters.push({
+                name: param.name,
+                default: param.default || "",
+              })
+            }
+          }
+        }
+
+        const query = this.constructQuery(
+          datasourceId,
+          name,
+          methodName,
+          path,
+          url,
+          queryString,
+          headers,
+          parameters,
+          requestBody
+        )
+        queries.push(query)
+      }
+    }
+
+    return queries
+  }
+}

package/src/api/controllers/query/import/sources/tests/curl/curl.spec.js

@@ -0,0 +1,103 @@
+const { Curl } = require("../../curl")
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file) => {
+  return fs.readFileSync(path.join(__dirname, `./data/${file}.txt`), "utf8")
+}
+
+describe("Curl Import", () => {
+  let curl 
+
+  beforeEach(() => {
+    curl = new Curl()
+  })
+
+  it("validates unsupported data", async () => {
+    let data
+    let supported
+
+    // JSON
+    data = "{}"
+    supported = await curl.isSupported(data)
+    expect(supported).toBe(false)
+
+    // Empty
+    data = ""
+    supported = await curl.isSupported(data)
+    expect(supported).toBe(false)
+  })
+
+  const init = async (file) => {
+    await curl.isSupported(getData(file))
+  }
+
+  it("returns import info", async () => {
+    await init("get")
+    const info = await curl.getInfo()
+    expect(info.name).toBe("example.com")
+  })
+  
+  describe("Returns queries", () => {
+    
+    const getQueries = async (file) => {
+      await init(file)
+      const queries = await curl.getQueries()
+      expect(queries.length).toBe(1)
+      return queries
+    }
+
+    const testVerb = async (file, verb) => {
+      const queries = await getQueries(file)
+      expect(queries[0].queryVerb).toBe(verb)
+    }
+
+    it("populates verb", async () => {
+      await testVerb("get", "read")
+      await testVerb("post", "create")
+      await testVerb("put", "update")
+      await testVerb("delete", "delete")
+      await testVerb("patch", "patch")
+    }) 
+
+    const testPath = async (file, urlPath) => {
+      const queries = await getQueries(file)
+      expect(queries[0].fields.path).toBe(urlPath)
+    }
+
+    it("populates path", async () => {
+        await testPath("get", "http://example.com/")
+        await testPath("path", "http://example.com/paths/abc")
+    })
+
+    const testHeaders = async (file, headers) => {
+      const queries = await getQueries(file)
+      expect(queries[0].fields.headers).toStrictEqual(headers)
+    }
+
+    it("populates headers", async () => {
+      await testHeaders("get", {})
+      await testHeaders("headers", { "x-bb-header-1" : "123", "x-bb-header-2" : "456"} )
+    })
+
+    const testQuery = async (file, queryString) => {
+      const queries = await getQueries(file)
+      expect(queries[0].fields.queryString).toBe(queryString)
+    }
+    it("populates query", async () => {
+      await testQuery("get", "")
+      await testQuery("query", "q1=v1&q1=v2")
+    })
+
+    const testBody = async (file, body) => {
+      const queries = await getQueries(file)
+      expect(queries[0].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
+    }
+
+    it("populates body", async () => {
+      await testBody("get", undefined)
+      await testBody("post", { "key" : "val" })
+      await testBody("empty-body", {})
+    })
+  })
+})

package/src/api/controllers/query/import/sources/tests/curl/data/delete.txt

@@ -0,0 +1 @@
+curl -X DELETE 'http://example.com'

package/src/api/controllers/query/import/sources/tests/curl/data/empty-body.txt

@@ -0,0 +1,2 @@
+curl -X POST 'http://example.com' \
+  --data-raw '{}' 

package/src/api/controllers/query/import/sources/tests/curl/data/get.txt

@@ -0,0 +1 @@
+curl 'http://example.com'

package/src/api/controllers/query/import/sources/tests/curl/data/headers.txt

@@ -0,0 +1,3 @@
+curl 'http://example.com' \
+  -H 'x-bb-header-1: 123' \
+  -H 'x-bb-header-2: 456' 

package/src/api/controllers/query/import/sources/tests/curl/data/patch.txt

@@ -0,0 +1,2 @@
+curl -X PATCH 'http://example.com/paths/abc' \
+  --data-raw '{ "key" : "val" }' 

package/src/api/controllers/query/import/sources/tests/curl/data/path.txt

@@ -0,0 +1 @@
+curl 'http://example.com/paths/abc'

package/src/api/controllers/query/import/sources/tests/curl/data/post.txt

@@ -0,0 +1,2 @@
+curl -X POST 'http://example.com' \
+  --data-raw '{ "key" : "val" }' 

package/src/api/controllers/query/import/sources/tests/curl/data/put.txt

@@ -0,0 +1,2 @@
+curl -X PUT 'http://example.com/paths/abc' \
+  --data-raw '{ "key" : "val" }' 

package/src/api/controllers/query/import/sources/tests/curl/data/query.txt

@@ -0,0 +1 @@
+curl 'http://example.com/paths/abc?q1=v1&q1=v2'