@budibase/server 1.0.6 → 1.0.8-alpha.2

package/dist/utilities/csvParser.js

@@ -103,8 +103,11 @@ function transform({ schema, csvString, existingTable }) {
         if (existingTable) {
             schema = updateSchema({ schema, existingTable });
         }
-        for (let key of Object.keys(schema)) {
-            colParser[key] = PARSERS[schema[key].type] || schema[key].type;
+        for (let [key, field] of Object.entries(schema)) {
+            // don't import data to auto columns
+            if (!field.autocolumn) {
+                colParser[key] = PARSERS[field.type] || field.type;
+            }
         }
         try {
             const data = yield csv({ colParser }).fromString(csvString);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.6",
+  "version": "1.0.8-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -69,9 +69,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/auth": "^1.0.6",
-    "@budibase/client": "^1.0.6",
-    "@budibase/string-templates": "^1.0.6",
+    "@apidevtools/swagger-parser": "^10.0.3",
+    "@budibase/auth": "^1.0.8-alpha.2",
+    "@budibase/client": "^1.0.8-alpha.2",
+    "@budibase/string-templates": "^1.0.8-alpha.2",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -85,12 +86,14 @@
     "bull": "^3.22.4",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
+    "curlconverter": "^3.21.0",
     "dotenv": "8.2.0",
     "download": "8.0.0",
     "fix-path": "3.0.0",
     "fs-extra": "8.1.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
+    "js-yaml": "^4.1.0",
     "jsonschema": "1.4.0",
     "knex": "^0.95.6",
     "koa": "2.7.0",
@@ -146,6 +149,7 @@
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
+    "openapi-types": "^9.3.1",
     "path-to-regexp": "^6.2.0",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
@@ -158,5 +162,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "a6f684b996fe4f12e915e6f9ab0cbc79df4bdd0a"
+  "gitHead": "cc8b159e1124eec08661697ae70c61c1b12b78b7"
 }

package/src/api/controllers/query/import/index.ts

@@ -0,0 +1,85 @@
+import CouchDB from "../../../../db"
+import { queryValidation } from "../validation"
+import { generateQueryID } from "../../../../db/utils"
+import { ImportInfo, ImportSource } from "./sources/base"
+import { OpenAPI2 } from "./sources/openapi2"
+import { Query } from './../../../../definitions/common';
+import { Curl } from "./sources/curl"
+interface ImportResult {
+  errorQueries: Query[]
+  queries: Query[]
+}
+
+export class RestImporter {
+  data: string
+  sources: ImportSource[]
+  source!: ImportSource
+
+  constructor(data: string) {
+    this.data = data
+    this.sources = [new OpenAPI2(), new Curl()]
+  }
+
+  init = async () => {
+    for (let source of this.sources) {
+      if (await source.isSupported(this.data)) {
+        this.source = source
+        break
+      }
+    }
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    return this.source.getInfo()
+  }
+
+  importQueries = async (
+    appId: string,
+    datasourceId: string
+  ): Promise<ImportResult> => {
+    // constuct the queries
+    let queries = await this.source.getQueries(datasourceId)
+
+    // validate queries
+    const errorQueries: Query[] = []
+    const schema = queryValidation()
+    queries = queries
+      .filter(query => {
+        const validation = schema.validate(query)
+        if (validation.error) {
+          errorQueries.push(query)
+          return false
+        }
+        return true
+      })
+      .map(query => {
+        query._id = generateQueryID(query.datasourceId)
+        return query
+      })
+
+    // persist queries
+    const db = new CouchDB(appId)
+    const response = await db.bulkDocs(queries)
+
+    // create index to seperate queries and errors
+    const queryIndex = queries.reduce((acc, query) => {
+      if (query._id) {
+        acc[query._id] = query
+      }
+      return acc
+    }, {} as { [key: string]: Query })
+
+    // check for failed writes
+    response.forEach((query: any) => {
+      if (!query.ok) {
+        errorQueries.push(queryIndex[query.id])
+        delete queryIndex[query.id]
+      }
+    })
+
+    return {
+      errorQueries,
+      queries: Object.values(queryIndex),
+    }
+  }
+}

package/src/api/controllers/query/import/sources/base/index.ts

@@ -0,0 +1,86 @@
+
+import { Query, QueryParameter } from "../../../../../../definitions/common"
+
+export interface ImportInfo {
+  url: string
+  name: string
+}
+
+enum MethodToVerb {
+  get = "read",
+  post = "create",
+  put = "update",
+  patch = "patch",
+  delete = "delete",
+}
+
+export abstract class ImportSource {
+  abstract isSupported(data: string): Promise<boolean>
+  abstract getInfo(): Promise<ImportInfo>
+  abstract getQueries(datasourceId: string): Promise<Query[]>
+
+  constructQuery = (
+    datasourceId: string,
+    name: string,
+    method: string,
+    path: string,
+    queryString: string,
+    headers: object = {},
+    parameters: QueryParameter[] = [],
+    body: object | undefined = undefined
+  ): Query => {
+    const readable = true
+    const queryVerb = this.verbFromMethod(method)
+    const transformer = "return data"
+    const schema = {}
+    path = this.processPath(path)
+    queryString = this.processQuery(queryString)
+    const requestBody = JSON.stringify(body, null, 2)
+
+    const query: Query = {
+      datasourceId,
+      name,
+      parameters,
+      fields: {
+        headers,
+        queryString,
+        path,
+        requestBody,
+      },
+      transformer,
+      schema,
+      readable,
+      queryVerb,
+    }
+
+    return query
+  }
+
+  verbFromMethod = (method: string) => {
+    const verb = (<any>MethodToVerb)[method]
+    if (!verb) {
+      throw new Error(`Unsupported method: ${method}`)
+    }
+    return verb
+  }
+
+  processPath = (path: string): string => {
+    if (path?.startsWith("/")) {
+      path = path.substring(1)
+    }
+
+    // add extra braces around params for binding
+    path = path.replace(/[{]/g, "{{")
+    path = path.replace(/[}]/g, "}}")
+
+    return path
+  }
+
+  processQuery = (queryString: string): string => {
+    if (queryString?.startsWith("?")) {
+      return queryString.substring(1)
+    }
+
+    return queryString
+  }
+}

package/src/api/controllers/query/import/sources/base/openapi.ts

@@ -0,0 +1,24 @@
+import { ImportSource } from "."
+import SwaggerParser from "@apidevtools/swagger-parser"
+import { OpenAPI } from "openapi-types"
+const yaml = require("js-yaml")
+
+export abstract class OpenAPISource extends ImportSource {
+  parseData = async (data: string): Promise<OpenAPI.Document> => {
+    let json: OpenAPI.Document
+    try {
+      json = JSON.parse(data)
+    } catch (jsonErr) {
+      // couldn't parse json
+      // try to convert yaml -> json
+      try {
+        json = yaml.load(data)
+      } catch (yamlErr) {
+        // couldn't parse yaml
+        throw new Error("Could not parse JSON or YAML")
+      }
+    }
+
+    return SwaggerParser.validate(json, {})
+  }
+}

package/src/api/controllers/query/import/sources/curl.ts

@@ -0,0 +1,94 @@
+import { ImportSource, ImportInfo } from "./base"
+import { Query } from "../../../../../definitions/common"
+import { URL } from "url"
+const curlconverter = require("curlconverter")
+
+const parseCurl = (data: string): any => {
+  const curlJson = curlconverter.toJsonString(data)
+  return JSON.parse(curlJson)
+}
+
+/**
+ * The curl converter parses the request body into the key field of an object
+ * e.g. --d '{"key":"val"}' produces an object { "{"key":"val"}" : "" }
+ * This is not what we want, so we need to parse out the key from the object
+ */
+const parseBody = (curl: any) => {
+  if (curl.data) {
+    const keys = Object.keys(curl.data)
+    if (keys.length) {
+      const key = keys[0]
+      try {
+        return JSON.parse(key)
+      } catch (e) {
+        // do nothing
+      }
+    }
+  }
+  return undefined
+}
+
+const parseCookie = (curl: any) => {
+  if (curl.cookies){
+    return Object.entries(curl.cookies).reduce((acc, entry) => {
+      const [key, value] = entry
+      return acc + `${key}=${value}; `
+    }, "")
+  }
+
+  return null
+}
+
+/**
+ * Curl
+ * https://curl.se/docs/manpage.html
+ */
+export class Curl extends ImportSource {
+  curl: any
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const curl = parseCurl(data)
+      this.curl = curl
+    } catch (err) {
+      return false
+    }
+    return true
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const url = new URL(this.curl.url)
+    return {
+      url: url.origin,
+      name: url.hostname,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const url = new URL(this.curl.raw_url)
+    const name = url.pathname
+    const path = url.pathname
+    const method = this.curl.method
+    const queryString = url.search
+    const headers = this.curl.headers
+    const requestBody = parseBody(this.curl)
+
+    const cookieHeader = parseCookie(this.curl)
+    if (cookieHeader) {
+      headers["Cookie"] = cookieHeader
+    }
+
+    const query = this.constructQuery(
+      datasourceId,
+      name,
+      method,
+      path,
+      queryString,
+      headers,
+      [],
+      requestBody
+    )
+
+    return [query]
+  }
+}

package/src/api/controllers/query/import/sources/openapi2.ts

@@ -0,0 +1,159 @@
+import { ImportInfo } from "./base"
+import { Query, QueryParameter } from "../../../../../definitions/common"
+import { OpenAPIV2 } from "openapi-types"
+import { OpenAPISource } from "./base/openapi"
+
+const parameterNotRef = (
+  param: OpenAPIV2.Parameter | OpenAPIV2.ReferenceObject
+): param is OpenAPIV2.Parameter => {
+  // all refs are deferenced by parser library
+  return true
+}
+
+const isOpenAPI2 = (document: any): document is OpenAPIV2.Document => {
+  if (document.swagger === "2.0") {
+    return true
+  } else {
+    return false
+  }
+}
+
+const methods: string[] = Object.values(OpenAPIV2.HttpMethods)
+
+const isOperation = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV2.OperationObject => {
+  return methods.includes(key)
+}
+
+const isParameter = (
+  key: string,
+  pathItem: any
+): pathItem is OpenAPIV2.Parameter => {
+  return !isOperation(key, pathItem)
+}
+
+/**
+ * OpenAPI Version 2.0 - aka "Swagger"
+ * https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md
+ */
+export class OpenAPI2 extends OpenAPISource {
+  document!: OpenAPIV2.Document
+
+  isSupported = async (data: string): Promise<boolean> => {
+    try {
+      const document: any = await this.parseData(data)
+      if (isOpenAPI2(document)) {
+        this.document = document
+        return true
+      } else {
+        return false
+      }
+    } catch (err) {
+      return false
+    }
+  }
+
+  getInfo = async (): Promise<ImportInfo> => {
+    const scheme = this.document.schemes?.includes("https") ? "https" : "http"
+    const basePath = this.document.basePath || ""
+    const host = this.document.host || "<host>"
+    const url = `${scheme}://${host}${basePath}`
+    const name = this.document.info.title || "Swagger Import"
+
+    return {
+      url: url,
+      name: name,
+    }
+  }
+
+  getQueries = async (datasourceId: string): Promise<Query[]> => {
+    const queries = []
+
+    for (let [path, pathItem] of Object.entries(this.document.paths)) {
+      // parameters that apply to every operation in the path
+      let pathParams: OpenAPIV2.Parameter[] = []
+
+      for (let [key, opOrParams] of Object.entries(pathItem)) {
+        if (isParameter(key, opOrParams)) {
+          const pathParameters = opOrParams as OpenAPIV2.Parameter[]
+          pathParams.push(...pathParameters)
+          continue
+        }
+        // can not be a parameter, must be an operation
+        const operation = opOrParams as OpenAPIV2.OperationObject
+
+        const methodName = key
+        const name = operation.operationId || path
+        let queryString = ""
+        const headers: any = {}
+        let requestBody = undefined
+        const parameters: QueryParameter[] = []
+
+        if (operation.consumes) {
+          headers["Content-Type"] = operation.consumes[0]
+        }
+
+        // combine the path parameters with the operation parameters
+        const operationParams = operation.parameters || []
+        const allParams = [...pathParams, ...operationParams]
+
+        for (let param of allParams) {
+          if (parameterNotRef(param)) {
+            switch (param.in) {
+              case "query":
+                let prefix = ""
+                if (queryString) {
+                  prefix = "&"
+                }
+                queryString = `${queryString}${prefix}${param.name}={{${param.name}}}`
+                break
+              case "header":
+                headers[param.name] = `{{${param.name}}}`
+                break
+              case "path":
+                // do nothing: param is already in the path
+                break
+              case "formData":
+                // future enhancement
+                break
+              case "body":
+                // set the request body to the example provided
+                // future enhancement: generate an example from the schema
+                let bodyParam: OpenAPIV2.InBodyParameterObject =
+                  param as OpenAPIV2.InBodyParameterObject
+                if (param.schema.example) {
+                  const schema = bodyParam.schema as OpenAPIV2.SchemaObject
+                  requestBody = schema.example
+                }
+                break
+            }
+
+            // add the parameter if it can be bound in our config
+            if (["query", "header", "path"].includes(param.in)) {
+              parameters.push({
+                name: param.name,
+                default: param.default || "",
+              })
+            }
+          }
+        }
+
+        const query = this.constructQuery(
+          datasourceId,
+          name,
+          methodName,
+          path,
+          queryString,
+          headers,
+          parameters,
+          requestBody
+        )
+        queries.push(query)
+      }
+    }
+
+    return queries
+  }
+}

package/src/api/controllers/query/import/sources/tests/curl/curl.spec.js

@@ -0,0 +1,104 @@
+const { Curl } = require("../../curl")
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file) => {
+  return fs.readFileSync(path.join(__dirname, `./data/${file}.txt`), "utf8")
+}
+
+describe("Curl Import", () => {
+  let curl 
+
+  beforeEach(() => {
+    curl = new Curl()
+  })
+
+  it("validates unsupported data", async () => {
+    let data
+    let supported
+
+    // JSON
+    data = "{}"
+    supported = await curl.isSupported(data)
+    expect(supported).toBe(false)
+
+    // Empty
+    data = ""
+    supported = await curl.isSupported(data)
+    expect(supported).toBe(false)
+  })
+
+  const init = async (file) => {
+    await curl.isSupported(getData(file))
+  }
+
+  it("returns import info", async () => {
+    await init("get")
+    const info = await curl.getInfo()
+    expect(info.url).toBe("http://example.com")
+    expect(info.name).toBe("example.com")
+  })
+  
+  describe("Returns queries", () => {
+    
+    const getQueries = async (file) => {
+      await init(file)
+      const queries = await curl.getQueries()
+      expect(queries.length).toBe(1)
+      return queries
+    }
+
+    const testVerb = async (file, verb) => {
+      const queries = await getQueries(file)
+      expect(queries[0].queryVerb).toBe(verb)
+    }
+
+    it("populates verb", async () => {
+      await testVerb("get", "read")
+      await testVerb("post", "create")
+      await testVerb("put", "update")
+      await testVerb("delete", "delete")
+      await testVerb("patch", "patch")
+    }) 
+
+    const testPath = async (file, urlPath) => {
+      const queries = await getQueries(file)
+      expect(queries[0].fields.path).toBe(urlPath)
+    }
+
+    it("populates path", async () => {
+        await testPath("get", "")
+        await testPath("path", "paths/abc")
+    })
+
+    const testHeaders = async (file, headers) => {
+      const queries = await getQueries(file)
+      expect(queries[0].fields.headers).toStrictEqual(headers)
+    }
+
+    it("populates headers", async () => {
+      await testHeaders("get", {})
+      await testHeaders("headers", { "x-bb-header-1" : "123", "x-bb-header-2" : "456"} )
+    })
+
+    const testQuery = async (file, queryString) => {
+      const queries = await getQueries(file)
+      expect(queries[0].fields.queryString).toBe(queryString)
+    }
+    it("populates query", async () => {
+      await testQuery("get", "")
+      await testQuery("query", "q1=v1&q1=v2")
+    })
+
+    const testBody = async (file, body) => {
+      const queries = await getQueries(file)
+      expect(queries[0].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
+    }
+
+    it("populates body", async () => {
+      await testBody("get", undefined)
+      await testBody("post", { "key" : "val" })
+      await testBody("empty-body", {})
+    })
+  })
+})

package/src/api/controllers/query/import/sources/tests/curl/data/delete.txt

@@ -0,0 +1 @@
+curl -X DELETE 'http://example.com'

package/src/api/controllers/query/import/sources/tests/curl/data/empty-body.txt

@@ -0,0 +1,2 @@
+curl -X POST 'http://example.com' \
+  --data-raw '{}' 

package/src/api/controllers/query/import/sources/tests/curl/data/get.txt

@@ -0,0 +1 @@
+curl 'http://example.com'

package/src/api/controllers/query/import/sources/tests/curl/data/headers.txt

@@ -0,0 +1,3 @@
+curl 'http://example.com' \
+  -H 'x-bb-header-1: 123' \
+  -H 'x-bb-header-2: 456' 

package/src/api/controllers/query/import/sources/tests/curl/data/patch.txt

@@ -0,0 +1,2 @@
+curl -X PATCH 'http://example.com/paths/abc' \
+  --data-raw '{ "key" : "val" }' 

package/src/api/controllers/query/import/sources/tests/curl/data/path.txt

@@ -0,0 +1 @@
+curl 'http://example.com/paths/abc'

package/src/api/controllers/query/import/sources/tests/curl/data/post.txt

@@ -0,0 +1,2 @@
+curl -X POST 'http://example.com' \
+  --data-raw '{ "key" : "val" }' 

package/src/api/controllers/query/import/sources/tests/curl/data/put.txt

@@ -0,0 +1,2 @@
+curl -X PUT 'http://example.com/paths/abc' \
+  --data-raw '{ "key" : "val" }' 

package/src/api/controllers/query/import/sources/tests/curl/data/query.txt

@@ -0,0 +1 @@
+curl 'http://example.com/paths/abc?q1=v1&q1=v2'