@budibase/server 1.0.6 → 1.0.8-alpha.2

package/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js

@@ -0,0 +1,239 @@
+const { OpenAPI2 } = require("../../openapi2")
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file, extension) => {
+  return fs.readFileSync(path.join(__dirname, `./data/${file}/${file}.${extension}`), "utf8")
+}
+
+describe("OpenAPI2 Import", () => {
+  let openapi2 
+
+  beforeEach(() => {
+    openapi2 = new OpenAPI2()
+  })
+
+  it("validates unsupported data", async () => {
+    let data
+    let supported
+
+    // non json / yaml
+    data = "curl http://example.com"
+    supported = await openapi2.isSupported(data)
+    expect(supported).toBe(false)
+
+    // Empty
+    data = ""
+    supported = await openapi2.isSupported(data)
+    expect(supported).toBe(false)
+  })
+
+  const init = async (file, extension) => {
+    await openapi2.isSupported(getData(file, extension))
+  }
+
+  const runTests = async (filename, test, assertions) => {
+    for (let extension of ["json", "yaml"]) {
+      await test(filename, extension, assertions)
+    }
+  }
+
+  const testImportInfo = async (file, extension) => {
+    await init(file, extension)
+    const info = await openapi2.getInfo()
+    expect(info.url).toBe("https://petstore.swagger.io/v2")
+    expect(info.name).toBe("Swagger Petstore")
+  }
+
+  it("returns import info", async () => {
+    await runTests("petstore", testImportInfo)
+  })
+  
+  describe("Returns queries", () => {
+    const indexQueries = (queries) => {
+      return queries.reduce((acc, query) => {
+        acc[query.name] = query
+        return acc
+      }, {})
+    }
+
+    const getQueries = async (file, extension) => {
+      await init(file, extension)
+      const queries = await openapi2.getQueries()
+      expect(queries.length).toBe(6)
+      return indexQueries(queries)
+    }
+
+    const testVerb = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, method] of Object.entries(assertions)) {
+        expect(queries[operationId].queryVerb).toBe(method)
+      }
+    }
+
+    it("populates verb", async () => {
+      const assertions = {
+        "createEntity" : "create",
+        "getEntities" : "read",
+        "getEntity" : "read",
+        "updateEntity" : "update",
+        "patchEntity" : "patch",
+        "deleteEntity" : "delete"
+      }
+      await runTests("crud", testVerb, assertions)
+    }) 
+
+    const testPath = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, urlPath] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.path).toBe(urlPath)
+      }
+    }
+
+    it("populates path", async () => {
+      const assertions = {
+        "createEntity" : "entities",
+        "getEntities" : "entities",
+        "getEntity" : "entities/{{entityId}}",
+        "updateEntity" : "entities/{{entityId}}",
+        "patchEntity" : "entities/{{entityId}}",
+        "deleteEntity" : "entities/{{entityId}}"
+      }
+      await runTests("crud", testPath, assertions)
+    })
+
+    const testHeaders = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, headers] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.headers).toStrictEqual(headers)
+      }
+    }
+
+    const contentTypeHeader = {
+      "Content-Type" : "application/json",
+    }
+
+    it("populates headers", async () => {
+      const assertions = {
+        "createEntity" : {
+         ...contentTypeHeader
+        },
+        "getEntities" : {
+        },
+        "getEntity" : {
+        },
+        "updateEntity" : {
+          ...contentTypeHeader
+        },
+        "patchEntity" : {
+          ...contentTypeHeader
+        },
+        "deleteEntity" : {
+            "x-api-key" : "{{x-api-key}}",
+        }
+      }
+
+      await runTests("crud", testHeaders, assertions)
+    })
+
+    const testQuery = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, queryString] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.queryString).toStrictEqual(queryString)
+      }
+    }
+
+    it("populates query", async () => {
+      const assertions = {
+        "createEntity" : "",
+        "getEntities" : "page={{page}}&size={{size}}",
+        "getEntity" : "",
+        "updateEntity" : "",
+        "patchEntity" : "",
+        "deleteEntity" : ""
+      }
+      await runTests("crud", testQuery, assertions)
+    })
+
+    const testParameters = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, parameters] of Object.entries(assertions)) {
+        expect(queries[operationId].parameters).toStrictEqual(parameters)
+      }
+    }
+
+    it("populates parameters", async () => {
+      const assertions = {
+        "createEntity" : [],
+        "getEntities" : [
+          {
+            "name" : "page",
+            "default" : "",
+          },
+          {
+            "name" : "size",
+            "default" : "",
+          }
+        ],
+        "getEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "updateEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "patchEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "deleteEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          },
+          {
+            "name" : "x-api-key",
+            "default" : "",
+          }
+        ]
+      }
+      await runTests("crud", testParameters, assertions)
+    })
+
+    const testBody = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, body] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
+      }
+    }
+    it("populates body", async () => {
+      const assertions = {
+        "createEntity" : {
+          "name" : "name",
+          "type" : "type",
+        },
+        "getEntities" : undefined,
+        "getEntity" : undefined,
+        "updateEntity" : {
+          "id": 1,
+          "name" : "name",
+          "type" : "type",
+        },
+        "patchEntity" : {
+          "id": 1,
+          "name" : "name",
+          "type" : "type",
+        },
+        "deleteEntity" : undefined
+      }
+      await runTests("crud", testBody, assertions)
+    })
+  })
+})

package/src/api/controllers/query/import/tests/index.spec.js

@@ -0,0 +1,115 @@
+
+const bulkDocs = jest.fn()
+const db = jest.fn(() => {
+  return {
+    bulkDocs
+  }
+})
+jest.mock("../../../../../db", () => db)
+
+const { RestImporter } = require("../index")
+
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file) => {
+  return fs.readFileSync(path.join(__dirname, `../sources/tests/${file}`), "utf8")
+}
+
+// openapi2 (swagger)
+const oapi2CrudJson = getData("openapi2/data/crud/crud.json")
+const oapi2CrudYaml = getData("openapi2/data/crud/crud.json")
+const oapi2PetstoreJson = getData("openapi2/data/petstore/petstore.json")
+const oapi2PetstoreYaml = getData("openapi2/data/petstore/petstore.json")
+
+// curl
+const curl = getData("curl/data/post.txt")
+
+const datasets = {
+  oapi2CrudJson,
+  oapi2CrudYaml,
+  oapi2PetstoreJson,
+  oapi2PetstoreYaml,
+  curl
+}
+
+describe("Rest Importer", () => {
+  let restImporter 
+
+  const init = async (data) => {
+    restImporter = new RestImporter(data)
+    await restImporter.init()
+  }
+
+  const runTest = async (test, assertions) => {
+    for (let [key, data] of Object.entries(datasets)) {
+      await test(key, data, assertions)
+    }
+  }
+
+  const testGetInfo = async (key, data, assertions) => {
+    await init(data)
+    const info = await restImporter.getInfo()
+    expect(info.name).toBe(assertions[key].name)
+    expect(info.url).toBe(assertions[key].url)
+  }
+
+  it("gets info", async () => {
+    const assertions = {
+      "oapi2CrudJson" : {
+        name: "CRUD",
+        url: "http://example.com"
+      },
+      "oapi2CrudYaml" : {
+        name: "CRUD",
+        url: "http://example.com"
+      },
+      "oapi2PetstoreJson" : {
+        name: "Swagger Petstore",
+        url: "https://petstore.swagger.io/v2"
+      },
+      "oapi2PetstoreYaml" :{
+        name: "Swagger Petstore",
+        url: "https://petstore.swagger.io/v2"
+      },
+      "curl": {
+        name: "example.com",
+        url: "http://example.com"
+      }
+    }
+    await runTest(testGetInfo, assertions)
+  })
+
+  const testImportQueries = async (key, data, assertions) => {
+    await init(data)
+    bulkDocs.mockReturnValue([])
+    const importResult = await restImporter.importQueries("appId", "datasourceId")
+    expect(importResult.errorQueries.length).toBe(0)
+    expect(importResult.queries.length).toBe(assertions[key].count)
+    expect(bulkDocs).toHaveBeenCalledTimes(1)
+    jest.clearAllMocks()
+  }
+
+  it("imports queries", async () => {
+    // simple sanity assertions that the whole dataset 
+    // makes it through the importer
+    const assertions = {
+      "oapi2CrudJson" : {
+        count: 6,
+      },
+      "oapi2CrudYaml" :{
+        count: 6,
+      },
+      "oapi2PetstoreJson" : {
+        count: 20,
+      },
+      "oapi2PetstoreYaml" :{
+        count: 20,
+      },
+      "curl": {
+        count: 1
+      }
+    }
+    await runTest(testImportQueries, assertions)
+  })
+})

package/src/api/controllers/{query.js → query/index.js}

@@ -1,12 +1,14 @@
 const { processString } = require("@budibase/string-templates")
-const CouchDB = require("../../db")
+const CouchDB = require("../../../db")
 const {
   generateQueryID,
   getQueryParams,
   isProdAppID,
-} = require("../../db/utils")
-const { BaseQueryVerbs } = require("../../constants")
-const { Thread, ThreadType } = require("../../threads")
+} = require("../../../db/utils")
+const { BaseQueryVerbs } = require("../../../constants")
+const { Thread, ThreadType } = require("../../../threads")
+const { save: saveDatasource } = require("../datasource")
+const { RestImporter } = require("./import")
 
 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
 
@@ -30,9 +32,49 @@ exports.fetch = async function (ctx) {
       include_docs: true,
     })
   )
+
   ctx.body = enrichQueries(body.rows.map(row => row.doc))
 }
 
+exports.import = async ctx => {
+  const body = ctx.request.body
+  const data = body.data
+
+  const importer = new RestImporter(data)
+  await importer.init()
+
+  let datasourceId
+  if (!body.datasourceId) {
+    // construct new datasource
+    const info = await importer.getInfo()
+    let datasource = {
+      type: "datasource",
+      source: "REST",
+      config: {
+        url: info.url,
+        defaultHeaders: [],
+      },
+      name: info.name,
+    }
+    // save the datasource
+    const datasourceCtx = { ...ctx }
+    datasourceCtx.request.body.datasource = datasource
+    await saveDatasource(datasourceCtx)
+    datasourceId = datasourceCtx.body.datasource._id
+  } else {
+    // use existing datasource
+    datasourceId = body.datasourceId
+  }
+
+  const importResult = await importer.importQueries(ctx.appId, datasourceId)
+
+  ctx.body = {
+    ...importResult,
+    datasourceId,
+  }
+  ctx.status = 200
+}
+
 exports.save = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = ctx.request.body

package/src/api/controllers/query/validation.js

@@ -0,0 +1,40 @@
+const joiValidator = require("../../../middleware/joi-validator")
+const Joi = require("joi")
+
+exports.queryValidation = () => {
+  return Joi.object({
+    _id: Joi.string(),
+    _rev: Joi.string(),
+    name: Joi.string().required(),
+    fields: Joi.object().required(),
+    datasourceId: Joi.string().required(),
+    readable: Joi.boolean(),
+    parameters: Joi.array().items(
+      Joi.object({
+        name: Joi.string(),
+        default: Joi.string().allow(""),
+      })
+    ),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
+  })
+}
+
+exports.generateQueryValidation = () => {
+  // prettier-ignore
+  return joiValidator.body(exports.queryValidation())
+}
+
+exports.generateQueryPreviewValidation = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    fields: Joi.object().required(),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
+    parameters: Joi.object({}).required().unknown(true)
+  }))
+}

package/src/api/controllers/table/utils.js

@@ -75,6 +75,7 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
   if (!dataImport || !dataImport.csvString) {
     return table
   }
+
   const db = new CouchDB(appId)
   // Populate the table with rows imported from CSV in a bulk update
   const data = await csvParser.transform({

package/src/api/controllers/view/index.js

@@ -5,6 +5,7 @@ const exporters = require("./exporters")
 const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
 const { getTable } = require("../table/utils")
+const { FieldTypes } = require("../../../constants")
 
 exports.fetch = async ctx => {
   const db = new CouchDB(ctx.appId)
@@ -77,6 +78,7 @@ exports.exportView = async ctx => {
   }
 
   await fetchView(ctx)
+  let rows = ctx.body
 
   let schema = view && view.meta && view.meta.schema
   if (!schema) {
@@ -85,11 +87,23 @@ exports.exportView = async ctx => {
     schema = table.schema
   }
 
+  // remove any relationships
+  const relationships = Object.entries(schema)
+    .filter(entry => entry[1].type === FieldTypes.LINK)
+    .map(entry => entry[0])
+  // iterate relationship columns and remove from and row and schema
+  relationships.forEach(column => {
+    rows.forEach(row => {
+      delete row[column]
+    })
+    delete schema[column]
+  })
+
   // make sure no "undefined" entries appear in the CSV
   if (format === exporters.ExportFormats.CSV) {
     const schemaKeys = Object.keys(schema)
     for (let key of schemaKeys) {
-      for (let row of ctx.body) {
+      for (let row of rows) {
         if (row[key] == null) {
           row[key] = ""
         }
@@ -103,5 +117,5 @@ exports.exportView = async ctx => {
   const filename = `${viewName}.${format}`
   // send down the file
   ctx.attachment(filename)
-  ctx.body = apiFileReturn(exporter(headers, ctx.body))
+  ctx.body = apiFileReturn(exporter(headers, rows))
 }

package/src/api/routes/query.js

@@ -1,53 +1,23 @@
 const Router = require("@koa/router")
 const queryController = require("../controllers/query")
 const authorized = require("../../middleware/authorized")
-const Joi = require("joi")
 const {
   PermissionLevels,
   PermissionTypes,
   BUILDER,
 } = require("@budibase/auth/permissions")
-const joiValidator = require("../../middleware/joi-validator")
 const {
   bodyResource,
   bodySubResource,
   paramResource,
 } = require("../../middleware/resourceId")
+const {
+  generateQueryPreviewValidation,
+  generateQueryValidation,
+} = require("../controllers/query/validation")
 
 const router = Router()
 
-function generateQueryValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _id: Joi.string(),
-    _rev: Joi.string(),
-    name: Joi.string().required(),
-    fields: Joi.object().required(),
-    datasourceId: Joi.string().required(),
-    readable: Joi.boolean(),
-    parameters: Joi.array().items(Joi.object({
-      name: Joi.string(),
-      default: Joi.string().allow(""),
-    })),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true),
-    transformer: Joi.string().optional(),
-  }))
-}
-
-function generateQueryPreviewValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    fields: Joi.object().required(),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    datasourceId: Joi.string().required(),
-    transformer: Joi.string().optional(),
-    parameters: Joi.object({}).required().unknown(true)
-  }))
-}
-
 router
   .get("/api/queries", authorized(BUILDER), queryController.fetch)
   .post(
@@ -57,6 +27,7 @@ router
     generateQueryValidation(),
     queryController.save
   )
+  .post("/api/queries/import", authorized(BUILDER), queryController.import)
   .post(
     "/api/queries/preview",
     bodyResource("datasourceId"),

package/src/definitions/common.ts

@@ -107,3 +107,25 @@ export interface Datasource extends Base {
     [key: string]: Table
   }
 }
+
+export interface QueryParameter {
+  name: string
+  default: string
+}
+
+export interface Query {
+  _id?: string
+  datasourceId: string
+  name: string
+  parameters: QueryParameter[]
+  fields: {
+    headers: object
+    queryString: string | null
+    path: string
+    requestBody: string | undefined
+  }
+  transformer: string | null
+  schema: any
+  readable: boolean
+  queryVerb: string
+}

package/src/integrations/oracle.ts

@@ -450,7 +450,7 @@ module OracleModule {
             })
             return lastRow.rows
           } else {
-            return [{ [ operation.toLowerCase() ]: true }]
+            return [{ [operation.toLowerCase()]: true }]
           }
         }
       }

package/src/utilities/csvParser.js

@@ -102,8 +102,11 @@ async function transform({ schema, csvString, existingTable }) {
     schema = updateSchema({ schema, existingTable })
   }
 
-  for (let key of Object.keys(schema)) {
-    colParser[key] = PARSERS[schema[key].type] || schema[key].type
+  for (let [key, field] of Object.entries(schema)) {
+    // don't import data to auto columns
+    if (!field.autocolumn) {
+      colParser[key] = PARSERS[field.type] || field.type
+    }
   }
 
   try {