@budibase/server 1.0.21 → 1.0.23-alpha.2

package/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js

@@ -0,0 +1,239 @@
+const { OpenAPI2 } = require("../../openapi2")
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file, extension) => {
+  return fs.readFileSync(path.join(__dirname, `./data/${file}/${file}.${extension}`), "utf8")
+}
+
+describe("OpenAPI2 Import", () => {
+  let openapi2 
+
+  beforeEach(() => {
+    openapi2 = new OpenAPI2()
+  })
+
+  it("validates unsupported data", async () => {
+    let data
+    let supported
+
+    // non json / yaml
+    data = "curl http://example.com"
+    supported = await openapi2.isSupported(data)
+    expect(supported).toBe(false)
+
+    // Empty
+    data = ""
+    supported = await openapi2.isSupported(data)
+    expect(supported).toBe(false)
+  })
+
+  const init = async (file, extension) => {
+    await openapi2.isSupported(getData(file, extension))
+  }
+
+  const runTests = async (filename, test, assertions) => {
+    for (let extension of ["json", "yaml"]) {
+      await test(filename, extension, assertions)
+    }
+  }
+
+  const testImportInfo = async (file, extension) => {
+    await init(file, extension)
+    const info = await openapi2.getInfo()
+    expect(info.url).toBe("https://petstore.swagger.io/v2")
+    expect(info.name).toBe("Swagger Petstore")
+  }
+
+  it("returns import info", async () => {
+    await runTests("petstore", testImportInfo)
+  })
+  
+  describe("Returns queries", () => {
+    const indexQueries = (queries) => {
+      return queries.reduce((acc, query) => {
+        acc[query.name] = query
+        return acc
+      }, {})
+    }
+
+    const getQueries = async (file, extension) => {
+      await init(file, extension)
+      const queries = await openapi2.getQueries()
+      expect(queries.length).toBe(6)
+      return indexQueries(queries)
+    }
+
+    const testVerb = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, method] of Object.entries(assertions)) {
+        expect(queries[operationId].queryVerb).toBe(method)
+      }
+    }
+
+    it("populates verb", async () => {
+      const assertions = {
+        "createEntity" : "create",
+        "getEntities" : "read",
+        "getEntity" : "read",
+        "updateEntity" : "update",
+        "patchEntity" : "patch",
+        "deleteEntity" : "delete"
+      }
+      await runTests("crud", testVerb, assertions)
+    }) 
+
+    const testPath = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, urlPath] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.path).toBe(urlPath)
+      }
+    }
+
+    it("populates path", async () => {
+      const assertions = {
+        "createEntity" : "entities",
+        "getEntities" : "entities",
+        "getEntity" : "entities/{{entityId}}",
+        "updateEntity" : "entities/{{entityId}}",
+        "patchEntity" : "entities/{{entityId}}",
+        "deleteEntity" : "entities/{{entityId}}"
+      }
+      await runTests("crud", testPath, assertions)
+    })
+
+    const testHeaders = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, headers] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.headers).toStrictEqual(headers)
+      }
+    }
+
+    const contentTypeHeader = {
+      "Content-Type" : "application/json",
+    }
+
+    it("populates headers", async () => {
+      const assertions = {
+        "createEntity" : {
+         ...contentTypeHeader
+        },
+        "getEntities" : {
+        },
+        "getEntity" : {
+        },
+        "updateEntity" : {
+          ...contentTypeHeader
+        },
+        "patchEntity" : {
+          ...contentTypeHeader
+        },
+        "deleteEntity" : {
+            "x-api-key" : "{{x-api-key}}",
+        }
+      }
+
+      await runTests("crud", testHeaders, assertions)
+    })
+
+    const testQuery = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, queryString] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.queryString).toStrictEqual(queryString)
+      }
+    }
+
+    it("populates query", async () => {
+      const assertions = {
+        "createEntity" : "",
+        "getEntities" : "page={{page}}&size={{size}}",
+        "getEntity" : "",
+        "updateEntity" : "",
+        "patchEntity" : "",
+        "deleteEntity" : ""
+      }
+      await runTests("crud", testQuery, assertions)
+    })
+
+    const testParameters = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, parameters] of Object.entries(assertions)) {
+        expect(queries[operationId].parameters).toStrictEqual(parameters)
+      }
+    }
+
+    it("populates parameters", async () => {
+      const assertions = {
+        "createEntity" : [],
+        "getEntities" : [
+          {
+            "name" : "page",
+            "default" : "",
+          },
+          {
+            "name" : "size",
+            "default" : "",
+          }
+        ],
+        "getEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "updateEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "patchEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "deleteEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          },
+          {
+            "name" : "x-api-key",
+            "default" : "",
+          }
+        ]
+      }
+      await runTests("crud", testParameters, assertions)
+    })
+
+    const testBody = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, body] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
+      }
+    }
+    it("populates body", async () => {
+      const assertions = {
+        "createEntity" : {
+          "name" : "name",
+          "type" : "type",
+        },
+        "getEntities" : undefined,
+        "getEntity" : undefined,
+        "updateEntity" : {
+          "id": 1,
+          "name" : "name",
+          "type" : "type",
+        },
+        "patchEntity" : {
+          "id": 1,
+          "name" : "name",
+          "type" : "type",
+        },
+        "deleteEntity" : undefined
+      }
+      await runTests("crud", testBody, assertions)
+    })
+  })
+})

package/src/api/controllers/query/import/tests/index.spec.js

@@ -0,0 +1,115 @@
+
+const bulkDocs = jest.fn()
+const db = jest.fn(() => {
+  return {
+    bulkDocs
+  }
+})
+jest.mock("../../../../../db", () => db)
+
+const { RestImporter } = require("../index")
+
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file) => {
+  return fs.readFileSync(path.join(__dirname, `../sources/tests/${file}`), "utf8")
+}
+
+// openapi2 (swagger)
+const oapi2CrudJson = getData("openapi2/data/crud/crud.json")
+const oapi2CrudYaml = getData("openapi2/data/crud/crud.json")
+const oapi2PetstoreJson = getData("openapi2/data/petstore/petstore.json")
+const oapi2PetstoreYaml = getData("openapi2/data/petstore/petstore.json")
+
+// curl
+const curl = getData("curl/data/post.txt")
+
+const datasets = {
+  oapi2CrudJson,
+  oapi2CrudYaml,
+  oapi2PetstoreJson,
+  oapi2PetstoreYaml,
+  curl
+}
+
+describe("Rest Importer", () => {
+  let restImporter 
+
+  const init = async (data) => {
+    restImporter = new RestImporter(data)
+    await restImporter.init()
+  }
+
+  const runTest = async (test, assertions) => {
+    for (let [key, data] of Object.entries(datasets)) {
+      await test(key, data, assertions)
+    }
+  }
+
+  const testGetInfo = async (key, data, assertions) => {
+    await init(data)
+    const info = await restImporter.getInfo()
+    expect(info.name).toBe(assertions[key].name)
+    expect(info.url).toBe(assertions[key].url)
+  }
+
+  it("gets info", async () => {
+    const assertions = {
+      "oapi2CrudJson" : {
+        name: "CRUD",
+        url: "http://example.com"
+      },
+      "oapi2CrudYaml" : {
+        name: "CRUD",
+        url: "http://example.com"
+      },
+      "oapi2PetstoreJson" : {
+        name: "Swagger Petstore",
+        url: "https://petstore.swagger.io/v2"
+      },
+      "oapi2PetstoreYaml" :{
+        name: "Swagger Petstore",
+        url: "https://petstore.swagger.io/v2"
+      },
+      "curl": {
+        name: "example.com",
+        url: "http://example.com"
+      }
+    }
+    await runTest(testGetInfo, assertions)
+  })
+
+  const testImportQueries = async (key, data, assertions) => {
+    await init(data)
+    bulkDocs.mockReturnValue([])
+    const importResult = await restImporter.importQueries("appId", "datasourceId")
+    expect(importResult.errorQueries.length).toBe(0)
+    expect(importResult.queries.length).toBe(assertions[key].count)
+    expect(bulkDocs).toHaveBeenCalledTimes(1)
+    jest.clearAllMocks()
+  }
+
+  it("imports queries", async () => {
+    // simple sanity assertions that the whole dataset 
+    // makes it through the importer
+    const assertions = {
+      "oapi2CrudJson" : {
+        count: 6,
+      },
+      "oapi2CrudYaml" :{
+        count: 6,
+      },
+      "oapi2PetstoreJson" : {
+        count: 20,
+      },
+      "oapi2PetstoreYaml" :{
+        count: 20,
+      },
+      "curl": {
+        count: 1
+      }
+    }
+    await runTest(testImportQueries, assertions)
+  })
+})

package/src/api/controllers/{query.js → query/index.js}

@@ -1,12 +1,14 @@
 const { processString } = require("@budibase/string-templates")
-const CouchDB = require("../../db")
+const CouchDB = require("../../../db")
 const {
   generateQueryID,
   getQueryParams,
   isProdAppID,
-} = require("../../db/utils")
-const { BaseQueryVerbs } = require("../../constants")
-const { Thread, ThreadType } = require("../../threads")
+} = require("../../../db/utils")
+const { BaseQueryVerbs } = require("../../../constants")
+const { Thread, ThreadType } = require("../../../threads")
+const { save: saveDatasource } = require("../datasource")
+const { RestImporter } = require("./import")
 
 const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
 
@@ -30,9 +32,49 @@ exports.fetch = async function (ctx) {
       include_docs: true,
     })
   )
+
   ctx.body = enrichQueries(body.rows.map(row => row.doc))
 }
 
+exports.import = async ctx => {
+  const body = ctx.request.body
+  const data = body.data
+
+  const importer = new RestImporter(data)
+  await importer.init()
+
+  let datasourceId
+  if (!body.datasourceId) {
+    // construct new datasource
+    const info = await importer.getInfo()
+    let datasource = {
+      type: "datasource",
+      source: "REST",
+      config: {
+        url: info.url,
+        defaultHeaders: [],
+      },
+      name: info.name,
+    }
+    // save the datasource
+    const datasourceCtx = { ...ctx }
+    datasourceCtx.request.body.datasource = datasource
+    await saveDatasource(datasourceCtx)
+    datasourceId = datasourceCtx.body.datasource._id
+  } else {
+    // use existing datasource
+    datasourceId = body.datasourceId
+  }
+
+  const importResult = await importer.importQueries(ctx.appId, datasourceId)
+
+  ctx.body = {
+    ...importResult,
+    datasourceId,
+  }
+  ctx.status = 200
+}
+
 exports.save = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const query = ctx.request.body
@@ -81,7 +123,7 @@ async function enrichQueryFields(fields, parameters = {}) {
           enrichedQuery.requestBody
       )
     } catch (err) {
-      throw { message: `JSON Invalid - error: ${err}` }
+      // no json found, ignore
     }
     delete enrichedQuery.customData
   }
@@ -109,7 +151,7 @@ exports.preview = async function (ctx) {
   const enrichedQuery = await enrichQueryFields(fields, parameters)
 
   try {
-    const { rows, keys } = await Runner.run({
+    const { rows, keys, info, extra } = await Runner.run({
       datasource,
       queryVerb,
       query: enrichedQuery,
@@ -119,6 +161,8 @@ exports.preview = async function (ctx) {
     ctx.body = {
       rows,
       schemaFields: [...new Set(keys)],
+      info,
+      extra,
     }
   } catch (err) {
     ctx.throw(400, err)

package/src/api/controllers/query/validation.js

@@ -0,0 +1,41 @@
+const joiValidator = require("../../../middleware/joi-validator")
+const Joi = require("joi")
+
+exports.queryValidation = () => {
+  return Joi.object({
+    _id: Joi.string(),
+    _rev: Joi.string(),
+    name: Joi.string().required(),
+    fields: Joi.object().required(),
+    datasourceId: Joi.string().required(),
+    readable: Joi.boolean(),
+    parameters: Joi.array().items(
+      Joi.object({
+        name: Joi.string(),
+        default: Joi.string().allow(""),
+      })
+    ),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
+    flags: Joi.object().optional(),
+  })
+}
+
+exports.generateQueryValidation = () => {
+  // prettier-ignore
+  return joiValidator.body(exports.queryValidation())
+}
+
+exports.generateQueryPreviewValidation = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    fields: Joi.object().required(),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
+    parameters: Joi.object({}).required().unknown(true)
+  }))
+}

package/src/api/controllers/user.js

@@ -2,6 +2,7 @@ const CouchDB = require("../../db")
 const {
   generateUserMetadataID,
   getUserMetadataParams,
+  generateUserFlagID,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
 const { getGlobalUsers, getRawGlobalUser } = require("../../utilities/global")
@@ -195,3 +196,35 @@ exports.destroyMetadata = async function (ctx) {
 exports.findMetadata = async function (ctx) {
   ctx.body = await getFullUser(ctx, ctx.params.id)
 }
+
+exports.setFlag = async function (ctx) {
+  const userId = ctx.user._id
+  const { flag, value } = ctx.request.body
+  if (!flag) {
+    ctx.throw(400, "Must supply a 'flag' field in request body.")
+  }
+  const flagDocId = generateUserFlagID(userId)
+  const db = new CouchDB(ctx.appId)
+  let doc
+  try {
+    doc = await db.get(flagDocId)
+  } catch (err) {
+    doc = { _id: flagDocId }
+  }
+  doc[flag] = value || true
+  await db.put(doc)
+  ctx.body = { message: "Flag set successfully" }
+}
+
+exports.getFlags = async function (ctx) {
+  const userId = ctx.user._id
+  const docId = generateUserFlagID(userId)
+  const db = new CouchDB(ctx.appId)
+  let doc
+  try {
+    doc = await db.get(docId)
+  } catch (err) {
+    doc = { _id: docId }
+  }
+  ctx.body = doc
+}

package/src/api/controllers/webhook.js

@@ -65,6 +65,10 @@ exports.buildSchema = async ctx => {
     let automation = await db.get(webhook.action.target)
     const autoOutputs = automation.definition.trigger.schema.outputs
     let properties = webhook.bodySchema.properties
+    // reset webhook outputs
+    autoOutputs.properties = {
+      body: autoOutputs.properties.body,
+    }
     for (let prop of Object.keys(properties)) {
       autoOutputs.properties[prop] = {
         type: properties[prop].type,

package/src/api/routes/query.js

@@ -1,53 +1,23 @@
 const Router = require("@koa/router")
 const queryController = require("../controllers/query")
 const authorized = require("../../middleware/authorized")
-const Joi = require("joi")
 const {
   PermissionLevels,
   PermissionTypes,
   BUILDER,
 } = require("@budibase/auth/permissions")
-const joiValidator = require("../../middleware/joi-validator")
 const {
   bodyResource,
   bodySubResource,
   paramResource,
 } = require("../../middleware/resourceId")
+const {
+  generateQueryPreviewValidation,
+  generateQueryValidation,
+} = require("../controllers/query/validation")
 
 const router = Router()
 
-function generateQueryValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _id: Joi.string(),
-    _rev: Joi.string(),
-    name: Joi.string().required(),
-    fields: Joi.object().required(),
-    datasourceId: Joi.string().required(),
-    readable: Joi.boolean(),
-    parameters: Joi.array().items(Joi.object({
-      name: Joi.string(),
-      default: Joi.string().allow(""),
-    })),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true),
-    transformer: Joi.string().optional(),
-  }))
-}
-
-function generateQueryPreviewValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    fields: Joi.object().required(),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    datasourceId: Joi.string().required(),
-    transformer: Joi.string().optional(),
-    parameters: Joi.object({}).required().unknown(true)
-  }))
-}
-
 router
   .get("/api/queries", authorized(BUILDER), queryController.fetch)
   .post(
@@ -57,6 +27,7 @@ router
     generateQueryValidation(),
     queryController.save
   )
+  .post("/api/queries/import", authorized(BUILDER), queryController.import)
   .post(
     "/api/queries/preview",
     bodyResource("datasourceId"),

package/src/api/routes/user.js

@@ -39,5 +39,15 @@ router
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
     controller.syncUser
   )
+  .post(
+    "/api/users/flags",
+    authorized(PermissionTypes.USER, PermissionLevels.WRITE),
+    controller.setFlag
+  )
+  .get(
+    "/api/users/flags",
+    authorized(PermissionTypes.USER, PermissionLevels.READ),
+    controller.getFlags
+  )
 
 module.exports = router

package/src/automations/steps/outgoingWebhook.js

@@ -13,12 +13,11 @@ const RequestType = {
 const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH]
 
 /**
- * Note, there is some functionality in this that is not currently exposed as it
- * is complex and maybe better to be opinionated here.
- * GET/DELETE requests cannot handle body elements so they will not be sent if configured.
+ * NOTE: this functionality is deprecated - it no longer should be used.
  */
 
 exports.definition = {
+  deprecated: true,
   name: "Outgoing webhook",
   tagline: "Send a {{inputs.requestMethod}} request",
   icon: "Send",

package/src/automations/utils.js

@@ -7,6 +7,7 @@ const newid = require("../db/newid")
 const { updateEntityMetadata } = require("../utilities")
 const { MetadataTypes } = require("../constants")
 const { getDeployedAppID } = require("@budibase/auth/db")
+const { cloneDeep } = require("lodash/fp")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
@@ -42,6 +43,16 @@ exports.updateTestHistory = async (appId, automation, history) => {
   )
 }
 
+exports.removeDeprecated = definitions => {
+  const base = cloneDeep(definitions)
+  for (let key of Object.keys(base)) {
+    if (base[key].deprecated) {
+      delete base[key]
+    }
+  }
+  return base
+}
+
 // end the repetition and the job itself
 exports.disableAllCrons = async appId => {
   const promises = []

package/src/db/utils.js

@@ -40,6 +40,7 @@ const DocumentTypes = {
   DEPLOYMENTS: "deployments",
   METADATA: "metadata",
   MEM_VIEW: "view",
+  USER_FLAG: "flag",
 }
 
 const ViewNames = {
@@ -339,6 +340,14 @@ exports.getQueryParams = (datasourceId = null, otherProps = {}) => {
   )
 }
 
+/**
+ * Generates a new flag document ID.
+ * @returns {string} The ID of the flag document that was generated.
+ */
+exports.generateUserFlagID = userId => {
+  return `${DocumentTypes.USER_FLAG}${SEPARATOR}${userId}`
+}
+
 exports.generateMetadataID = (type, entityId) => {
   return `${DocumentTypes.METADATA}${SEPARATOR}${type}${SEPARATOR}${entityId}`
 }