@budibase/server 1.0.35 → 1.0.39

package/src/api/controllers/query/import/sources/tests/openapi2/openapi2.spec.js

@@ -0,0 +1,238 @@
+const { OpenAPI2 } = require("../../openapi2")
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file, extension) => {
+  return fs.readFileSync(path.join(__dirname, `./data/${file}/${file}.${extension}`), "utf8")
+}
+
+describe("OpenAPI2 Import", () => {
+  let openapi2 
+
+  beforeEach(() => {
+    openapi2 = new OpenAPI2()
+  })
+
+  it("validates unsupported data", async () => {
+    let data
+    let supported
+
+    // non json / yaml
+    data = "curl http://example.com"
+    supported = await openapi2.isSupported(data)
+    expect(supported).toBe(false)
+
+    // Empty
+    data = ""
+    supported = await openapi2.isSupported(data)
+    expect(supported).toBe(false)
+  })
+
+  const init = async (file, extension) => {
+    await openapi2.isSupported(getData(file, extension))
+  }
+
+  const runTests = async (filename, test, assertions) => {
+    for (let extension of ["json", "yaml"]) {
+      await test(filename, extension, assertions)
+    }
+  }
+
+  const testImportInfo = async (file, extension) => {
+    await init(file, extension)
+    const info = await openapi2.getInfo()
+    expect(info.name).toBe("Swagger Petstore")
+  }
+
+  it("returns import info", async () => {
+    await runTests("petstore", testImportInfo)
+  })
+  
+  describe("Returns queries", () => {
+    const indexQueries = (queries) => {
+      return queries.reduce((acc, query) => {
+        acc[query.name] = query
+        return acc
+      }, {})
+    }
+
+    const getQueries = async (file, extension) => {
+      await init(file, extension)
+      const queries = await openapi2.getQueries()
+      expect(queries.length).toBe(6)
+      return indexQueries(queries)
+    }
+
+    const testVerb = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, method] of Object.entries(assertions)) {
+        expect(queries[operationId].queryVerb).toBe(method)
+      }
+    }
+
+    it("populates verb", async () => {
+      const assertions = {
+        "createEntity" : "create",
+        "getEntities" : "read",
+        "getEntity" : "read",
+        "updateEntity" : "update",
+        "patchEntity" : "patch",
+        "deleteEntity" : "delete"
+      }
+      await runTests("crud", testVerb, assertions)
+    }) 
+
+    const testPath = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, urlPath] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.path).toBe(urlPath)
+      }
+    }
+
+    it("populates path", async () => {
+      const assertions = {
+        "createEntity" : "http://example.com/entities",
+        "getEntities" : "http://example.com/entities",
+        "getEntity" : "http://example.com/entities/{{entityId}}",
+        "updateEntity" : "http://example.com/entities/{{entityId}}",
+        "patchEntity" : "http://example.com/entities/{{entityId}}",
+        "deleteEntity" : "http://example.com/entities/{{entityId}}"
+      }
+      await runTests("crud", testPath, assertions)
+    })
+
+    const testHeaders = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, headers] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.headers).toStrictEqual(headers)
+      }
+    }
+
+    const contentTypeHeader = {
+      "Content-Type" : "application/json",
+    }
+
+    it("populates headers", async () => {
+      const assertions = {
+        "createEntity" : {
+         ...contentTypeHeader
+        },
+        "getEntities" : {
+        },
+        "getEntity" : {
+        },
+        "updateEntity" : {
+          ...contentTypeHeader
+        },
+        "patchEntity" : {
+          ...contentTypeHeader
+        },
+        "deleteEntity" : {
+            "x-api-key" : "{{x-api-key}}",
+        }
+      }
+
+      await runTests("crud", testHeaders, assertions)
+    })
+
+    const testQuery = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, queryString] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.queryString).toStrictEqual(queryString)
+      }
+    }
+
+    it("populates query", async () => {
+      const assertions = {
+        "createEntity" : "",
+        "getEntities" : "page={{page}}&size={{size}}",
+        "getEntity" : "",
+        "updateEntity" : "",
+        "patchEntity" : "",
+        "deleteEntity" : ""
+      }
+      await runTests("crud", testQuery, assertions)
+    })
+
+    const testParameters = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, parameters] of Object.entries(assertions)) {
+        expect(queries[operationId].parameters).toStrictEqual(parameters)
+      }
+    }
+
+    it("populates parameters", async () => {
+      const assertions = {
+        "createEntity" : [],
+        "getEntities" : [
+          {
+            "name" : "page",
+            "default" : "",
+          },
+          {
+            "name" : "size",
+            "default" : "",
+          }
+        ],
+        "getEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "updateEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "patchEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          }
+        ],
+        "deleteEntity" : [
+          {
+            "name" : "entityId",
+            "default" : "",
+          },
+          {
+            "name" : "x-api-key",
+            "default" : "",
+          }
+        ]
+      }
+      await runTests("crud", testParameters, assertions)
+    })
+
+    const testBody = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, body] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.requestBody).toStrictEqual(JSON.stringify(body, null, 2))
+      }
+    }
+    it("populates body", async () => {
+      const assertions = {
+        "createEntity" : {
+          "name" : "name",
+          "type" : "type",
+        },
+        "getEntities" : undefined,
+        "getEntity" : undefined,
+        "updateEntity" : {
+          "id": 1,
+          "name" : "name",
+          "type" : "type",
+        },
+        "patchEntity" : {
+          "id": 1,
+          "name" : "name",
+          "type" : "type",
+        },
+        "deleteEntity" : undefined
+      }
+      await runTests("crud", testBody, assertions)
+    })
+  })
+})

package/src/api/controllers/query/import/tests/index.spec.js

@@ -0,0 +1,109 @@
+
+const bulkDocs = jest.fn()
+const db = jest.fn(() => {
+  return {
+    bulkDocs
+  }
+})
+jest.mock("../../../../../db", () => db)
+
+const { RestImporter } = require("../index")
+
+const fs = require("fs")
+const path = require('path')
+
+const getData = (file) => {
+  return fs.readFileSync(path.join(__dirname, `../sources/tests/${file}`), "utf8")
+}
+
+// openapi2 (swagger)
+const oapi2CrudJson = getData("openapi2/data/crud/crud.json")
+const oapi2CrudYaml = getData("openapi2/data/crud/crud.json")
+const oapi2PetstoreJson = getData("openapi2/data/petstore/petstore.json")
+const oapi2PetstoreYaml = getData("openapi2/data/petstore/petstore.json")
+
+// curl
+const curl = getData("curl/data/post.txt")
+
+const datasets = {
+  oapi2CrudJson,
+  oapi2CrudYaml,
+  oapi2PetstoreJson,
+  oapi2PetstoreYaml,
+  curl
+}
+
+describe("Rest Importer", () => {
+  let restImporter 
+
+  const init = async (data) => {
+    restImporter = new RestImporter(data)
+    await restImporter.init()
+  }
+
+  const runTest = async (test, assertions) => {
+    for (let [key, data] of Object.entries(datasets)) {
+      await test(key, data, assertions)
+    }
+  }
+
+  const testGetInfo = async (key, data, assertions) => {
+    await init(data)
+    const info = await restImporter.getInfo()
+    expect(info.name).toBe(assertions[key].name)
+  }
+
+  it("gets info", async () => {
+    const assertions = {
+      "oapi2CrudJson" : {
+        name: "CRUD",
+      },
+      "oapi2CrudYaml" : {
+        name: "CRUD",
+      },
+      "oapi2PetstoreJson" : {
+        name: "Swagger Petstore",
+      },
+      "oapi2PetstoreYaml" :{
+        name: "Swagger Petstore",
+      },
+      "curl": {
+        name: "example.com",
+      }
+    }
+    await runTest(testGetInfo, assertions)
+  })
+
+  const testImportQueries = async (key, data, assertions) => {
+    await init(data)
+    bulkDocs.mockReturnValue([])
+    const importResult = await restImporter.importQueries("appId", "datasourceId")
+    expect(importResult.errorQueries.length).toBe(0)
+    expect(importResult.queries.length).toBe(assertions[key].count)
+    expect(bulkDocs).toHaveBeenCalledTimes(1)
+    jest.clearAllMocks()
+  }
+
+  it("imports queries", async () => {
+    // simple sanity assertions that the whole dataset 
+    // makes it through the importer
+    const assertions = {
+      "oapi2CrudJson" : {
+        count: 6,
+      },
+      "oapi2CrudYaml" :{
+        count: 6,
+      },
+      "oapi2PetstoreJson" : {
+        count: 20,
+      },
+      "oapi2PetstoreYaml" :{
+        count: 20,
+      },
+      "curl": {
+        count: 1
+      }
+    }
+    await runTest(testImportQueries, assertions)
+  })
+})

package/src/api/controllers/query/index.js

@@ -0,0 +1,196 @@
+const CouchDB = require("../../../db")
+const {
+  generateQueryID,
+  getQueryParams,
+  isProdAppID,
+} = require("../../../db/utils")
+const { BaseQueryVerbs } = require("../../../constants")
+const { Thread, ThreadType } = require("../../../threads")
+const { save: saveDatasource } = require("../datasource")
+const { RestImporter } = require("./import")
+const { invalidateDynamicVariables } = require("../../../threads/utils")
+
+const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
+
+// simple function to append "readable" to all read queries
+function enrichQueries(input) {
+  const wasArray = Array.isArray(input)
+  const queries = wasArray ? input : [input]
+  for (let query of queries) {
+    if (query.queryVerb === BaseQueryVerbs.READ) {
+      query.readable = true
+    }
+  }
+  return wasArray ? queries : queries[0]
+}
+
+exports.fetch = async function (ctx) {
+  const db = new CouchDB(ctx.appId)
+
+  const body = await db.allDocs(
+    getQueryParams(null, {
+      include_docs: true,
+    })
+  )
+
+  ctx.body = enrichQueries(body.rows.map(row => row.doc))
+}
+
+exports.import = async ctx => {
+  const body = ctx.request.body
+  const data = body.data
+
+  const importer = new RestImporter(data)
+  await importer.init()
+
+  let datasourceId
+  if (!body.datasourceId) {
+    // construct new datasource
+    const info = await importer.getInfo()
+    let datasource = {
+      type: "datasource",
+      source: "REST",
+      config: {
+        url: info.url,
+        defaultHeaders: [],
+      },
+      name: info.name,
+    }
+    // save the datasource
+    const datasourceCtx = { ...ctx }
+    datasourceCtx.request.body.datasource = datasource
+    await saveDatasource(datasourceCtx)
+    datasourceId = datasourceCtx.body.datasource._id
+  } else {
+    // use existing datasource
+    datasourceId = body.datasourceId
+  }
+
+  const importResult = await importer.importQueries(ctx.appId, datasourceId)
+
+  ctx.body = {
+    ...importResult,
+    datasourceId,
+  }
+  ctx.status = 200
+}
+
+exports.save = async function (ctx) {
+  const db = new CouchDB(ctx.appId)
+  const query = ctx.request.body
+
+  if (!query._id) {
+    query._id = generateQueryID(query.datasourceId)
+  }
+
+  const response = await db.put(query)
+  query._rev = response.rev
+
+  ctx.body = query
+  ctx.message = `Query ${query.name} saved successfully.`
+}
+
+exports.find = async function (ctx) {
+  const db = new CouchDB(ctx.appId)
+  const query = enrichQueries(await db.get(ctx.params.queryId))
+  // remove properties that could be dangerous in real app
+  if (isProdAppID(ctx.appId)) {
+    delete query.fields
+    delete query.parameters
+  }
+  ctx.body = query
+}
+
+exports.preview = async function (ctx) {
+  const db = new CouchDB(ctx.appId)
+
+  const datasource = await db.get(ctx.request.body.datasourceId)
+  // preview may not have a queryId as it hasn't been saved, but if it does
+  // this stops dynamic variables from calling the same query
+  const { fields, parameters, queryVerb, transformer, queryId } =
+    ctx.request.body
+
+  try {
+    const { rows, keys, info, extra } = await Runner.run({
+      appId: ctx.appId,
+      datasource,
+      queryVerb,
+      fields,
+      parameters,
+      transformer,
+      queryId,
+    })
+
+    ctx.body = {
+      rows,
+      schemaFields: [...new Set(keys)],
+      info,
+      extra,
+    }
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+async function execute(ctx, opts = { rowsOnly: false }) {
+  const db = new CouchDB(ctx.appId)
+
+  const query = await db.get(ctx.params.queryId)
+  const datasource = await db.get(query.datasourceId)
+
+  // call the relevant CRUD method on the integration class
+  try {
+    const { rows, pagination, extra } = await Runner.run({
+      appId: ctx.appId,
+      datasource,
+      queryVerb: query.queryVerb,
+      fields: query.fields,
+      pagination: ctx.request.body.pagination,
+      parameters: ctx.request.body.parameters,
+      transformer: query.transformer,
+      queryId: ctx.params.queryId,
+    })
+    if (opts && opts.rowsOnly) {
+      ctx.body = rows
+    } else {
+      ctx.body = { data: rows, pagination, ...extra }
+    }
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+exports.executeV1 = async function (ctx) {
+  return execute(ctx, { rowsOnly: true })
+}
+
+exports.executeV2 = async function (ctx) {
+  return execute(ctx, { rowsOnly: false })
+}
+
+const removeDynamicVariables = async (db, queryId) => {
+  const query = await db.get(queryId)
+  const datasource = await db.get(query.datasourceId)
+  const dynamicVariables = datasource.config.dynamicVariables
+
+  if (dynamicVariables) {
+    // delete dynamic variables from the datasource
+    const newVariables = dynamicVariables.filter(dv => dv.queryId !== queryId)
+    datasource.config.dynamicVariables = newVariables
+    await db.put(datasource)
+
+    // invalidate the deleted variables
+    const variablesToDelete = dynamicVariables.filter(
+      dv => dv.queryId === queryId
+    )
+    await invalidateDynamicVariables(variablesToDelete)
+  }
+}
+
+exports.destroy = async function (ctx) {
+  const db = new CouchDB(ctx.appId)
+  await removeDynamicVariables(db, ctx.params.queryId)
+  await db.remove(ctx.params.queryId, ctx.params.revId)
+  ctx.message = `Query deleted.`
+  ctx.status = 200
+}

package/src/api/controllers/query/validation.js

@@ -0,0 +1,42 @@
+const joiValidator = require("../../../middleware/joi-validator")
+const Joi = require("joi")
+
+exports.queryValidation = () => {
+  return Joi.object({
+    _id: Joi.string(),
+    _rev: Joi.string(),
+    name: Joi.string().required(),
+    fields: Joi.object().required(),
+    datasourceId: Joi.string().required(),
+    readable: Joi.boolean(),
+    parameters: Joi.array().items(
+      Joi.object({
+        name: Joi.string(),
+        default: Joi.string().allow(""),
+      })
+    ),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
+    flags: Joi.object().optional(),
+  })
+}
+
+exports.generateQueryValidation = () => {
+  // prettier-ignore
+  return joiValidator.body(exports.queryValidation())
+}
+
+exports.generateQueryPreviewValidation = () => {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    fields: Joi.object().required(),
+    queryVerb: Joi.string().allow().required(),
+    extra: Joi.object().optional(),
+    datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
+    parameters: Joi.object({}).required().unknown(true),
+    queryId: Joi.string().optional(),
+  }))
+}

package/src/api/controllers/role.js

@@ -4,7 +4,7 @@ const {
   getRole,
   isBuiltin,
   getAllRoles,
-} = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/roles")
 const {
   generateRoleID,
   getUserMetadataParams,

package/src/api/controllers/routing.js

@@ -2,7 +2,7 @@ const { getRoutingInfo } = require("../../utilities/routing")
 const {
   getUserRoleHierarchy,
   BUILTIN_ROLE_IDS,
-} = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/roles")
 
 const URL_SEPARATOR = "/"
 

package/src/api/controllers/row/internal.js

@@ -11,6 +11,7 @@ const {
   inputProcessing,
   outputProcessing,
   processAutoColumn,
+  cleanupAttachments,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
 const { isEqual } = require("lodash")
@@ -25,6 +26,7 @@ const {
   getFromDesignDoc,
   getFromMemoryDoc,
 } = require("../view/utils")
+const { cloneDeep } = require("lodash/fp")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -109,14 +111,14 @@ exports.patch = async ctx => {
   const inputs = ctx.request.body
   const tableId = inputs.tableId
   const isUserTable = tableId === InternalTables.USER_METADATA
-  let dbRow
+  let oldRow
   try {
-    dbRow = await db.get(inputs._id)
+    oldRow = await db.get(inputs._id)
   } catch (err) {
     if (isUserTable) {
       // don't include the rev, it'll be the global rev
       // this time
-      dbRow = {
+      oldRow = {
         _id: inputs._id,
       }
     } else {
@@ -125,13 +127,14 @@ exports.patch = async ctx => {
   }
   let dbTable = await db.get(tableId)
   // need to build up full patch fields before coerce
+  let combinedRow = cloneDeep(oldRow)
   for (let key of Object.keys(inputs)) {
     if (!dbTable.schema[key]) continue
-    dbRow[key] = inputs[key]
+    combinedRow[key] = inputs[key]
   }
 
   // this returns the table and row incase they have been updated
-  let { table, row } = inputProcessing(ctx.user, dbTable, dbRow)
+  let { table, row } = inputProcessing(ctx.user, dbTable, combinedRow)
   const validateResult = await validate({
     row,
     table,
@@ -149,6 +152,8 @@ exports.patch = async ctx => {
     tableId: row.tableId,
     table,
   })
+  // check if any attachments removed
+  await cleanupAttachments(appId, table, { oldRow, row })
 
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
@@ -295,6 +300,8 @@ exports.destroy = async function (ctx) {
     row,
     tableId: row.tableId,
   })
+  // remove any attachments that were on the row from object storage
+  await cleanupAttachments(appId, table, { row })
 
   let response
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -341,6 +348,8 @@ exports.bulkDestroy = async ctx => {
   } else {
     await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
   }
+  // remove any attachments that were on the rows from object storage
+  await cleanupAttachments(appId, table, { rows })
   await Promise.all(updates)
   return { response: { ok: true }, rows }
 }

package/src/api/controllers/row/internalSearch.js

@@ -1,6 +1,6 @@
 const { SearchIndexes } = require("../../../db/utils")
 const fetch = require("node-fetch")
-const { getCouchUrl } = require("@budibase/auth/db")
+const { getCouchUrl } = require("@budibase/backend-core/db")
 
 /**
  * Class to build lucene query URLs.
@@ -191,7 +191,8 @@ class QueryBuilder {
     }
     if (this.query.equal) {
       build(this.query.equal, (key, value) => {
-        if (!value) {
+        // 0 evaluates to false, which means we would return all rows if we don't check it
+        if (!value && value !== 0) {
           return null
         }
         return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`