@budibase/server 1.0.35 → 1.0.39

package/src/api/controllers/row/utils.js

@@ -50,10 +50,10 @@ exports.validate = async ({ appId, tableId, row, table }) => {
   const errors = {}
   for (let fieldName of Object.keys(table.schema)) {
     const constraints = cloneDeep(table.schema[fieldName].constraints)
+    const type = table.schema[fieldName].type
     // special case for options, need to always allow unselected (null)
     if (
-      table.schema[fieldName].type ===
-        (FieldTypes.OPTIONS || FieldTypes.ARRAY) &&
+      (type === FieldTypes.OPTIONS || type === FieldTypes.ARRAY) &&
       constraints.inclusion
     ) {
       constraints.inclusion.push(null)
@@ -61,17 +61,20 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (
-      table.schema[fieldName].type === FieldTypes.ARRAY &&
-      row[fieldName] &&
-      row[fieldName].length
-    ) {
+    if (type === FieldTypes.ARRAY && row[fieldName] && row[fieldName].length) {
       row[fieldName].map(val => {
         if (!constraints.inclusion.includes(val)) {
           errors[fieldName] = "Field not in list"
         }
       })
-    } else if (table.schema[fieldName].type === FieldTypes.FORMULA) {
+    } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
+      // this should only happen if there is an error
+      try {
+        JSON.parse(row[fieldName])
+      } catch (err) {
+        errors[fieldName] = [`Contains invalid JSON`]
+      }
+    } else if (type === FieldTypes.FORMULA) {
       res = validateJs.single(
         processStringSync(table.schema[fieldName].formula, row),
         constraints

package/src/api/controllers/screen.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../db")
 const { getScreenParams, generateScreenID } = require("../../db/utils")
-const { AccessController } = require("@budibase/auth/roles")
+const { AccessController } = require("@budibase/backend-core/roles")
 
 exports.fetch = async ctx => {
   const appId = ctx.appId

package/src/api/controllers/user.js

@@ -2,15 +2,19 @@ const CouchDB = require("../../db")
 const {
   generateUserMetadataID,
   getUserMetadataParams,
+  generateUserFlagID,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
 const { getGlobalUsers, getRawGlobalUser } = require("../../utilities/global")
 const { getFullUser } = require("../../utilities/users")
 const { isEqual } = require("lodash")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { getDevelopmentAppID, getDeployedAppIDs } = require("@budibase/auth/db")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const {
+  getDevelopmentAppID,
+  getDeployedAppIDs,
+} = require("@budibase/backend-core/db")
 const { doesDatabaseExist } = require("../../utilities")
-const { UserStatus } = require("@budibase/auth/constants")
+const { UserStatus } = require("@budibase/backend-core/constants")
 
 async function rawMetadata(db) {
   return (
@@ -195,3 +199,35 @@ exports.destroyMetadata = async function (ctx) {
 exports.findMetadata = async function (ctx) {
   ctx.body = await getFullUser(ctx, ctx.params.id)
 }
+
+exports.setFlag = async function (ctx) {
+  const userId = ctx.user._id
+  const { flag, value } = ctx.request.body
+  if (!flag) {
+    ctx.throw(400, "Must supply a 'flag' field in request body.")
+  }
+  const flagDocId = generateUserFlagID(userId)
+  const db = new CouchDB(ctx.appId)
+  let doc
+  try {
+    doc = await db.get(flagDocId)
+  } catch (err) {
+    doc = { _id: flagDocId }
+  }
+  doc[flag] = value || true
+  await db.put(doc)
+  ctx.body = { message: "Flag set successfully" }
+}
+
+exports.getFlags = async function (ctx) {
+  const userId = ctx.user._id
+  const docId = generateUserFlagID(userId)
+  const db = new CouchDB(ctx.appId)
+  let doc
+  try {
+    doc = await db.get(docId)
+  } catch (err) {
+    doc = { _id: docId }
+  }
+  ctx.body = doc
+}

package/src/api/controllers/webhook.js

@@ -3,7 +3,7 @@ const { generateWebhookID, getWebhookParams } = require("../../db/utils")
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
 const triggers = require("../../automations/triggers")
-const { getDeployedAppID } = require("@budibase/auth/db")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 
@@ -65,6 +65,10 @@ exports.buildSchema = async ctx => {
     let automation = await db.get(webhook.action.target)
     const autoOutputs = automation.definition.trigger.schema.outputs
     let properties = webhook.bodySchema.properties
+    // reset webhook outputs
+    autoOutputs.properties = {
+      body: autoOutputs.properties.body,
+    }
     for (let prop of Object.keys(properties)) {
       autoOutputs.properties[prop] = {
         type: properties[prop].type,

package/src/api/index.js

@@ -4,7 +4,7 @@ const {
   auditLog,
   buildTenancyMiddleware,
   buildAppTenancyMiddleware,
-} = require("@budibase/auth").auth
+} = require("@budibase/backend-core/auth")
 const currentApp = require("../middleware/currentapp")
 const compress = require("koa-compress")
 const zlib = require("zlib")

package/src/api/routes/apikeys.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/apikeys")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/application.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/application")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 const usage = require("../../middleware/usageQuota")
 
 const router = Router()

package/src/api/routes/automation.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const Joi = require("joi")
 const { bodyResource, paramResource } = require("../../middleware/resourceId")
 const {

package/src/api/routes/backup.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/backup")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/cloud.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/cloud")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/component.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/component")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/datasource.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const Joi = require("joi")
 const { DataSourceOperation } = require("../../constants")
 

package/src/api/routes/deploy.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/deploy")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/dev.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/dev")
 const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/hosting.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/hosting")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/integration.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/integration")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/layout.js

@@ -1,6 +1,6 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 const controller = require("../controllers/layout")
 
 const router = Router()

package/src/api/routes/metadata.js

@@ -5,7 +5,7 @@ const {
   AppType,
 } = require("../../middleware/appInfo")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/permission.js

@@ -1,7 +1,10 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/permission")
 const authorized = require("../../middleware/authorized")
-const { BUILDER, PermissionLevels } = require("@budibase/auth/permissions")
+const {
+  BUILDER,
+  PermissionLevels,
+} = require("@budibase/backend-core/permissions")
 const Joi = require("joi")
 const joiValidator = require("../../middleware/joi-validator")
 

package/src/api/routes/query.js

@@ -1,53 +1,23 @@
 const Router = require("@koa/router")
 const queryController = require("../controllers/query")
 const authorized = require("../../middleware/authorized")
-const Joi = require("joi")
 const {
   PermissionLevels,
   PermissionTypes,
   BUILDER,
-} = require("@budibase/auth/permissions")
-const joiValidator = require("../../middleware/joi-validator")
+} = require("@budibase/backend-core/permissions")
 const {
   bodyResource,
   bodySubResource,
   paramResource,
 } = require("../../middleware/resourceId")
+const {
+  generateQueryPreviewValidation,
+  generateQueryValidation,
+} = require("../controllers/query/validation")
 
 const router = Router()
 
-function generateQueryValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    _id: Joi.string(),
-    _rev: Joi.string(),
-    name: Joi.string().required(),
-    fields: Joi.object().required(),
-    datasourceId: Joi.string().required(),
-    readable: Joi.boolean(),
-    parameters: Joi.array().items(Joi.object({
-      name: Joi.string(),
-      default: Joi.string().allow(""),
-    })),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true),
-    transformer: Joi.string().optional(),
-  }))
-}
-
-function generateQueryPreviewValidation() {
-  // prettier-ignore
-  return joiValidator.body(Joi.object({
-    fields: Joi.object().required(),
-    queryVerb: Joi.string().allow().required(),
-    extra: Joi.object().optional(),
-    datasourceId: Joi.string().required(),
-    transformer: Joi.string().optional(),
-    parameters: Joi.object({}).required().unknown(true)
-  }))
-}
-
 router
   .get("/api/queries", authorized(BUILDER), queryController.fetch)
   .post(
@@ -57,6 +27,7 @@ router
     generateQueryValidation(),
     queryController.save
   )
+  .post("/api/queries/import", authorized(BUILDER), queryController.import)
   .post(
     "/api/queries/preview",
     bodyResource("datasourceId"),
@@ -70,11 +41,18 @@ router
     authorized(PermissionTypes.QUERY, PermissionLevels.READ),
     queryController.find
   )
+  // DEPRECATED - use new query endpoint for future work
   .post(
     "/api/queries/:queryId",
     paramResource("queryId"),
     authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
-    queryController.execute
+    queryController.executeV1
+  )
+  .post(
+    "/api/v2/queries/:queryId",
+    paramResource("queryId"),
+    authorized(PermissionTypes.QUERY, PermissionLevels.WRITE),
+    queryController.executeV2
   )
   .delete(
     "/api/queries/:queryId/:revId",

package/src/api/routes/role.js

@@ -7,7 +7,7 @@ const {
   BUILTIN_PERMISSION_IDS,
   BUILDER,
   PermissionLevels,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/routing.js

@@ -1,6 +1,6 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 const controller = require("../controllers/routing")
 
 const router = Router()

package/src/api/routes/row.js

@@ -9,7 +9,7 @@ const {
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/screen.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/screen")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

package/src/api/routes/script.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/script")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/static.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const env = require("../../environment")
 const { paramResource } = require("../../middleware/resourceId")
 

package/src/api/routes/table.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

package/src/api/routes/templates.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/templates")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/tests/datasource.spec.js

@@ -4,6 +4,7 @@ let setup = require("./utilities")
 let { basicDatasource } = setup.structures
 let { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const pg = require("pg")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 
 describe("/datasources", () => {
   let request = setup.getRequest()
@@ -31,6 +32,50 @@ describe("/datasources", () => {
     })
   })
 
+  describe("update", () => {
+    it("should update an existing datasource", async () => {
+      datasource.name = "Updated Test"
+      const res = await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect('Content-Type', /json/)
+        .expect(200)
+
+      expect(res.body.datasource.name).toEqual("Updated Test")
+      expect(res.body.errors).toBeUndefined()
+    })
+
+    describe("dynamic variables", () => {
+      async function preview(datasource, fields) {
+        return config.previewQuery(request, config, datasource, fields)
+      }
+
+      it("should invalidate changed or removed variables", async () => {
+        const { datasource, query } = await config.dynamicVariableDatasource()
+        // preview once to cache variables
+        await preview(datasource, { path: "www.test.com", queryString: "test={{ variable3 }}" })
+        // check variables in cache
+        let contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents.rows.length).toEqual(1)
+        
+        // update the datasource to remove the variables
+        datasource.config.dynamicVariables = []
+        const res = await request
+          .put(`/api/datasources/${datasource._id}`)
+          .send(datasource)
+          .set(config.defaultHeaders())
+          .expect('Content-Type', /json/)
+          .expect(200)
+        expect(res.body.errors).toBeUndefined()
+
+        // check variables no longer in cache
+        contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents).toBe(null)
+      })
+    })
+  })
+
   describe("fetch", () => {
     it("returns all the datasources from the server", async () => {
       const res = await request

package/src/api/routes/tests/permissions.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 

package/src/api/routes/tests/query.spec.js

@@ -1,15 +1,17 @@
 // Mock out postgres for this
 jest.mock("pg")
+jest.mock("node-fetch")
 
 // Mock isProdAppID to we can later mock the implementation and pretend we are
 // using prod app IDs
-const authDb = require("@budibase/auth/db")
+const authDb = require("@budibase/backend-core/db")
 const { isProdAppID } = authDb
 const mockIsProdAppID = jest.fn(isProdAppID)
 authDb.isProdAppID = mockIsProdAppID
 
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 const { basicQuery, basicDatasource } = setup.structures
 
 describe("/queries", () => {
@@ -226,4 +228,71 @@ describe("/queries", () => {
         .expect(400)
     })
   })
+
+  describe("variables", () => {
+
+    async function preview(datasource, fields) {
+      return config.previewQuery(request, config, datasource, fields)
+    }
+
+    it("should work with static variables", async () => {
+      const datasource = await config.restDatasource({
+        staticVariables: {
+          variable: "google",
+          variable2: "1",
+        },
+      })
+      const res = await preview(datasource, {
+        path: "www.{{ variable }}.com",
+        queryString: "test={{ variable2 }}",
+      })
+      // these responses come from the mock
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
+    })
+
+    it("should work with dynamic variables", async () => {
+      const { datasource } = await config.dynamicVariableDatasource()
+      const res = await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toContain("doctype html")
+    })
+
+    it("check that it automatically retries on fail with cached dynamics", async () => {
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      const contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+      const res = await preview(datasource, {
+        path: "www.failonce.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
+      expect(res.body.rows[0].fails).toEqual(1)
+    })
+
+    it("deletes variables when linked query is deleted", async () => {
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      let contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+
+      // delete the query
+      await request
+        .delete(`/api/queries/${base._id}/${base._rev}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+
+      // check variables no longer in cache
+      contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents).toBe(null)
+    })
+  })
 })

package/src/api/routes/tests/role.spec.js

@@ -1,7 +1,7 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const setup = require("./utilities")
 const { basicRole } = setup.structures
 

package/src/api/routes/tests/routing.spec.js

@@ -1,7 +1,7 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 
 const route = "/test"
 

package/src/api/routes/tests/user.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 

package/src/api/routes/tests/utilities/TestFunctions.js

@@ -2,7 +2,7 @@ const rowController = require("../../../controllers/row")
 const appController = require("../../../controllers/application")
 const CouchDB = require("../../../../db")
 const { AppStatus } = require("../../../../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { TENANT_ID } = require("../../../../tests/utilities/structures")
 
 function Request(appId, params) {