@budibase/server 1.0.27-alpha.1 → 1.0.27-alpha.13

package/src/api/routes/tests/datasource.spec.js

@@ -4,6 +4,7 @@ let setup = require("./utilities")
 let { basicDatasource } = setup.structures
 let { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const pg = require("pg")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 
 describe("/datasources", () => {
   let request = setup.getRequest()
@@ -31,6 +32,50 @@ describe("/datasources", () => {
     })
   })
 
+  describe("update", () => {
+    it("should update an existing datasource", async () => {
+      datasource.name = "Updated Test"
+      const res = await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect('Content-Type', /json/)
+        .expect(200)
+
+      expect(res.body.datasource.name).toEqual("Updated Test")
+      expect(res.body.errors).toBeUndefined()
+    })
+
+    describe("dynamic variables", () => {
+      async function preview(datasource, fields) {
+        return config.previewQuery(request, config, datasource, fields)
+      }
+
+      it("should invalidate changed or removed variables", async () => {
+        const { datasource, query } = await config.dynamicVariableDatasource()
+        // preview once to cache variables
+        await preview(datasource, { path: "www.test.com", queryString: "test={{ variable3 }}" })
+        // check variables in cache
+        let contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents.rows.length).toEqual(1)
+        
+        // update the datasource to remove the variables
+        datasource.config.dynamicVariables = []
+        const res = await request
+          .put(`/api/datasources/${datasource._id}`)
+          .send(datasource)
+          .set(config.defaultHeaders())
+          .expect('Content-Type', /json/)
+          .expect(200)
+        expect(res.body.errors).toBeUndefined()
+
+        // check variables no longer in cache
+        contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents).toBe(null)
+      })
+    })
+  })
+
   describe("fetch", () => {
     it("returns all the datasources from the server", async () => {
       const res = await request

package/src/api/routes/tests/permissions.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 

package/src/api/routes/tests/query.spec.js

@@ -1,15 +1,17 @@
 // Mock out postgres for this
 jest.mock("pg")
+jest.mock("node-fetch")
 
 // Mock isProdAppID to we can later mock the implementation and pretend we are
 // using prod app IDs
-const authDb = require("@budibase/auth/db")
+const authDb = require("@budibase/backend-core/db")
 const { isProdAppID } = authDb
 const mockIsProdAppID = jest.fn(isProdAppID)
 authDb.isProdAppID = mockIsProdAppID
 
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 const { basicQuery, basicDatasource } = setup.structures
 
 describe("/queries", () => {
@@ -226,4 +228,71 @@ describe("/queries", () => {
         .expect(400)
     })
   })
+
+  describe("variables", () => {
+
+    async function preview(datasource, fields) {
+      return config.previewQuery(request, config, datasource, fields)
+    }
+
+    it("should work with static variables", async () => {
+      const datasource = await config.restDatasource({
+        staticVariables: {
+          variable: "google",
+          variable2: "1",
+        },
+      })
+      const res = await preview(datasource, {
+        path: "www.{{ variable }}.com",
+        queryString: "test={{ variable2 }}",
+      })
+      // these responses come from the mock
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
+    })
+
+    it("should work with dynamic variables", async () => {
+      const { datasource } = await config.dynamicVariableDatasource()
+      const res = await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toContain("doctype html")
+    })
+
+    it("check that it automatically retries on fail with cached dynamics", async () => {
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      const contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+      const res = await preview(datasource, {
+        path: "www.failonce.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
+      expect(res.body.rows[0].fails).toEqual(1)
+    })
+
+    it("deletes variables when linked query is deleted", async () => {
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      let contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+
+      // delete the query
+      await request
+        .delete(`/api/queries/${base._id}/${base._rev}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+
+      // check variables no longer in cache
+      contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents).toBe(null)
+    })
+  })
 })

package/src/api/routes/tests/role.spec.js

@@ -1,7 +1,7 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const setup = require("./utilities")
 const { basicRole } = setup.structures
 

package/src/api/routes/tests/routing.spec.js

@@ -1,7 +1,7 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 
 const route = "/test"
 

package/src/api/routes/tests/user.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 

package/src/api/routes/tests/utilities/TestFunctions.js

@@ -2,7 +2,7 @@ const rowController = require("../../../controllers/row")
 const appController = require("../../../controllers/application")
 const CouchDB = require("../../../../db")
 const { AppStatus } = require("../../../../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { TENANT_ID } = require("../../../../tests/utilities/structures")
 
 function Request(appId, params) {

package/src/api/routes/user.js

@@ -4,7 +4,7 @@ const authorized = require("../../middleware/authorized")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/view.js

@@ -7,7 +7,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/webhook.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/webhook")
 const authorized = require("../../middleware/authorized")
 const joiValidator = require("../../middleware/joi-validator")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 const Joi = require("joi")
 
 const router = Router()

package/src/app.ts

@@ -3,7 +3,7 @@ import { ExtendableContext } from "koa"
 
 const env = require("./environment")
 const CouchDB = require("./db")
-require("@budibase/auth").init(CouchDB)
+require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")

package/src/automations/bullboard.js

@@ -6,7 +6,7 @@ const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
   : require("bull")
 const { JobQueues } = require("../constants")
-const { utils } = require("@budibase/auth/redis")
+const { utils } = require("@budibase/backend-core/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000

package/src/automations/steps/executeQuery.js

@@ -35,6 +35,11 @@ exports.definition = {
           type: "object",
           description: "The response from the datasource execution",
         },
+        info: {
+          type: "object",
+          description:
+            "Some query types may return extra data, like headers from a REST query",
+        },
         success: {
           type: "boolean",
           description: "Whether the action was successful",
@@ -67,14 +72,17 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    await queryController.execute(ctx)
+    await queryController.executeV1(ctx)
+    const { data, ...rest } = ctx.body
     return {
-      response: ctx.body,
+      response: data,
+      info: rest,
       success: true,
     }
   } catch (err) {
     return {
       success: false,
+      info: {},
       response: automationUtils.getError(err),
     }
   }

package/src/automations/utils.js

@@ -6,7 +6,7 @@ const { queue } = require("./bullboard")
 const newid = require("../db/newid")
 const { updateEntityMetadata } = require("../utilities")
 const { MetadataTypes } = require("../constants")
-const { getDeployedAppID } = require("@budibase/auth/db")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 const { cloneDeep } = require("lodash/fp")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId

package/src/constants/index.js

@@ -1,6 +1,6 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { UserStatus } = require("@budibase/auth").constants
-const { ObjectStoreBuckets } = require("@budibase/auth").objectStore
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { UserStatus } = require("@budibase/backend-core/constants")
+const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore")
 
 exports.JobQueues = {
   AUTOMATIONS: "automationQueue",

package/src/constants/screens.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { BASE_LAYOUT_PROP_IDS } = require("./layouts")
 
 exports.createHomeScreen = () => ({

package/src/db/client.js

@@ -1,5 +1,5 @@
 const PouchDB = require("pouchdb")
-const { getCouchUrl } = require("@budibase/auth/db")
+const { getCouchUrl } = require("@budibase/backend-core/db")
 const replicationStream = require("pouchdb-replication-stream")
 const allDbs = require("pouchdb-all-dbs")
 const find = require("pouchdb-find")

package/src/db/utils.js

@@ -9,7 +9,7 @@ const {
   StaticDatabases,
   isDevAppID,
   isProdAppID,
-} = require("@budibase/auth/db")
+} = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
 

package/src/definitions/datasource.ts

@@ -240,6 +240,16 @@ export interface RestConfig {
     [key: string]: any
   }
   authConfigs: AuthConfig[]
+  staticVariables: {
+    [key: string]: string
+  }
+  dynamicVariables: [
+    {
+      name: string
+      queryId: string
+      value: string
+    }
+  ]
 }
 
 export interface Query {

package/src/integrations/base/sqlTable.ts

@@ -36,7 +36,7 @@ function generateSchema(
       case FieldTypes.STRING:
       case FieldTypes.OPTIONS:
       case FieldTypes.LONGFORM:
-        schema.string(key)
+        schema.text(key)
         break
       case FieldTypes.NUMBER:
         // if meta is specified then this is a junction table entry

package/src/integrations/rest.ts

@@ -48,7 +48,10 @@ module RestModule {
   const { performance } = require("perf_hooks")
   const FormData = require("form-data")
   const { URLSearchParams } = require("url")
-  const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
+  const {
+    parseStringPromise: xmlParser,
+    Builder: XmlBuilder,
+  } = require("xml2js")
 
   const SCHEMA: Integration = {
     docs: "https://github.com/node-fetch/node-fetch",
@@ -168,7 +171,7 @@ module RestModule {
     getUrl(path: string, queryString: string): string {
       const main = `${path}?${queryString}`
       let complete = main
-      if (this.config.url && !main.startsWith(this.config.url)) {
+      if (this.config.url && !main.startsWith("http")) {
         complete = !this.config.url ? main : `${this.config.url}/${main}`
       }
       if (!complete.startsWith("http")) {
@@ -211,7 +214,7 @@ module RestModule {
           break
         case BodyTypes.XML:
           if (object != null) {
-            string = (new XmlBuilder()).buildObject(object)
+            string = new XmlBuilder().buildObject(object)
           }
           input.body = string
           input.headers["Content-Type"] = "application/xml"

package/src/middleware/authorized.js

@@ -2,11 +2,11 @@ const {
   getUserRoleHierarchy,
   getRequiredResourceRole,
   BUILTIN_ROLE_IDS,
-} = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/roles")
 const {
   PermissionTypes,
   doesHaveBasePermission,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const builderMiddleware = require("./builder")
 const { isWebhookEndpoint } = require("./utils")
 

package/src/middleware/builder.js

@@ -7,8 +7,8 @@ const {
 } = require("../utilities/redis")
 const CouchDB = require("../db")
 const { DocumentTypes } = require("../db/utils")
-const { PermissionTypes } = require("@budibase/auth/permissions")
-const { app: appCache } = require("@budibase/auth/cache")
+const { PermissionTypes } = require("@budibase/backend-core/permissions")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 const DEBOUNCE_TIME_SEC = 30
 

package/src/middleware/currentapp.js

@@ -1,11 +1,15 @@
-const { getAppId, setCookie, getCookie, clearCookie } =
-  require("@budibase/auth").utils
-const { Cookies } = require("@budibase/auth").constants
-const { getRole } = require("@budibase/auth/roles")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const {
+  getAppId,
+  setCookie,
+  getCookie,
+  clearCookie,
+} = require("@budibase/backend-core/utils")
+const { Cookies } = require("@budibase/backend-core/constants")
+const { getRole } = require("@budibase/backend-core/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { generateUserMetadataID, isDevAppID } = require("../db/utils")
-const { dbExists } = require("@budibase/auth/db")
-const { isUserInAppTenant } = require("@budibase/auth/tenancy")
+const { dbExists } = require("@budibase/backend-core/db")
+const { isUserInAppTenant } = require("@budibase/backend-core/tenancy")
 const { getCachedSelf } = require("../utilities/global")
 const CouchDB = require("../db")
 const env = require("../environment")

package/src/middleware/tests/authorized.spec.js

@@ -9,8 +9,8 @@ jest.mock("../../environment", () => ({
 )
 const authorizedMiddleware = require("../authorized")
 const env = require("../../environment")
-const { PermissionTypes, PermissionLevels } = require("@budibase/auth/permissions")
-require("@budibase/auth").init(require("../../db"))
+const { PermissionTypes, PermissionLevels } = require("@budibase/backend-core/permissions")
+require("@budibase/backend-core").init(require("../../db"))
 
 class TestConfiguration {
   constructor(role) {

package/src/middleware/tests/currentapp.spec.js

@@ -23,7 +23,7 @@ function mockReset() {
 function mockAuthWithNoCookie() {
   jest.resetModules()
   mockWorker()
-  jest.mock("@budibase/auth/cache", () => ({
+  jest.mock("@budibase/backend-core/cache", () => ({
     user: {
       getUser: () => {
         return {
@@ -32,34 +32,30 @@ function mockAuthWithNoCookie() {
       },
     },
   }))
-  jest.mock("@budibase/auth", () => ({
-    utils: {
-      getAppId: jest.fn(),
-      setCookie: jest.fn(),
-      getCookie: jest.fn(),
-    },
-    constants: {
-      Cookies: {},
-    },
+  jest.mock("@budibase/backend-core/utils", () => ({
+    getAppId: jest.fn(),
+    setCookie: jest.fn(),
+    getCookie: jest.fn(),
+  }))
+  jest.mock("@budibase/backend-core/constants", () => ({
+    Cookies: {},
   }))
 }
 
 function mockAuthWithCookie() {
   jest.resetModules()
   mockWorker()
-  jest.mock("@budibase/auth", () => ({
-    utils: {
-      getAppId: () => {
-        return "app_test"
-      },
-      setCookie: jest.fn(),
-      getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}),
+  jest.mock("@budibase/backend-core/utils", () => ({
+    getAppId: () => {
+      return "app_test"
     },
-    constants: {
-      Cookies: {
-        Auth: "auth",
-        CurrentApp: "currentapp",
-      },
+    setCookie: jest.fn(),
+    getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}),
+  }))
+  jest.mock("@budibase/backend-core/constants", () => ({
+    Cookies: {
+      Auth: "auth",
+      CurrentApp: "currentapp",
     },
   }))
 }
@@ -121,7 +117,7 @@ describe("Current app middleware", () => {
     async function checkExpected(setCookie) {
       config.setUser()
       await config.executeMiddleware()
-      const cookieFn = require("@budibase/auth").utils.setCookie
+      let { setCookie: cookieFn } = require("@budibase/backend-core/utils")
       if (setCookie) {
         expect(cookieFn).toHaveBeenCalled()
       } else {
@@ -140,32 +136,30 @@ describe("Current app middleware", () => {
 
     it("should perform correct when no cookie exists", async () => {
       mockReset()
-      jest.mock("@budibase/auth", () => ({
-        utils: {
-          getAppId: () => {
-            return "app_test"
-          },
-          setCookie: jest.fn(),
-          getCookie: jest.fn(),
-        },
-        constants: {
-          Cookies: {},
+      jest.mock("@budibase/backend-core/utils", () => ({
+        getAppId: () => {
+          return "app_test"
         },
+        setCookie: jest.fn(),
+        getCookie: jest.fn(),
+      }))
+      jest.mock("@budibase/backend-core/constants", () => ({
+        Cookies: {},
       }))
       await checkExpected(true)
     })
 
     it("lastly check what occurs when cookie doesn't need updated", async () => {
       mockReset()
-      jest.mock("@budibase/auth", () => ({
-        utils: {
-          getAppId: () => {
-            return "app_test"
-          },
-          setCookie: jest.fn(),
-          getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}),
+      jest.mock("@budibase/backend-core/utils", () => ({
+        getAppId: () => {
+          return "app_test"
         },
-        constants: { Cookies: {} },
+        setCookie: jest.fn(),
+        getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}),
+      }))
+      jest.mock("@budibase/backend-core/constants", () => ({
+        Cookies: {},
       }))
       await checkExpected(false)
     })

package/src/middleware/tests/usageQuota.spec.js

@@ -6,7 +6,7 @@ jest.mock("../../environment", () => ({
   isDev: () => true,
   _set: () => {},
 }))
-jest.mock("@budibase/auth/tenancy", () => ({
+jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
 

package/src/middleware/usageQuota.js

@@ -1,7 +1,7 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
 const env = require("../environment")
-const { getTenantId } = require("@budibase/auth/tenancy")
+const { getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   isExternalTable,
   isRowId: isExternalRowId,