@budibase/server 1.0.27-alpha.0 → 1.0.27-alpha.12

package/src/api/routes/tests/datasource.spec.js

@@ -4,6 +4,7 @@ let setup = require("./utilities")
 let { basicDatasource } = setup.structures
 let { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const pg = require("pg")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 
 describe("/datasources", () => {
   let request = setup.getRequest()
@@ -31,6 +32,50 @@ describe("/datasources", () => {
     })
   })
 
+  describe("update", () => {
+    it("should update an existing datasource", async () => {
+      datasource.name = "Updated Test"
+      const res = await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect('Content-Type', /json/)
+        .expect(200)
+
+      expect(res.body.datasource.name).toEqual("Updated Test")
+      expect(res.body.errors).toBeUndefined()
+    })
+
+    describe("dynamic variables", () => {
+      async function preview(datasource, fields) {
+        return config.previewQuery(request, config, datasource, fields)
+      }
+
+      it("should invalidate changed or removed variables", async () => {
+        const { datasource, query } = await config.dynamicVariableDatasource()
+        // preview once to cache variables
+        await preview(datasource, { path: "www.test.com", queryString: "test={{ variable3 }}" })
+        // check variables in cache
+        let contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents.rows.length).toEqual(1)
+        
+        // update the datasource to remove the variables
+        datasource.config.dynamicVariables = []
+        const res = await request
+          .put(`/api/datasources/${datasource._id}`)
+          .send(datasource)
+          .set(config.defaultHeaders())
+          .expect('Content-Type', /json/)
+          .expect(200)
+        expect(res.body.errors).toBeUndefined()
+
+        // check variables no longer in cache
+        contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents).toBe(null)
+      })
+    })
+  })
+
   describe("fetch", () => {
     it("returns all the datasources from the server", async () => {
       const res = await request

package/src/api/routes/tests/permissions.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 

package/src/api/routes/tests/query.spec.js

@@ -1,15 +1,17 @@
 // Mock out postgres for this
 jest.mock("pg")
+jest.mock("node-fetch")
 
 // Mock isProdAppID to we can later mock the implementation and pretend we are
 // using prod app IDs
-const authDb = require("@budibase/auth/db")
+const authDb = require("@budibase/backend-core/db")
 const { isProdAppID } = authDb
 const mockIsProdAppID = jest.fn(isProdAppID)
 authDb.isProdAppID = mockIsProdAppID
 
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 const { basicQuery, basicDatasource } = setup.structures
 
 describe("/queries", () => {
@@ -226,4 +228,71 @@ describe("/queries", () => {
         .expect(400)
     })
   })
+
+  describe("variables", () => {
+
+    async function preview(datasource, fields) {
+      return config.previewQuery(request, config, datasource, fields)
+    }
+
+    it("should work with static variables", async () => {
+      const datasource = await config.restDatasource({
+        staticVariables: {
+          variable: "google",
+          variable2: "1",
+        },
+      })
+      const res = await preview(datasource, {
+        path: "www.{{ variable }}.com",
+        queryString: "test={{ variable2 }}",
+      })
+      // these responses come from the mock
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=1")
+    })
+
+    it("should work with dynamic variables", async () => {
+      const { datasource } = await config.dynamicVariableDatasource()
+      const res = await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["url", "opts", "value"])
+      expect(res.body.rows[0].url).toContain("doctype html")
+    })
+
+    it("check that it automatically retries on fail with cached dynamics", async () => {
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      const contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+      const res = await preview(datasource, {
+        path: "www.failonce.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
+      expect(res.body.rows[0].fails).toEqual(1)
+    })
+
+    it("deletes variables when linked query is deleted", async () => {
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      let contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+
+      // delete the query
+      await request
+        .delete(`/api/queries/${base._id}/${base._rev}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+
+      // check variables no longer in cache
+      contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents).toBe(null)
+    })
+  })
 })

package/src/api/routes/tests/role.spec.js

@@ -1,7 +1,7 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const setup = require("./utilities")
 const { basicRole } = setup.structures
 

package/src/api/routes/tests/routing.spec.js

@@ -1,7 +1,7 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 
 const route = "/test"
 

package/src/api/routes/tests/user.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 

package/src/api/routes/tests/utilities/TestFunctions.js

@@ -2,7 +2,7 @@ const rowController = require("../../../controllers/row")
 const appController = require("../../../controllers/application")
 const CouchDB = require("../../../../db")
 const { AppStatus } = require("../../../../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { TENANT_ID } = require("../../../../tests/utilities/structures")
 
 function Request(appId, params) {

package/src/api/routes/user.js

@@ -4,7 +4,7 @@ const authorized = require("../../middleware/authorized")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/view.js

@@ -7,7 +7,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/webhook.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/webhook")
 const authorized = require("../../middleware/authorized")
 const joiValidator = require("../../middleware/joi-validator")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 const Joi = require("joi")
 
 const router = Router()

package/src/app.ts

@@ -3,7 +3,7 @@ import { ExtendableContext } from "koa"
 
 const env = require("./environment")
 const CouchDB = require("./db")
-require("@budibase/auth").init(CouchDB)
+require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")

package/src/automations/bullboard.js

@@ -6,7 +6,7 @@ const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
   : require("bull")
 const { JobQueues } = require("../constants")
-const { utils } = require("@budibase/auth/redis")
+const { utils } = require("@budibase/backend-core/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000

package/src/automations/steps/executeQuery.js

@@ -35,6 +35,11 @@ exports.definition = {
           type: "object",
           description: "The response from the datasource execution",
         },
+        info: {
+          type: "object",
+          description:
+            "Some query types may return extra data, like headers from a REST query",
+        },
         success: {
           type: "boolean",
           description: "Whether the action was successful",
@@ -68,13 +73,16 @@ exports.run = async function ({ inputs, appId, emitter }) {
 
   try {
     await queryController.execute(ctx)
+    const { data, ...rest } = ctx.body
     return {
-      response: ctx.body,
+      response: data,
+      info: rest,
       success: true,
     }
   } catch (err) {
     return {
       success: false,
+      info: {},
       response: automationUtils.getError(err),
     }
   }

package/src/automations/utils.js

@@ -6,7 +6,7 @@ const { queue } = require("./bullboard")
 const newid = require("../db/newid")
 const { updateEntityMetadata } = require("../utilities")
 const { MetadataTypes } = require("../constants")
-const { getDeployedAppID } = require("@budibase/auth/db")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 const { cloneDeep } = require("lodash/fp")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId

package/src/constants/index.js

@@ -1,6 +1,6 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { UserStatus } = require("@budibase/auth").constants
-const { ObjectStoreBuckets } = require("@budibase/auth").objectStore
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { UserStatus } = require("@budibase/backend-core").constants
+const { ObjectStoreBuckets } = require("@budibase/backend-core").objectStore
 
 exports.JobQueues = {
   AUTOMATIONS: "automationQueue",

package/src/constants/screens.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { BASE_LAYOUT_PROP_IDS } = require("./layouts")
 
 exports.createHomeScreen = () => ({

package/src/db/client.js

@@ -1,5 +1,5 @@
 const PouchDB = require("pouchdb")
-const { getCouchUrl } = require("@budibase/auth/db")
+const { getCouchUrl } = require("@budibase/backend-core/db")
 const replicationStream = require("pouchdb-replication-stream")
 const allDbs = require("pouchdb-all-dbs")
 const find = require("pouchdb-find")

package/src/db/utils.js

@@ -9,7 +9,7 @@ const {
   StaticDatabases,
   isDevAppID,
   isProdAppID,
-} = require("@budibase/auth/db")
+} = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
 

package/src/definitions/datasource.ts

@@ -240,6 +240,16 @@ export interface RestConfig {
     [key: string]: any
   }
   authConfigs: AuthConfig[]
+  staticVariables: {
+    [key: string]: string
+  }
+  dynamicVariables: [
+    {
+      name: string
+      queryId: string
+      value: string
+    }
+  ]
 }
 
 export interface Query {

package/src/integrations/base/sqlTable.ts

@@ -36,7 +36,7 @@ function generateSchema(
       case FieldTypes.STRING:
       case FieldTypes.OPTIONS:
       case FieldTypes.LONGFORM:
-        schema.string(key)
+        schema.text(key)
         break
       case FieldTypes.NUMBER:
         // if meta is specified then this is a junction table entry

package/src/integrations/rest.ts

@@ -48,7 +48,10 @@ module RestModule {
   const { performance } = require("perf_hooks")
   const FormData = require("form-data")
   const { URLSearchParams } = require("url")
-  const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
+  const {
+    parseStringPromise: xmlParser,
+    Builder: XmlBuilder,
+  } = require("xml2js")
 
   const SCHEMA: Integration = {
     docs: "https://github.com/node-fetch/node-fetch",
@@ -168,7 +171,7 @@ module RestModule {
     getUrl(path: string, queryString: string): string {
       const main = `${path}?${queryString}`
       let complete = main
-      if (this.config.url && !main.startsWith(this.config.url)) {
+      if (this.config.url && !main.startsWith("http")) {
         complete = !this.config.url ? main : `${this.config.url}/${main}`
       }
       if (!complete.startsWith("http")) {
@@ -211,7 +214,7 @@ module RestModule {
           break
         case BodyTypes.XML:
           if (object != null) {
-            string = (new XmlBuilder()).buildObject(object)
+            string = new XmlBuilder().buildObject(object)
           }
           input.body = string
           input.headers["Content-Type"] = "application/xml"

package/src/middleware/authorized.js

@@ -2,11 +2,11 @@ const {
   getUserRoleHierarchy,
   getRequiredResourceRole,
   BUILTIN_ROLE_IDS,
-} = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/roles")
 const {
   PermissionTypes,
   doesHaveBasePermission,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const builderMiddleware = require("./builder")
 const { isWebhookEndpoint } = require("./utils")
 

package/src/middleware/builder.js

@@ -7,8 +7,8 @@ const {
 } = require("../utilities/redis")
 const CouchDB = require("../db")
 const { DocumentTypes } = require("../db/utils")
-const { PermissionTypes } = require("@budibase/auth/permissions")
-const { app: appCache } = require("@budibase/auth/cache")
+const { PermissionTypes } = require("@budibase/backend-core/permissions")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 const DEBOUNCE_TIME_SEC = 30
 

package/src/middleware/currentapp.js

@@ -1,11 +1,11 @@
 const { getAppId, setCookie, getCookie, clearCookie } =
-  require("@budibase/auth").utils
-const { Cookies } = require("@budibase/auth").constants
-const { getRole } = require("@budibase/auth/roles")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+  require("@budibase/backend-core").utils
+const { Cookies } = require("@budibase/backend-core").constants
+const { getRole } = require("@budibase/backend-core/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { generateUserMetadataID, isDevAppID } = require("../db/utils")
-const { dbExists } = require("@budibase/auth/db")
-const { isUserInAppTenant } = require("@budibase/auth/tenancy")
+const { dbExists } = require("@budibase/backend-core/db")
+const { isUserInAppTenant } = require("@budibase/backend-core/tenancy")
 const { getCachedSelf } = require("../utilities/global")
 const CouchDB = require("../db")
 const env = require("../environment")

package/src/middleware/tests/authorized.spec.js

@@ -9,8 +9,8 @@ jest.mock("../../environment", () => ({
 )
 const authorizedMiddleware = require("../authorized")
 const env = require("../../environment")
-const { PermissionTypes, PermissionLevels } = require("@budibase/auth/permissions")
-require("@budibase/auth").init(require("../../db"))
+const { PermissionTypes, PermissionLevels } = require("@budibase/backend-core/permissions")
+require("@budibase/backend-core").init(require("../../db"))
 
 class TestConfiguration {
   constructor(role) {

package/src/middleware/tests/currentapp.spec.js

@@ -23,7 +23,7 @@ function mockReset() {
 function mockAuthWithNoCookie() {
   jest.resetModules()
   mockWorker()
-  jest.mock("@budibase/auth/cache", () => ({
+  jest.mock("@budibase/backend-core/cache", () => ({
     user: {
       getUser: () => {
         return {
@@ -32,7 +32,7 @@ function mockAuthWithNoCookie() {
       },
     },
   }))
-  jest.mock("@budibase/auth", () => ({
+  jest.mock("@budibase/backend-core", () => ({
     utils: {
       getAppId: jest.fn(),
       setCookie: jest.fn(),
@@ -47,7 +47,7 @@ function mockAuthWithNoCookie() {
 function mockAuthWithCookie() {
   jest.resetModules()
   mockWorker()
-  jest.mock("@budibase/auth", () => ({
+  jest.mock("@budibase/backend-core", () => ({
     utils: {
       getAppId: () => {
         return "app_test"
@@ -121,7 +121,7 @@ describe("Current app middleware", () => {
     async function checkExpected(setCookie) {
       config.setUser()
       await config.executeMiddleware()
-      const cookieFn = require("@budibase/auth").utils.setCookie
+      const cookieFn = require("@budibase/backend-core").utils.setCookie
       if (setCookie) {
         expect(cookieFn).toHaveBeenCalled()
       } else {
@@ -140,7 +140,7 @@ describe("Current app middleware", () => {
 
     it("should perform correct when no cookie exists", async () => {
       mockReset()
-      jest.mock("@budibase/auth", () => ({
+      jest.mock("@budibase/backend-core", () => ({
         utils: {
           getAppId: () => {
             return "app_test"
@@ -157,7 +157,7 @@ describe("Current app middleware", () => {
 
     it("lastly check what occurs when cookie doesn't need updated", async () => {
       mockReset()
-      jest.mock("@budibase/auth", () => ({
+      jest.mock("@budibase/backend-core", () => ({
         utils: {
           getAppId: () => {
             return "app_test"

package/src/middleware/tests/usageQuota.spec.js

@@ -6,7 +6,7 @@ jest.mock("../../environment", () => ({
   isDev: () => true,
   _set: () => {},
 }))
-jest.mock("@budibase/auth/tenancy", () => ({
+jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
 

package/src/middleware/usageQuota.js

@@ -1,7 +1,7 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
 const env = require("../environment")
-const { getTenantId } = require("@budibase/auth/tenancy")
+const { getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   isExternalTable,
   isRowId: isExternalRowId,

package/src/tests/utilities/TestConfiguration.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const env = require("../../environment")
 const {
   basicTable,
@@ -15,14 +15,14 @@ const {
 const controllers = require("./controllers")
 const supertest = require("supertest")
 const { cleanup } = require("../../utilities/fileSystem")
-const { Cookies, Headers } = require("@budibase/auth").constants
-const { jwt } = require("@budibase/auth").auth
-const auth = require("@budibase/auth")
-const { getGlobalDB } = require("@budibase/auth/tenancy")
-const { createASession } = require("@budibase/auth/sessions")
-const { user: userCache } = require("@budibase/auth/cache")
+const { Cookies, Headers } = require("@budibase/backend-core").constants
+const { jwt } = require("@budibase/backend-core").auth
+const core = require("@budibase/backend-core")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { createASession } = require("@budibase/backend-core/sessions")
+const { user: userCache } = require("@budibase/backend-core/cache")
 const CouchDB = require("../../db")
-auth.init(CouchDB)
+core.init(CouchDB)
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -316,6 +316,63 @@ class TestConfiguration {
     return this.datasource
   }
 
+  async updateDatasource(datasource) {
+    const response = await this._req(
+      datasource,
+      { datasourceId: datasource._id },
+      controllers.datasource.update
+    )
+    this.datasource = response.datasource
+    return this.datasource
+  }
+
+  async restDatasource(cfg) {
+    return this.createDatasource({
+      datasource: {
+        ...basicDatasource().datasource,
+        source: "REST",
+        config: cfg || {},
+      },
+    })
+  }
+
+  async dynamicVariableDatasource() {
+    let datasource = await this.restDatasource()
+    const basedOnQuery = await this.createQuery({
+      ...basicQuery(datasource._id),
+      fields: {
+        path: "www.google.com",
+      },
+    })
+    datasource = await this.updateDatasource({
+      ...datasource,
+      config: {
+        dynamicVariables: [
+          {
+            queryId: basedOnQuery._id,
+            name: "variable3",
+            value: "{{ data.0.[value] }}",
+          },
+        ],
+      },
+    })
+    return { datasource, query: basedOnQuery }
+  }
+
+  async previewQuery(request, config, datasource, fields) {
+    return request
+      .post(`/api/queries/preview`)
+      .send({
+        datasourceId: datasource._id,
+        parameters: {},
+        fields,
+        queryVerb: "read",
+      })
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  }
+
   async createQuery(config = null) {
     if (!this.datasource && !config) {
       throw "No data source created for query."

package/src/tests/utilities/structures.js

@@ -1,5 +1,5 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { BUILTIN_PERMISSION_IDS } = require("@budibase/auth/permissions")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { BUILTIN_PERMISSION_IDS } = require("@budibase/backend-core/permissions")
 const { createHomeScreen } = require("../../constants/screens")
 const { EMPTY_LAYOUT } = require("../../constants/layouts")
 const { cloneDeep } = require("lodash/fp")

package/src/threads/automation.js

@@ -4,10 +4,10 @@ const actions = require("../automations/actions")
 const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
-const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
+const { DEFAULT_TENANT_ID } = require("@budibase/backend-core").constants
 const CouchDB = require("../db")
 const { DocumentTypes, isDevAppID } = require("../db/utils")
-const { doInTenant } = require("@budibase/auth/tenancy")
+const { doInTenant } = require("@budibase/backend-core/tenancy")
 const usage = require("../utilities/usageQuota")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")