@budibase/server 1.0.32 → 1.0.39

package/src/middleware/tests/currentapp.spec.js

@@ -23,7 +23,7 @@ function mockReset() {
 function mockAuthWithNoCookie() {
   jest.resetModules()
   mockWorker()
-  jest.mock("@budibase/auth/cache", () => ({
+  jest.mock("@budibase/backend-core/cache", () => ({
     user: {
       getUser: () => {
         return {
@@ -32,34 +32,30 @@ function mockAuthWithNoCookie() {
       },
     },
   }))
-  jest.mock("@budibase/auth", () => ({
-    utils: {
-      getAppId: jest.fn(),
-      setCookie: jest.fn(),
-      getCookie: jest.fn(),
-    },
-    constants: {
-      Cookies: {},
-    },
+  jest.mock("@budibase/backend-core/utils", () => ({
+    getAppId: jest.fn(),
+    setCookie: jest.fn(),
+    getCookie: jest.fn(),
+  }))
+  jest.mock("@budibase/backend-core/constants", () => ({
+    Cookies: {},
   }))
 }
 
 function mockAuthWithCookie() {
   jest.resetModules()
   mockWorker()
-  jest.mock("@budibase/auth", () => ({
-    utils: {
-      getAppId: () => {
-        return "app_test"
-      },
-      setCookie: jest.fn(),
-      getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}),
+  jest.mock("@budibase/backend-core/utils", () => ({
+    getAppId: () => {
+      return "app_test"
     },
-    constants: {
-      Cookies: {
-        Auth: "auth",
-        CurrentApp: "currentapp",
-      },
+    setCookie: jest.fn(),
+    getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}),
+  }))
+  jest.mock("@budibase/backend-core/constants", () => ({
+    Cookies: {
+      Auth: "auth",
+      CurrentApp: "currentapp",
     },
   }))
 }
@@ -121,7 +117,7 @@ describe("Current app middleware", () => {
     async function checkExpected(setCookie) {
       config.setUser()
       await config.executeMiddleware()
-      const cookieFn = require("@budibase/auth").utils.setCookie
+      let { setCookie: cookieFn } = require("@budibase/backend-core/utils")
       if (setCookie) {
         expect(cookieFn).toHaveBeenCalled()
       } else {
@@ -140,32 +136,30 @@ describe("Current app middleware", () => {
 
     it("should perform correct when no cookie exists", async () => {
       mockReset()
-      jest.mock("@budibase/auth", () => ({
-        utils: {
-          getAppId: () => {
-            return "app_test"
-          },
-          setCookie: jest.fn(),
-          getCookie: jest.fn(),
-        },
-        constants: {
-          Cookies: {},
+      jest.mock("@budibase/backend-core/utils", () => ({
+        getAppId: () => {
+          return "app_test"
         },
+        setCookie: jest.fn(),
+        getCookie: jest.fn(),
+      }))
+      jest.mock("@budibase/backend-core/constants", () => ({
+        Cookies: {},
       }))
       await checkExpected(true)
     })
 
     it("lastly check what occurs when cookie doesn't need updated", async () => {
       mockReset()
-      jest.mock("@budibase/auth", () => ({
-        utils: {
-          getAppId: () => {
-            return "app_test"
-          },
-          setCookie: jest.fn(),
-          getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}),
+      jest.mock("@budibase/backend-core/utils", () => ({
+        getAppId: () => {
+          return "app_test"
         },
-        constants: { Cookies: {} },
+        setCookie: jest.fn(),
+        getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}),
+      }))
+      jest.mock("@budibase/backend-core/constants", () => ({
+        Cookies: {},
       }))
       await checkExpected(false)
     })

package/src/middleware/tests/usageQuota.spec.js

@@ -6,7 +6,7 @@ jest.mock("../../environment", () => ({
   isDev: () => true,
   _set: () => {},
 }))
-jest.mock("@budibase/auth/tenancy", () => ({
+jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
 

package/src/middleware/usageQuota.js

@@ -1,7 +1,7 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
 const env = require("../environment")
-const { getTenantId } = require("@budibase/auth/tenancy")
+const { getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   isExternalTable,
   isRowId: isExternalRowId,

package/src/tests/utilities/TestConfiguration.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const env = require("../../environment")
 const {
   basicTable,
@@ -15,14 +15,14 @@ const {
 const controllers = require("./controllers")
 const supertest = require("supertest")
 const { cleanup } = require("../../utilities/fileSystem")
-const { Cookies, Headers } = require("@budibase/auth").constants
-const { jwt } = require("@budibase/auth").auth
-const auth = require("@budibase/auth")
-const { getGlobalDB } = require("@budibase/auth/tenancy")
-const { createASession } = require("@budibase/auth/sessions")
-const { user: userCache } = require("@budibase/auth/cache")
+const { Cookies, Headers } = require("@budibase/backend-core/constants")
+const { jwt } = require("@budibase/backend-core/auth")
+const core = require("@budibase/backend-core")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { createASession } = require("@budibase/backend-core/sessions")
+const { user: userCache } = require("@budibase/backend-core/cache")
 const CouchDB = require("../../db")
-auth.init(CouchDB)
+core.init(CouchDB)
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -316,6 +316,63 @@ class TestConfiguration {
     return this.datasource
   }
 
+  async updateDatasource(datasource) {
+    const response = await this._req(
+      datasource,
+      { datasourceId: datasource._id },
+      controllers.datasource.update
+    )
+    this.datasource = response.datasource
+    return this.datasource
+  }
+
+  async restDatasource(cfg) {
+    return this.createDatasource({
+      datasource: {
+        ...basicDatasource().datasource,
+        source: "REST",
+        config: cfg || {},
+      },
+    })
+  }
+
+  async dynamicVariableDatasource() {
+    let datasource = await this.restDatasource()
+    const basedOnQuery = await this.createQuery({
+      ...basicQuery(datasource._id),
+      fields: {
+        path: "www.google.com",
+      },
+    })
+    datasource = await this.updateDatasource({
+      ...datasource,
+      config: {
+        dynamicVariables: [
+          {
+            queryId: basedOnQuery._id,
+            name: "variable3",
+            value: "{{ data.0.[value] }}",
+          },
+        ],
+      },
+    })
+    return { datasource, query: basedOnQuery }
+  }
+
+  async previewQuery(request, config, datasource, fields) {
+    return request
+      .post(`/api/queries/preview`)
+      .send({
+        datasourceId: datasource._id,
+        parameters: {},
+        fields,
+        queryVerb: "read",
+      })
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  }
+
   async createQuery(config = null) {
     if (!this.datasource && !config) {
       throw "No data source created for query."

package/src/tests/utilities/structures.js

@@ -1,5 +1,5 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { BUILTIN_PERMISSION_IDS } = require("@budibase/auth/permissions")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { BUILTIN_PERMISSION_IDS } = require("@budibase/backend-core/permissions")
 const { createHomeScreen } = require("../../constants/screens")
 const { EMPTY_LAYOUT } = require("../../constants/layouts")
 const { cloneDeep } = require("lodash/fp")

package/src/threads/automation.js

@@ -4,10 +4,10 @@ const actions = require("../automations/actions")
 const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
-const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
+const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants")
 const CouchDB = require("../db")
 const { DocumentTypes, isDevAppID } = require("../db/utils")
-const { doInTenant } = require("@budibase/auth/tenancy")
+const { doInTenant } = require("@budibase/backend-core/tenancy")
 const usage = require("../utilities/usageQuota")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 

package/src/threads/query.js

@@ -1,60 +1,225 @@
-require("./utils").threadSetup()
+const threadUtils = require("./utils")
+threadUtils.threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
+const { processStringSync } = require("@budibase/string-templates")
+const CouchDB = require("../db")
 
-function formatResponse(resp) {
-  if (typeof resp === "string") {
-    try {
-      resp = JSON.parse(resp)
-    } catch (err) {
-      resp = { response: resp }
-    }
-  }
-  return resp
-}
+const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
+const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
 
-async function runAndTransform(datasource, queryVerb, query, transformer) {
-  const Integration = integrations[datasource.source]
-  if (!Integration) {
-    throw "Integration type does not exist."
+class QueryRunner {
+  constructor(input, flags = { noRecursiveQuery: false }) {
+    this.appId = input.appId
+    this.datasource = input.datasource
+    this.queryVerb = input.queryVerb
+    this.fields = input.fields
+    this.parameters = input.parameters
+    this.pagination = input.pagination
+    this.transformer = input.transformer
+    this.queryId = input.queryId
+    this.noRecursiveQuery = flags.noRecursiveQuery
+    this.cachedVariables = []
+    // allows the response from a query to be stored throughout this
+    // execution so that if it needs to be re-used for another variable
+    // it can be
+    this.queryResponse = {}
+    this.hasRerun = false
   }
-  const integration = new Integration(datasource.config)
 
-  let rows = formatResponse(await integration[queryVerb](query))
+  async execute() {
+    let { datasource, fields, queryVerb, transformer } = this
+    // pre-query, make sure datasource variables are added to parameters
+    const parameters = await this.addDatasourceVariables()
+    let query = this.enrichQueryFields(fields, parameters)
 
-  // transform as required
-  if (transformer) {
-    const runner = new ScriptRunner(transformer, { data: rows })
-    rows = runner.execute()
-  }
+    // Add pagination values for REST queries
+    if (this.pagination) {
+      query.paginationValues = this.pagination
+    }
+
+    const Integration = integrations[datasource.source]
+    if (!Integration) {
+      throw "Integration type does not exist."
+    }
+    const integration = new Integration(datasource.config)
+
+    let output = threadUtils.formatResponse(await integration[queryVerb](query))
+    let rows = output,
+      info = undefined,
+      extra = undefined,
+      pagination = undefined
+    if (threadUtils.hasExtraData(output)) {
+      rows = output.data
+      info = output.info
+      extra = output.extra
+      pagination = output.pagination
+    }
+
+    // transform as required
+    if (transformer) {
+      const runner = new ScriptRunner(transformer, {
+        data: rows,
+        params: parameters,
+      })
+      rows = runner.execute()
+    }
+
+    // if the request fails we retry once, invalidating the cached value
+    if (
+      info &&
+      info.code >= 400 &&
+      this.cachedVariables.length > 0 &&
+      !this.hasRerun
+    ) {
+      this.hasRerun = true
+      // invalidate the cache value
+      await threadUtils.invalidateDynamicVariables(this.cachedVariables)
+      return this.execute()
+    }
+
+    // needs to an array for next step
+    if (!Array.isArray(rows)) {
+      rows = [rows]
+    }
 
-  // needs to an array for next step
-  if (!Array.isArray(rows)) {
-    rows = [rows]
+    // map into JSON if just raw primitive here
+    if (rows.find(row => typeof row !== "object")) {
+      rows = rows.map(value => ({ value }))
+    }
+
+    // get all the potential fields in the schema
+    let keys = rows.flatMap(Object.keys)
+
+    if (integration.end) {
+      integration.end()
+    }
+
+    return { rows, keys, info, extra, pagination }
   }
 
-  // map into JSON if just raw primitive here
-  if (rows.find(row => typeof row !== "object")) {
-    rows = rows.map(value => ({ value }))
+  async runAnotherQuery(queryId, parameters) {
+    const db = new CouchDB(this.appId)
+    const query = await db.get(queryId)
+    const datasource = await db.get(query.datasourceId)
+    return new QueryRunner(
+      {
+        appId: this.appId,
+        datasource,
+        queryVerb: query.queryVerb,
+        fields: query.fields,
+        parameters,
+        transformer: query.transformer,
+      },
+      { noRecursiveQuery: true }
+    ).execute()
   }
 
-  // get all the potential fields in the schema
-  let keys = rows.flatMap(Object.keys)
+  async getDynamicVariable(variable) {
+    let { parameters } = this
+    const queryId = variable.queryId,
+      name = variable.name
+    let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
+    if (!value) {
+      value = this.queryResponse[queryId]
+        ? this.queryResponse[queryId]
+        : await this.runAnotherQuery(queryId, parameters)
+      // store incase this query is to be called again
+      this.queryResponse[queryId] = value
+      await threadUtils.storeDynamicVariable(queryId, name, value)
+    } else {
+      this.cachedVariables.push({ queryId, name })
+    }
+    return value
+  }
 
-  if (integration.end) {
-    integration.end()
+  async addDatasourceVariables() {
+    let { datasource, parameters, fields } = this
+    if (!datasource || !datasource.config) {
+      return parameters
+    }
+    const staticVars = datasource.config.staticVariables || {}
+    const dynamicVars = datasource.config.dynamicVariables || []
+    for (let [key, value] of Object.entries(staticVars)) {
+      if (!parameters[key]) {
+        parameters[key] = value
+      }
+    }
+    if (!this.noRecursiveQuery) {
+      // need to see if this uses any variables
+      const stringFields = JSON.stringify(fields)
+      const foundVars = dynamicVars.filter(variable => {
+        // don't allow a query to use its own dynamic variable (loop)
+        if (variable.queryId === this.queryId) {
+          return false
+        }
+        // look for {{ variable }} but allow spaces between handlebars
+        const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
+        return regex.test(stringFields)
+      })
+      const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
+      const responses = await Promise.all(dynamics)
+      for (let i = 0; i < foundVars.length; i++) {
+        const variable = foundVars[i]
+        parameters[variable.name] = processStringSync(variable.value, {
+          data: responses[i].rows,
+          info: responses[i].extra,
+        })
+        // make sure its known that this uses dynamic variables in case it fails
+        this.hasDynamicVariables = true
+      }
+    }
+    return parameters
   }
 
-  return { rows, keys }
+  enrichQueryFields(fields, parameters = {}) {
+    const enrichedQuery = {}
+
+    // enrich the fields with dynamic parameters
+    for (let key of Object.keys(fields)) {
+      if (fields[key] == null) {
+        continue
+      }
+      if (typeof fields[key] === "object") {
+        // enrich nested fields object
+        enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
+      } else if (typeof fields[key] === "string") {
+        // enrich string value as normal
+        let value = fields[key]
+        // add triple brace to avoid escaping e.g. '=' in cookie header
+        if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
+          value = `{${value}}`
+        }
+        enrichedQuery[key] = processStringSync(value, parameters, {
+          noHelpers: true,
+        })
+      } else {
+        enrichedQuery[key] = fields[key]
+      }
+    }
+    if (
+      enrichedQuery.json ||
+      enrichedQuery.customData ||
+      enrichedQuery.requestBody
+    ) {
+      try {
+        enrichedQuery.json = JSON.parse(
+          enrichedQuery.json ||
+            enrichedQuery.customData ||
+            enrichedQuery.requestBody
+        )
+      } catch (err) {
+        // no json found, ignore
+      }
+      delete enrichedQuery.customData
+    }
+    return enrichedQuery
+  }
 }
 
 module.exports = (input, callback) => {
-  runAndTransform(
-    input.datasource,
-    input.queryVerb,
-    input.query,
-    input.transformer
-  )
+  const Runner = new QueryRunner(input)
+  Runner.execute()
     .then(response => {
       callback(null, response)
     })

package/src/threads/utils.js

@@ -1,6 +1,22 @@
 const env = require("../environment")
 const CouchDB = require("../db")
-const { init } = require("@budibase/auth")
+const { init } = require("@budibase/backend-core")
+const redis = require("@budibase/backend-core/redis")
+const { SEPARATOR } = require("@budibase/backend-core/db")
+
+const VARIABLE_TTL_SECONDS = 3600
+let client
+
+async function getClient() {
+  if (!client) {
+    client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
+  }
+  return client
+}
+
+process.on("exit", async () => {
+  if (client) await client.finish()
+})
 
 exports.threadSetup = () => {
   // don't run this if not threading
@@ -11,3 +27,52 @@ exports.threadSetup = () => {
   env.setInThread()
   init(CouchDB)
 }
+
+function makeVariableKey(queryId, variable) {
+  return `${queryId}${SEPARATOR}${variable}`
+}
+
+exports.checkCacheForDynamicVariable = async (queryId, variable) => {
+  const cache = await getClient()
+  return cache.get(makeVariableKey(queryId, variable))
+}
+
+exports.invalidateDynamicVariables = async cachedVars => {
+  const cache = await getClient()
+  let promises = []
+  for (let variable of cachedVars) {
+    promises.push(
+      cache.delete(makeVariableKey(variable.queryId, variable.name))
+    )
+  }
+  await Promise.all(promises)
+}
+
+exports.storeDynamicVariable = async (queryId, variable, value) => {
+  const cache = await getClient()
+  await cache.store(
+    makeVariableKey(queryId, variable),
+    value,
+    VARIABLE_TTL_SECONDS
+  )
+}
+
+exports.formatResponse = resp => {
+  if (typeof resp === "string") {
+    try {
+      resp = JSON.parse(resp)
+    } catch (err) {
+      resp = { response: resp }
+    }
+  }
+  return resp
+}
+
+exports.hasExtraData = response => {
+  return (
+    typeof response === "object" &&
+    !Array.isArray(response) &&
+    response.data != null &&
+    response.info != null
+  )
+}

package/src/utilities/budibaseDir.js

@@ -1,7 +1,7 @@
 const { join } = require("./centralPath")
 const { homedir } = require("os")
 const env = require("../environment")
-const { budibaseTempDir } = require("@budibase/auth").objectStore
+const { budibaseTempDir } = require("@budibase/backend-core/objectStore")
 
 module.exports.budibaseAppsDir = function () {
   return env.BUDIBASE_DIR || join(homedir(), ".budibase")

package/src/utilities/fileSystem/utilities.js

@@ -2,13 +2,14 @@ const {
   ObjectStore,
   makeSureBucketExists,
   upload,
+  deleteFiles,
   streamUpload,
   retrieve,
   retrieveToTmp,
   deleteFolder,
   uploadDirectory,
   downloadTarball,
-} = require("@budibase/auth").objectStore
+} = require("@budibase/backend-core/objectStore")
 
 /***********************************
  *              NOTE               *
@@ -28,3 +29,4 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.deleteFiles = deleteFiles

package/src/utilities/global.js

@@ -2,11 +2,14 @@ const {
   getMultiIDParams,
   getGlobalIDFromUserMetadataID,
 } = require("../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { getDeployedAppID } = require("@budibase/auth/db")
-const { getGlobalUserParams } = require("@budibase/auth/db")
-const { user: userCache } = require("@budibase/auth/cache")
-const { getGlobalDB, isUserInAppTenant } = require("@budibase/auth/tenancy")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
+const { getGlobalUserParams } = require("@budibase/backend-core/db")
+const { user: userCache } = require("@budibase/backend-core/cache")
+const {
+  getGlobalDB,
+  isUserInAppTenant,
+} = require("@budibase/backend-core/tenancy")
 const env = require("../environment")
 
 exports.updateAppRole = (appId, user) => {

package/src/utilities/index.js

@@ -1,6 +1,6 @@
 const env = require("../environment")
 const { OBJ_STORE_DIRECTORY } = require("../constants")
-const { sanitizeKey } = require("@budibase/auth/src/objectStore")
+const { sanitizeKey } = require("@budibase/backend-core/objectStore")
 const CouchDB = require("../db")
 const { generateMetadataID } = require("../db/utils")
 const Readable = require("stream").Readable
@@ -150,3 +150,14 @@ exports.doesDatabaseExist = async dbName => {
     return false
   }
 }
+
+exports.formatBytes = bytes => {
+  const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
+  const byteIncrements = 1024
+  let unit = 0
+  let size = parseInt(bytes, 10) || 0
+  while (size >= byteIncrements && ++unit) {
+    size /= byteIncrements
+  }
+  return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)}${units[unit]}`
+}

package/src/utilities/redis.js

@@ -1,4 +1,4 @@
-const { Client, utils } = require("@budibase/auth/redis")
+const { Client, utils } = require("@budibase/backend-core/redis")
 const { getGlobalIDFromUserMetadataID } = require("../db/utils")
 
 const APP_DEV_LOCK_SECONDS = 600