@budibase/server 1.0.27-alpha.10 → 1.0.27-alpha.14

package/src/tests/utilities/TestConfiguration.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const env = require("../../environment")
 const {
   basicTable,
@@ -15,14 +15,14 @@ const {
 const controllers = require("./controllers")
 const supertest = require("supertest")
 const { cleanup } = require("../../utilities/fileSystem")
-const { Cookies, Headers } = require("@budibase/auth").constants
-const { jwt } = require("@budibase/auth").auth
-const auth = require("@budibase/auth")
-const { getGlobalDB } = require("@budibase/auth/tenancy")
-const { createASession } = require("@budibase/auth/sessions")
-const { user: userCache } = require("@budibase/auth/cache")
+const { Cookies, Headers } = require("@budibase/backend-core/constants")
+const { jwt } = require("@budibase/backend-core/auth")
+const core = require("@budibase/backend-core")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { createASession } = require("@budibase/backend-core/sessions")
+const { user: userCache } = require("@budibase/backend-core/cache")
 const CouchDB = require("../../db")
-auth.init(CouchDB)
+core.init(CouchDB)
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -326,6 +326,53 @@ class TestConfiguration {
     return this.datasource
   }
 
+  async restDatasource(cfg) {
+    return this.createDatasource({
+      datasource: {
+        ...basicDatasource().datasource,
+        source: "REST",
+        config: cfg || {},
+      },
+    })
+  }
+
+  async dynamicVariableDatasource() {
+    let datasource = await this.restDatasource()
+    const basedOnQuery = await this.createQuery({
+      ...basicQuery(datasource._id),
+      fields: {
+        path: "www.google.com",
+      },
+    })
+    datasource = await this.updateDatasource({
+      ...datasource,
+      config: {
+        dynamicVariables: [
+          {
+            queryId: basedOnQuery._id,
+            name: "variable3",
+            value: "{{ data.0.[value] }}",
+          },
+        ],
+      },
+    })
+    return { datasource, query: basedOnQuery }
+  }
+
+  async previewQuery(request, config, datasource, fields) {
+    return request
+      .post(`/api/queries/preview`)
+      .send({
+        datasourceId: datasource._id,
+        parameters: {},
+        fields,
+        queryVerb: "read",
+      })
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  }
+
   async createQuery(config = null) {
     if (!this.datasource && !config) {
       throw "No data source created for query."

package/src/tests/utilities/structures.js

@@ -1,5 +1,5 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { BUILTIN_PERMISSION_IDS } = require("@budibase/auth/permissions")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { BUILTIN_PERMISSION_IDS } = require("@budibase/backend-core/permissions")
 const { createHomeScreen } = require("../../constants/screens")
 const { EMPTY_LAYOUT } = require("../../constants/layouts")
 const { cloneDeep } = require("lodash/fp")

package/src/threads/automation.js

@@ -4,10 +4,10 @@ const actions = require("../automations/actions")
 const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
-const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
+const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants")
 const CouchDB = require("../db")
 const { DocumentTypes, isDevAppID } = require("../db/utils")
-const { doInTenant } = require("@budibase/auth/tenancy")
+const { doInTenant } = require("@budibase/backend-core/tenancy")
 const usage = require("../utilities/usageQuota")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 

package/src/threads/query.js

@@ -5,6 +5,9 @@ const { integrations } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
 const CouchDB = require("../db")
 
+const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
+const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
+
 class QueryRunner {
   constructor(input, flags = { noRecursiveQuery: false }) {
     this.appId = input.appId
@@ -12,6 +15,7 @@ class QueryRunner {
     this.queryVerb = input.queryVerb
     this.fields = input.fields
     this.parameters = input.parameters
+    this.pagination = input.pagination
     this.transformer = input.transformer
     this.queryId = input.queryId
     this.noRecursiveQuery = flags.noRecursiveQuery
@@ -27,7 +31,13 @@ class QueryRunner {
     let { datasource, fields, queryVerb, transformer } = this
     // pre-query, make sure datasource variables are added to parameters
     const parameters = await this.addDatasourceVariables()
-    const query = threadUtils.enrichQueryFields(fields, parameters)
+    let query = this.enrichQueryFields(fields, parameters)
+
+    // Add pagination values for REST queries
+    if (this.pagination) {
+      query.paginationValues = this.pagination
+    }
+
     const Integration = integrations[datasource.source]
     if (!Integration) {
       throw "Integration type does not exist."
@@ -37,11 +47,13 @@ class QueryRunner {
     let output = threadUtils.formatResponse(await integration[queryVerb](query))
     let rows = output,
       info = undefined,
-      extra = undefined
+      extra = undefined,
+      pagination = undefined
     if (threadUtils.hasExtraData(output)) {
       rows = output.data
       info = output.info
       extra = output.extra
+      pagination = output.pagination
     }
 
     // transform as required
@@ -83,7 +95,7 @@ class QueryRunner {
       integration.end()
     }
 
-    return { rows, keys, info, extra }
+    return { rows, keys, info, extra, pagination }
   }
 
   async runAnotherQuery(queryId, parameters) {
@@ -159,6 +171,50 @@ class QueryRunner {
     }
     return parameters
   }
+
+  enrichQueryFields(fields, parameters = {}) {
+    const enrichedQuery = {}
+
+    // enrich the fields with dynamic parameters
+    for (let key of Object.keys(fields)) {
+      if (fields[key] == null) {
+        continue
+      }
+      if (typeof fields[key] === "object") {
+        // enrich nested fields object
+        enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
+      } else if (typeof fields[key] === "string") {
+        // enrich string value as normal
+        let value = fields[key]
+        // add triple brace to avoid escaping e.g. '=' in cookie header
+        if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
+          value = `{${value}}`
+        }
+        enrichedQuery[key] = processStringSync(value, parameters, {
+          noHelpers: true,
+        })
+      } else {
+        enrichedQuery[key] = fields[key]
+      }
+    }
+    if (
+      enrichedQuery.json ||
+      enrichedQuery.customData ||
+      enrichedQuery.requestBody
+    ) {
+      try {
+        enrichedQuery.json = JSON.parse(
+          enrichedQuery.json ||
+            enrichedQuery.customData ||
+            enrichedQuery.requestBody
+        )
+      } catch (err) {
+        // no json found, ignore
+      }
+      delete enrichedQuery.customData
+    }
+    return enrichedQuery
+  }
 }
 
 module.exports = (input, callback) => {

package/src/threads/utils.js

@@ -1,16 +1,12 @@
 const env = require("../environment")
 const CouchDB = require("../db")
-const { init } = require("@budibase/auth")
-const redis = require("@budibase/auth/redis")
-const { SEPARATOR } = require("@budibase/auth/db")
-const { processStringSync } = require("@budibase/string-templates")
+const { init } = require("@budibase/backend-core")
+const redis = require("@budibase/backend-core/redis")
+const { SEPARATOR } = require("@budibase/backend-core/db")
 
 const VARIABLE_TTL_SECONDS = 3600
 let client
 
-const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
-const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
-
 async function getClient() {
   if (!client) {
     client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
@@ -80,49 +76,3 @@ exports.hasExtraData = response => {
     response.info != null
   )
 }
-
-exports.enrichQueryFields = (fields, parameters = {}) => {
-  const enrichedQuery = {}
-
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      let value = fields[key]
-      // add triple brace to avoid escaping e.g. '=' in cookie header
-      if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
-        value = `{${value}}`
-      }
-      enrichedQuery[key] = processStringSync(value, parameters, {
-        noHelpers: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-
-  return enrichedQuery
-}

package/src/utilities/budibaseDir.js

@@ -1,7 +1,7 @@
 const { join } = require("./centralPath")
 const { homedir } = require("os")
 const env = require("../environment")
-const { budibaseTempDir } = require("@budibase/auth").objectStore
+const { budibaseTempDir } = require("@budibase/backend-core/objectStore")
 
 module.exports.budibaseAppsDir = function () {
   return env.BUDIBASE_DIR || join(homedir(), ".budibase")

package/src/utilities/fileSystem/utilities.js

@@ -9,7 +9,7 @@ const {
   deleteFolder,
   uploadDirectory,
   downloadTarball,
-} = require("@budibase/auth").objectStore
+} = require("@budibase/backend-core/objectStore")
 
 /***********************************
  *              NOTE               *

package/src/utilities/global.js

@@ -2,11 +2,14 @@ const {
   getMultiIDParams,
   getGlobalIDFromUserMetadataID,
 } = require("../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { getDeployedAppID } = require("@budibase/auth/db")
-const { getGlobalUserParams } = require("@budibase/auth/db")
-const { user: userCache } = require("@budibase/auth/cache")
-const { getGlobalDB, isUserInAppTenant } = require("@budibase/auth/tenancy")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
+const { getGlobalUserParams } = require("@budibase/backend-core/db")
+const { user: userCache } = require("@budibase/backend-core/cache")
+const {
+  getGlobalDB,
+  isUserInAppTenant,
+} = require("@budibase/backend-core/tenancy")
 const env = require("../environment")
 
 exports.updateAppRole = (appId, user) => {

package/src/utilities/index.js

@@ -1,6 +1,6 @@
 const env = require("../environment")
 const { OBJ_STORE_DIRECTORY } = require("../constants")
-const { sanitizeKey } = require("@budibase/auth/src/objectStore")
+const { sanitizeKey } = require("@budibase/backend-core/objectStore")
 const CouchDB = require("../db")
 const { generateMetadataID } = require("../db/utils")
 const Readable = require("stream").Readable
@@ -34,7 +34,7 @@ exports.checkSlashesInUrl = url => {
  * @return {string} The base URL of the object store (MinIO or S3).
  */
 exports.objectStoreUrl = () => {
-  if (env.SELF_HOSTED) {
+  if (env.SELF_HOSTED || env.MINIO_URL) {
     // can use a relative url for this as all goes through the proxy (this is hosted in minio)
     return OBJ_STORE_DIRECTORY
   } else {

package/src/utilities/redis.js

@@ -1,4 +1,4 @@
-const { Client, utils } = require("@budibase/auth/redis")
+const { Client, utils } = require("@budibase/backend-core/redis")
 const { getGlobalIDFromUserMetadataID } = require("../db/utils")
 
 const APP_DEV_LOCK_SECONDS = 600

package/src/utilities/rowProcessor/index.js

@@ -5,7 +5,11 @@ const { attachmentsRelativeURL } = require("../index")
 const { processFormulas } = require("./utils")
 const { deleteFiles } = require("../../utilities/fileSystem/utilities")
 const { ObjectStoreBuckets } = require("../../constants")
-const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db")
+const {
+  isProdAppID,
+  getDeployedAppID,
+  dbExists,
+} = require("@budibase/backend-core/db")
 const CouchDB = require("../../db")
 
 const BASE_AUTO_ID = 1

package/src/utilities/security.js

@@ -3,8 +3,11 @@ const {
   PermissionTypes,
   getBuiltinPermissionByID,
   isPermissionLevelHigherThanRead,
-} = require("@budibase/auth/permissions")
-const { lowerBuiltinRoleID, getBuiltinRoles } = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/permissions")
+const {
+  lowerBuiltinRoleID,
+  getBuiltinRoles,
+} = require("@budibase/backend-core/roles")
 const { DocumentTypes } = require("../db/utils")
 
 const CURRENTLY_SUPPORTED_LEVELS = [

package/src/utilities/usageQuota.js

@@ -1,9 +1,9 @@
 const env = require("../environment")
-const { getGlobalDB } = require("@budibase/auth/tenancy")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const {
   StaticDatabases,
   generateNewUsageQuotaDoc,
-} = require("@budibase/auth/db")
+} = require("@budibase/backend-core/db")
 
 function getNewQuotaReset() {
   return Date.now() + 2592000000

package/src/utilities/workerRequests.js

@@ -1,10 +1,10 @@
 const fetch = require("node-fetch")
 const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
-const { getDeployedAppID } = require("@budibase/auth/db")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 const { updateAppRole } = require("./global")
-const { Headers } = require("@budibase/auth/constants")
-const { getTenantId, isTenantIdSet } = require("@budibase/auth/tenancy")
+const { Headers } = require("@budibase/backend-core/constants")
+const { getTenantId, isTenantIdSet } = require("@budibase/backend-core/tenancy")
 
 function request(ctx, request) {
   if (!request.headers) {