@budibase/server 1.0.113 → 1.0.116

package/src/tests/utilities/TestConfiguration.js

@@ -1,6 +1,4 @@
-const core = require("@budibase/backend-core")
-const CouchDB = require("../../db")
-core.init(CouchDB)
+require("../../db").init()
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const env = require("../../environment")
 const {
@@ -20,7 +18,7 @@ const supertest = require("supertest")
 const { cleanup } = require("../../utilities/fileSystem")
 const { Cookies, Headers } = require("@budibase/backend-core/constants")
 const { jwt } = require("@budibase/backend-core/auth")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { doInTenant, doWithGlobalDB } = require("@budibase/backend-core/tenancy")
 const { createASession } = require("@budibase/backend-core/sessions")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const newid = require("../../db/newid")
@@ -57,8 +55,20 @@ class TestConfiguration {
     return this.appId
   }
 
-  getCouch() {
-    return CouchDB
+  async doInContext(appId, task) {
+    if (!appId) {
+      appId = this.appId
+    }
+    return doInTenant(TENANT_ID, () => {
+      // check if already in a context
+      if (context.getAppId() == null && appId !== null) {
+        return context.doInAppContext(appId, async () => {
+          return task()
+        })
+      } else {
+        return task()
+      }
+    })
   }
 
   async _req(config, params, controlFunc) {
@@ -72,35 +82,28 @@ class TestConfiguration {
     request.request = {
       body: config,
     }
-    async function run() {
+    return this.doInContext(this.appId, async () => {
       if (params) {
         request.params = params
       }
       await controlFunc(request)
       return request.body
-    }
-    // check if already in a context
-    if (context.getAppId() == null && this.appId !== null) {
-      return context.doInAppContext(this.appId, async () => {
-        return run()
-      })
-    } else {
-      return run()
-    }
+    })
   }
 
   async generateApiKey(userId = GLOBAL_USER_ID) {
-    const db = getGlobalDB(TENANT_ID)
-    const id = generateDevInfoID(userId)
-    let devInfo
-    try {
-      devInfo = await db.get(id)
-    } catch (err) {
-      devInfo = { _id: id, userId }
-    }
-    devInfo.apiKey = encrypt(`${TENANT_ID}${SEPARATOR}${newid()}`)
-    await db.put(devInfo)
-    return devInfo.apiKey
+    return doWithGlobalDB(TENANT_ID, async db => {
+      const id = generateDevInfoID(userId)
+      let devInfo
+      try {
+        devInfo = await db.get(id)
+      } catch (err) {
+        devInfo = { _id: id, userId }
+      }
+      devInfo.apiKey = encrypt(`${TENANT_ID}${SEPARATOR}${newid()}`)
+      await db.put(devInfo)
+      return devInfo.apiKey
+    })
   }
 
   async globalUser({
@@ -109,34 +112,35 @@ class TestConfiguration {
     email = EMAIL,
     roles,
   } = {}) {
-    const db = getGlobalDB(TENANT_ID)
-    let existing
-    try {
-      existing = await db.get(id)
-    } catch (err) {
-      existing = { email }
-    }
-    const user = {
-      _id: id,
-      ...existing,
-      roles: roles || {},
-      tenantId: TENANT_ID,
-    }
-    await createASession(id, {
-      sessionId: "sessionid",
-      tenantId: TENANT_ID,
-      csrfToken: CSRF_TOKEN,
+    return doWithGlobalDB(TENANT_ID, async db => {
+      let existing
+      try {
+        existing = await db.get(id)
+      } catch (err) {
+        existing = { email }
+      }
+      const user = {
+        _id: id,
+        ...existing,
+        roles: roles || {},
+        tenantId: TENANT_ID,
+      }
+      await createASession(id, {
+        sessionId: "sessionid",
+        tenantId: TENANT_ID,
+        csrfToken: CSRF_TOKEN,
+      })
+      if (builder) {
+        user.builder = { global: true }
+      } else {
+        user.builder = { global: false }
+      }
+      const resp = await db.put(user)
+      return {
+        _rev: resp._rev,
+        ...user,
+      }
     })
-    if (builder) {
-      user.builder = { global: true }
-    } else {
-      user.builder = { global: false }
-    }
-    const resp = await db.put(user)
-    return {
-      _rev: resp._rev,
-      ...user,
-    }
   }
 
   // use a new id as the name to avoid name collisions
@@ -205,9 +209,12 @@ class TestConfiguration {
 
   async createApp(appName) {
     // create dev app
+    // clear any old app
+    this.appId = null
+    await context.updateAppId(null)
     this.app = await this._req({ name: appName }, null, controllers.app.create)
     this.appId = this.app.appId
-    context.updateAppId(this.appId)
+    await context.updateAppId(this.appId)
 
     // create production app
     this.prodApp = await this.deploy()

package/src/threads/index.js

@@ -36,6 +36,7 @@ class Thread {
         maxConcurrentWorkers: this.count,
       }
       if (opts.timeoutMs) {
+        this.timeoutMs = opts.timeoutMs
         workerOpts.maxCallTime = opts.timeoutMs
       }
       this.workers = workerFarm(workerOpts, typeToFile(type))
@@ -43,6 +44,7 @@ class Thread {
   }
 
   run(data) {
+    const timeoutMs = this.timeoutMs
     return new Promise((resolve, reject) => {
       let fncToCall
       // if in test then don't use threading
@@ -52,7 +54,11 @@ class Thread {
         fncToCall = this.workers
       }
       fncToCall(data, (err, response) => {
-        if (err) {
+        if (err && err.type === "TimeoutError") {
+          reject(
+            new Error(`Query response time exceeded ${timeoutMs}ms timeout.`)
+          )
+        } else if (err) {
           reject(err)
         } else {
           resolve(response)

package/src/threads/query.js

@@ -2,14 +2,13 @@ const threadUtils = require("./utils")
 threadUtils.threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
-const {
-  processStringSync,
-  findHBSBlocks,
-} = require("@budibase/string-templates")
+const { processStringSync } = require("@budibase/string-templates")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const { isSQL } = require("../integrations/utils")
-
-const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
+const {
+  enrichQueryFields,
+  interpolateSQL,
+} = require("../integrations/queries/sql")
 
 class QueryRunner {
   constructor(input, flags = { noRecursiveQuery: false }) {
@@ -29,76 +28,6 @@ class QueryRunner {
     this.hasRerun = false
   }
 
-  interpolateSQL(fields, parameters, integration) {
-    let sql = fields.sql
-    if (!sql) {
-      return fields
-    }
-    const bindings = findHBSBlocks(sql)
-    let variables = [],
-      arrays = []
-    for (let binding of bindings) {
-      // look for array/list operations in the SQL statement, which will need handled later
-      const listRegexMatch = sql.match(
-        new RegExp(`(in|IN|In|iN)( )+[(]?${binding}[)]?`)
-      )
-      // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}'
-      // start by finding all the instances of const character strings
-      const charConstMatch = sql.match(CONST_CHAR_REGEX) || []
-      // now look within them to see if a binding is used
-      const charConstBindingMatch = charConstMatch.find(string =>
-        string.match(new RegExp(`'[^']*${binding}[^']*'`))
-      )
-      if (charConstBindingMatch) {
-        let [part1, part2] = charConstBindingMatch.split(binding)
-        part1 = `'${part1.substring(1)}'`
-        part2 = `'${part2.substring(0, part2.length - 1)}'`
-        sql = sql.replace(
-          charConstBindingMatch,
-          integration.getStringConcat([
-            part1,
-            integration.getBindingIdentifier(),
-            part2,
-          ])
-        )
-      }
-      // generate SQL parameterised array
-      else if (listRegexMatch) {
-        arrays.push(binding)
-        // determine the length of the array
-        const value = this.enrichQueryFields([binding], parameters)[0].split(
-          ","
-        )
-        // build a string like ($1, $2, $3)
-        let replacement = `${Array.apply(null, Array(value.length))
-          .map(() => integration.getBindingIdentifier())
-          .join(",")}`
-        // check if parentheses are needed
-        if (!listRegexMatch[0].includes(`(${binding})`)) {
-          replacement = `(${replacement})`
-        }
-        sql = sql.replace(binding, replacement)
-      } else {
-        sql = sql.replace(binding, integration.getBindingIdentifier())
-      }
-      variables.push(binding)
-    }
-    // replicate the knex structure
-    fields.sql = sql
-    fields.bindings = this.enrichQueryFields(variables, parameters)
-    // check for arrays in the data
-    let updated = []
-    for (let i = 0; i < variables.length; i++) {
-      if (arrays.includes(variables[i])) {
-        updated = updated.concat(fields.bindings[i].split(","))
-      } else {
-        updated.push(fields.bindings[i])
-      }
-    }
-    fields.bindings = updated
-    return fields
-  }
-
   async execute() {
     let { datasource, fields, queryVerb, transformer } = this
     const Integration = integrations[datasource.source]
@@ -112,9 +41,9 @@ class QueryRunner {
     let query
     // handle SQL injections by interpolating the variables
     if (isSQL(datasource)) {
-      query = this.interpolateSQL(fields, parameters, integration)
+      query = interpolateSQL(fields, parameters, integration)
     } else {
-      query = this.enrichQueryFields(fields, parameters)
+      query = enrichQueryFields(fields, parameters)
     }
 
     // Add pagination values for REST queries
@@ -259,47 +188,6 @@ class QueryRunner {
     }
     return parameters
   }
-
-  enrichQueryFields(fields, parameters = {}) {
-    const enrichedQuery = Array.isArray(fields) ? [] : {}
-
-    // enrich the fields with dynamic parameters
-    for (let key of Object.keys(fields)) {
-      if (fields[key] == null) {
-        continue
-      }
-      if (typeof fields[key] === "object") {
-        // enrich nested fields object
-        enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
-      } else if (typeof fields[key] === "string") {
-        // enrich string value as normal
-        enrichedQuery[key] = processStringSync(fields[key], parameters, {
-          noEscaping: true,
-          noHelpers: true,
-          escapeNewlines: true,
-        })
-      } else {
-        enrichedQuery[key] = fields[key]
-      }
-    }
-    if (
-      enrichedQuery.json ||
-      enrichedQuery.customData ||
-      enrichedQuery.requestBody
-    ) {
-      try {
-        enrichedQuery.json = JSON.parse(
-          enrichedQuery.json ||
-            enrichedQuery.customData ||
-            enrichedQuery.requestBody
-        )
-      } catch (err) {
-        // no json found, ignore
-      }
-      delete enrichedQuery.customData
-    }
-    return enrichedQuery
-  }
 }
 
 module.exports = (input, callback) => {

package/src/threads/utils.js

@@ -1,6 +1,5 @@
 const env = require("../environment")
-const CouchDB = require("../db")
-const { init } = require("@budibase/backend-core")
+const db = require("../db")
 const redis = require("@budibase/backend-core/redis")
 const { SEPARATOR } = require("@budibase/backend-core/db")
 
@@ -25,7 +24,7 @@ exports.threadSetup = () => {
   }
   // when thread starts, make sure it is recorded
   env.setInThread()
-  init(CouchDB)
+  db.init()
 }
 
 function makeVariableKey(queryId, variable) {

package/src/utilities/fileSystem/index.js

@@ -2,7 +2,7 @@ const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
 const uuid = require("uuid/v4")
-const CouchDB = require("../../db")
+const { doWithDB } = require("@budibase/backend-core/db")
 const { ObjectStoreBuckets } = require("../../constants")
 const {
   upload,
@@ -151,41 +151,41 @@ exports.streamBackup = async appId => {
  * @return {*} either a readable stream or a string
  */
 exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => {
-  const instanceDb = new CouchDB(dbName)
+  return doWithDB(dbName, async db => {
+    // Stream the dump if required
+    if (stream) {
+      const memStream = new MemoryStream()
+      db.dump(memStream, { filter })
+      return memStream
+    }
 
-  // Stream the dump if required
-  if (stream) {
-    const memStream = new MemoryStream()
-    instanceDb.dump(memStream, { filter })
-    return memStream
-  }
+    // Write the dump to file if required
+    if (exportName) {
+      const path = join(budibaseTempDir(), exportName)
+      const writeStream = fs.createWriteStream(path)
+      await db.dump(writeStream, { filter })
 
-  // Write the dump to file if required
-  if (exportName) {
-    const path = join(budibaseTempDir(), exportName)
-    const writeStream = fs.createWriteStream(path)
-    await instanceDb.dump(writeStream, { filter })
+      // Upload the dump to the object store if self hosted
+      if (env.SELF_HOSTED) {
+        await streamUpload(
+          ObjectStoreBuckets.BACKUPS,
+          join(dbName, exportName),
+          fs.createReadStream(path)
+        )
+      }
 
-    // Upload the dump to the object store if self hosted
-    if (env.SELF_HOSTED) {
-      await streamUpload(
-        ObjectStoreBuckets.BACKUPS,
-        join(dbName, exportName),
-        fs.createReadStream(path)
-      )
+      return fs.createReadStream(path)
     }
 
-    return fs.createReadStream(path)
-  }
-
-  // Stringify the dump in memory if required
-  const memStream = new MemoryStream()
-  let appString = ""
-  memStream.on("data", chunk => {
-    appString += chunk.toString()
+    // Stringify the dump in memory if required
+    const memStream = new MemoryStream()
+    let appString = ""
+    memStream.on("data", chunk => {
+      appString += chunk.toString()
+    })
+    await db.dump(memStream, { filter })
+    return appString
   })
-  await instanceDb.dump(memStream, { filter })
-  return appString
 }
 
 /**

package/src/utilities/usageQuota/rows.js

@@ -1,6 +1,9 @@
 const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
-const CouchDB = require("../../db")
-const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
+const {
+  isDevAppID,
+  getDevelopmentAppID,
+  doWithDB,
+} = require("@budibase/backend-core/db")
 
 const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
 
@@ -24,22 +27,23 @@ const getAppPairs = appIds => {
 
 const getAppRows = async appId => {
   // need to specify the app ID, as this is used for different apps in one call
-  const appDb = new CouchDB(appId)
-  const response = await appDb.allDocs(
-    getRowParams(null, null, {
-      include_docs: false,
-    })
-  )
-  return response.rows
-    .map(r => r.id)
-    .filter(id => {
-      for (let exclusion of ROW_EXCLUSIONS) {
-        if (id.startsWith(exclusion)) {
-          return false
+  return doWithDB(appId, async db => {
+    const response = await db.allDocs(
+      getRowParams(null, null, {
+        include_docs: false,
+      })
+    )
+    return response.rows
+      .map(r => r.id)
+      .filter(id => {
+        for (let exclusion of ROW_EXCLUSIONS) {
+          if (id.startsWith(exclusion)) {
+            return false
+          }
         }
-      }
-      return true
-    })
+        return true
+      })
+  })
 }
 
 /**
@@ -58,7 +62,7 @@ exports.getUniqueRows = async appIds => {
         continue
       }
       try {
-        appRows.push(await getAppRows(appId))
+        appRows = appRows.concat(await getAppRows(appId))
       } catch (e) {
         console.error(e)
         // don't error out if we can't count the app rows, just continue

package/coverage/lcov-report/db/client.js.html

@@ -1,173 +0,0 @@
-
-<!doctype html>
-<html lang="en">
-
-<head>
-    <title>Code coverage report for db/client.js</title>
-    <meta charset="utf-8" />
-    <link rel="stylesheet" href="../prettify.css" />
-    <link rel="stylesheet" href="../base.css" />
-    <link rel="shortcut icon" type="image/x-icon" href="../favicon.png" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <style type='text/css'>
-        .coverage-summary .sorter {
-            background-image: url(../sort-arrow-sprite.png);
-        }
-    </style>
-</head>
-    
-<body>
-<div class='wrapper'>
-    <div class='pad1'>
-        <h1><a href="../index.html">All files</a> / <a href="index.html">db</a> client.js</h1>
-        <div class='clearfix'>
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Statements</span>
-                <span class='fraction'>17/17</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">75% </span>
-                <span class="quiet">Branches</span>
-                <span class='fraction'>3/4</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Functions</span>
-                <span class='fraction'>0/0</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Lines</span>
-                <span class='fraction'>17/17</span>
-            </div>
-        
-            
-        </div>
-        <p class="quiet">
-            Press <em>n</em> or <em>j</em> to go to the next uncovered block, <em>b</em>, <em>p</em> or <em>k</em> for the previous block.
-        </p>
-    </div>
-    <div class='status-line high'></div>
-    <pre><table class="coverage">
-<tr><td class="line-count quiet"><a name='L1'></a><a href='#L1'>1</a>
-<a name='L2'></a><a href='#L2'>2</a>
-<a name='L3'></a><a href='#L3'>3</a>
-<a name='L4'></a><a href='#L4'>4</a>
-<a name='L5'></a><a href='#L5'>5</a>
-<a name='L6'></a><a href='#L6'>6</a>
-<a name='L7'></a><a href='#L7'>7</a>
-<a name='L8'></a><a href='#L8'>8</a>
-<a name='L9'></a><a href='#L9'>9</a>
-<a name='L10'></a><a href='#L10'>10</a>
-<a name='L11'></a><a href='#L11'>11</a>
-<a name='L12'></a><a href='#L12'>12</a>
-<a name='L13'></a><a href='#L13'>13</a>
-<a name='L14'></a><a href='#L14'>14</a>
-<a name='L15'></a><a href='#L15'>15</a>
-<a name='L16'></a><a href='#L16'>16</a>
-<a name='L17'></a><a href='#L17'>17</a>
-<a name='L18'></a><a href='#L18'>18</a>
-<a name='L19'></a><a href='#L19'>19</a>
-<a name='L20'></a><a href='#L20'>20</a>
-<a name='L21'></a><a href='#L21'>21</a>
-<a name='L22'></a><a href='#L22'>22</a>
-<a name='L23'></a><a href='#L23'>23</a>
-<a name='L24'></a><a href='#L24'>24</a>
-<a name='L25'></a><a href='#L25'>25</a>
-<a name='L26'></a><a href='#L26'>26</a>
-<a name='L27'></a><a href='#L27'>27</a>
-<a name='L28'></a><a href='#L28'>28</a>
-<a name='L29'></a><a href='#L29'>29</a>
-<a name='L30'></a><a href='#L30'>30</a>
-<a name='L31'></a><a href='#L31'>31</a>
-<a name='L32'></a><a href='#L32'>32</a></td><td class="line-coverage quiet"><span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-neutral">&nbsp;</span></td><td class="text"><pre class="prettyprint lang-js">const PouchDB = require("pouchdb")
-const { getCouchUrl } = require("@budibase/backend-core/db")
-const replicationStream = require("pouchdb-replication-stream")
-const allDbs = require("pouchdb-all-dbs")
-const find = require("pouchdb-find")
-const env = require("../environment")
-&nbsp;
-const COUCH_DB_URL = getCouchUrl() || "http://localhost:4005"
-&nbsp;
-PouchDB.plugin(replicationStream.plugin)
-PouchDB.plugin(find)
-PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
-&nbsp;
-let POUCH_DB_DEFAULTS = {
-  prefix: COUCH_DB_URL,
-}
-&nbsp;
-if (env.isTest()) {
-  PouchDB.plugin(require("pouchdb-adapter-memory"))
-  POUCH_DB_DEFAULTS = {
-    prefix: undefined,
-    adapter: "memory",
-  }
-}
-&nbsp;
-const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS)
-&nbsp;
-// have to still have pouch alldbs for testing
-allDbs(Pouch)
-&nbsp;
-module.exports = Pouch
-&nbsp;</pre></td></tr></table></pre>
-
-                <div class='push'></div><!-- for sticky footer -->
-            </div><!-- /wrapper -->
-            <div class='footer quiet pad2 space-top1 center small'>
-                Code coverage generated by
-                <a href="https://istanbul.js.org/" target="_blank">istanbul</a>
-                at Wed Apr 13 2022 12:20:55 GMT+0000 (Coordinated Universal Time)
-            </div>
-        </div>
-        <script src="../prettify.js"></script>
-        <script>
-            window.onload = function () {
-                prettyPrint();
-            };
-        </script>
-        <script src="../sorter.js"></script>
-        <script src="../block-navigation.js"></script>
-    </body>
-</html>
-