@budibase/server 1.0.113 → 1.0.116

package/src/integrations/couchdb.ts

@@ -53,51 +53,55 @@ module CouchDBModule {
 
   class CouchDBIntegration implements IntegrationBase {
     private config: CouchDBConfig
-    private client: any
+    private readonly client: any
 
     constructor(config: CouchDBConfig) {
       this.config = config
       this.client = new PouchDB(`${config.url}/${config.database}`)
     }
 
-    async create(query: { json: object }) {
+    async query(
+      command: string,
+      errorMsg: string,
+      query: { json?: object; id?: string }
+    ) {
       try {
-        return this.client.post(query.json)
+        const response = await this.client[command](query.id || query.json)
+        await this.client.close()
+        return response
       } catch (err) {
-        console.error("Error writing to couchDB", err)
+        console.error(errorMsg, err)
         throw err
       }
     }
 
+    async create(query: { json: object }) {
+      return this.query("post", "Error writing to couchDB", query)
+    }
+
     async read(query: { json: object }) {
-      try {
-        const result = await this.client.allDocs({
+      const result = await this.query("allDocs", "Error querying couchDB", {
+        json: {
           include_docs: true,
           ...query.json,
-        })
-        return result.rows.map((row: { doc: object }) => row.doc)
-      } catch (err) {
-        console.error("Error querying couchDB", err)
-        throw err
-      }
+        },
+      })
+      return result.rows.map((row: { doc: object }) => row.doc)
     }
 
     async update(query: { json: object }) {
-      try {
-        return this.client.put(query.json)
-      } catch (err) {
-        console.error("Error updating couchDB document", err)
-        throw err
-      }
+      return this.query("put", "Error updating couchDB document", query)
     }
 
     async delete(query: { id: string }) {
-      try {
-        return await this.client.remove(query.id)
-      } catch (err) {
-        console.error("Error deleting couchDB document", err)
-        throw err
-      }
+      const doc = await this.query(
+        "get",
+        "Cannot find doc to be deleted",
+        query
+      )
+      return this.query("remove", "Error deleting couchDB document", {
+        json: doc,
+      })
     }
   }
 

package/src/integrations/queries/sql.ts

@@ -0,0 +1,125 @@
+import { findHBSBlocks, processStringSync } from "@budibase/string-templates"
+import { Integration } from "../../definitions/datasource"
+import { DatasourcePlus } from "../base/datasourcePlus"
+
+const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
+
+export function enrichQueryFields(
+  fields: { [key: string]: any },
+  parameters = {}
+) {
+  const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {}
+
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = enrichQueryFields(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: true,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+  return enrichedQuery
+}
+
+export function interpolateSQL(
+  fields: { [key: string]: any },
+  parameters: { [key: string]: any },
+  integration: DatasourcePlus
+) {
+  let sql = fields.sql
+  if (!sql || typeof sql !== "string") {
+    return fields
+  }
+  const bindings = findHBSBlocks(sql)
+  let variables = [],
+    arrays = []
+  for (let binding of bindings) {
+    // look for array/list operations in the SQL statement, which will need handled later
+    const listRegexMatch = sql.match(
+      new RegExp(`(in|IN|In|iN)( )+[(]?${binding}[)]?`)
+    )
+    // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}'
+    // start by finding all the instances of const character strings
+    const charConstMatch = sql.match(CONST_CHAR_REGEX) || []
+    // now look within them to see if a binding is used
+    const charConstBindingMatch = charConstMatch.find((string: any) =>
+      string.match(new RegExp(`'[^']*${binding}[^']*'`))
+    )
+    if (charConstBindingMatch) {
+      let [part1, part2] = charConstBindingMatch.split(binding)
+      part1 = `'${part1.substring(1)}'`
+      part2 = `'${part2.substring(0, part2.length - 1)}'`
+      sql = sql.replace(
+        charConstBindingMatch,
+        integration.getStringConcat([
+          part1,
+          integration.getBindingIdentifier(),
+          part2,
+        ])
+      )
+    }
+    // generate SQL parameterised array
+    else if (listRegexMatch) {
+      arrays.push(binding)
+      // determine the length of the array
+      const value = enrichQueryFields([binding], parameters)[0]
+        .split(",")
+        .map((val: string) => val.trim())
+      // build a string like ($1, $2, $3)
+      let replacement = `${Array.apply(null, Array(value.length))
+        .map(() => integration.getBindingIdentifier())
+        .join(",")}`
+      // check if parentheses are needed
+      if (!listRegexMatch[0].includes(`(${binding})`)) {
+        replacement = `(${replacement})`
+      }
+      sql = sql.replace(binding, replacement)
+    } else {
+      sql = sql.replace(binding, integration.getBindingIdentifier())
+    }
+    variables.push(binding)
+  }
+  // replicate the knex structure
+  fields.sql = sql
+  fields.bindings = enrichQueryFields(variables, parameters)
+  // check for arrays in the data
+  let updated: string[] = []
+  for (let i = 0; i < variables.length; i++) {
+    if (arrays.includes(variables[i])) {
+      updated = updated.concat(
+        fields.bindings[i].split(",").map((val: string) => val.trim())
+      )
+    } else {
+      updated.push(fields.bindings[i])
+    }
+  }
+  fields.bindings = updated
+  return fields
+}

package/src/integrations/tests/airtable.spec.js

@@ -7,7 +7,9 @@ class TestConfiguration {
     this.integration = new AirtableIntegration.integration(config) 
     this.client = {
       create: jest.fn(),
-      select: jest.fn(),
+      select: jest.fn(() => ({
+        firstPage: jest.fn(() => []),
+      })),
       update: jest.fn(),
       destroy: jest.fn(),
     }

package/src/integrations/tests/couchdb.spec.js

@@ -2,8 +2,10 @@ jest.mock("pouchdb", () => function CouchDBMock() {
   this.post = jest.fn()
   this.allDocs = jest.fn(() => ({ rows: [] }))
   this.put = jest.fn()
+  this.get = jest.fn()
   this.remove = jest.fn()
   this.plugin = jest.fn()
+  this.close = jest.fn()
 })
 
 const CouchDBIntegration = require("../couchdb")
@@ -62,6 +64,7 @@ describe("CouchDB Integration", () => {
   it("calls the delete method with the correct params", async () => {
     const id = "1234"
     const response = await config.integration.delete({ id })
-    expect(config.integration.client.remove).toHaveBeenCalledWith(id)
+    expect(config.integration.client.get).toHaveBeenCalledWith(id)
+    expect(config.integration.client.remove).toHaveBeenCalled()
   })
 })

package/src/middleware/builder.js

@@ -5,7 +5,7 @@ const {
   checkDebounce,
   setDebounce,
 } = require("../utilities/redis")
-const { getDB } = require("@budibase/backend-core/db")
+const { doWithDB } = require("@budibase/backend-core/db")
 const { DocumentTypes } = require("../db/utils")
 const { PermissionTypes } = require("@budibase/backend-core/permissions")
 const { app: appCache } = require("@budibase/backend-core/cache")
@@ -48,14 +48,15 @@ async function updateAppUpdatedAt(ctx) {
   if (ctx.method === "GET" || (await checkDebounce(appId))) {
     return
   }
-  const db = getDB(appId)
-  const metadata = await db.get(DocumentTypes.APP_METADATA)
-  metadata.updatedAt = new Date().toISOString()
-  const response = await db.put(metadata)
-  metadata._rev = response.rev
-  await appCache.invalidateAppMetadata(appId, metadata)
-  // set a new debounce record with a short TTL
-  await setDebounce(appId, DEBOUNCE_TIME_SEC)
+  await doWithDB(appId, async db => {
+    const metadata = await db.get(DocumentTypes.APP_METADATA)
+    metadata.updatedAt = new Date().toISOString()
+    const response = await db.put(metadata)
+    metadata._rev = response.rev
+    await appCache.invalidateAppMetadata(appId, metadata)
+    // set a new debounce record with a short TTL
+    await setDebounce(appId, DEBOUNCE_TIME_SEC)
+  })
 }
 
 module.exports = async (ctx, permType) => {

package/src/middleware/tests/authorized.spec.js

@@ -10,7 +10,6 @@ jest.mock("../../environment", () => ({
 const authorizedMiddleware = require("../authorized")
 const env = require("../../environment")
 const { PermissionTypes, PermissionLevels } = require("@budibase/backend-core/permissions")
-require("@budibase/backend-core").init(require("../../db"))
 const { doInAppContext } = require("@budibase/backend-core/context")
 
 const APP_ID = ""

package/src/middleware/tests/usageQuota.spec.js

@@ -6,7 +6,10 @@ jest.mock("@budibase/backend-core/tenancy", () => ({
 
 const usageQuotaMiddleware = require("../usageQuota")
 const usageQuota = require("../../utilities/usageQuota")
-const CouchDB = require("../../db")
+
+jest.mock("@budibase/backend-core/db")
+const { dangerousGetDB } = require("@budibase/backend-core/db")
+
 const env = require("../../environment")
 
 class TestConfiguration {
@@ -80,7 +83,7 @@ describe("usageQuota middleware", () => {
       _rev: "test",
     })
 
-    CouchDB.mockImplementationOnce(() => ({ 
+    dangerousGetDB.mockImplementationOnce(() => ({
       get: async () => true
     }))
 
@@ -96,7 +99,7 @@ describe("usageQuota middleware", () => {
     })
     config.setProd(true)
 
-    CouchDB.mockImplementationOnce(() => ({ 
+    dangerousGetDB.mockImplementationOnce(() => ({
       get: async () => {
         throw new Error()
       } 
@@ -114,21 +117,4 @@ describe("usageQuota middleware", () => {
     expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
     expect(config.next).toHaveBeenCalled()
   })
-
-  // it("calculates the correct file size from a file upload call and adds it to quota", async () => {
-  //   config.setUrl("/upload")
-  //   config.setProd(true)
-  //   config.setFiles([
-  //     {
-  //       size: 100
-  //     },
-  //     {
-  //       size: 10000
-  //     },
-  //   ])
-  //   await config.executeMiddleware()
-
-  //   expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
-  //   expect(config.next).toHaveBeenCalled()
-  // })
 })

package/src/migrations/functions/tests/appUrls.spec.js

@@ -1,12 +1,10 @@
-const { DocumentTypes } = require("@budibase/backend-core/db")
-const env = require("../../../environment")
+const { DocumentTypes, doWithDB } = require("@budibase/backend-core/db")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const migration = require("../appUrls")
 
 describe("run", () => {
   let config = new TestConfig(false)
-  const CouchDB = config.getCouch()
 
   beforeEach(async () => {
     await config.init()
@@ -16,14 +14,13 @@ describe("run", () => {
 
   it("runs successfully", async () => {
     const app = await config.createApp("testApp")
-    const appDb = new CouchDB(app.appId)
-    let metadata = await appDb.get(DocumentTypes.APP_METADATA)
-    delete metadata.url
-    await appDb.put(metadata)
-    
-    await migration.run(appDb)
-
-    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    const metadata = await doWithDB(app.appId, async db => {
+      const metadataDoc = await db.get(DocumentTypes.APP_METADATA)
+      delete metadataDoc.url
+      await db.put(metadataDoc)
+      await migration.run(db)
+      return await db.get(DocumentTypes.APP_METADATA)
+    })
     expect(metadata.url).toEqual("/testapp")
   })
 })

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -1,5 +1,6 @@
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { TENANT_ID } = require("../../../tests/utilities/structures")
+const { getGlobalDB, doInTenant } = require("@budibase/backend-core/tenancy")
 
 // mock email view creation
 const coreDb = require("@budibase/backend-core/db")
@@ -9,17 +10,19 @@ coreDb.createUserEmailView = createUserEmailView
 const migration = require("../userEmailViewCasing")
 
 describe("run", () => {
-  let config = new TestConfig(false)
-  const globalDb = getGlobalDB()
+  doInTenant(TENANT_ID, () => {
+    let config = new TestConfig(false)
+    const globalDb = getGlobalDB()
 
-  beforeEach(async () => {
-    await config.init()
-  })
+    beforeEach(async () => {
+      await config.init()
+    })
 
-  afterAll(config.end)  
+    afterAll(config.end)
 
-  it("runs successfully", async () => {
-    await migration.run(globalDb)
-    expect(createUserEmailView).toHaveBeenCalledTimes(1)
+    it("runs successfully", async () => {
+      await migration.run(globalDb)
+      expect(createUserEmailView).toHaveBeenCalledTimes(1)
+    })
   })
 })

package/src/migrations/functions/usageQuotas/tests/syncApps.spec.js

@@ -1,37 +1,41 @@
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
 const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { TENANT_ID } = require("../../../../tests/utilities/structures")
 const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
 const syncApps = require("../syncApps")
 const env = require("../../../../environment")
 
 describe("syncApps", () => {
+
   let config = new TestConfig(false)
 
-  beforeEach(async () => {
-    await config.init()
-    env._set("USE_QUOTAS", 1)
-  })
+    beforeEach(async () => {
+      await config.init()
+      env._set("USE_QUOTAS", 1)
+    })
 
-  afterAll(config.end)  
+    afterAll(config.end)
 
-  it("runs successfully", async () => {
-    // create the usage quota doc and mock usages
-    const db = getGlobalDB()
-    await getUsageQuotaDoc(db)
-    await update(Properties.APPS, 3)
+    it("runs successfully", async () => {
+      await doInTenant(TENANT_ID, async () => {
+        const db = getGlobalDB()
+        // create the usage quota doc and mock usages
+        await getUsageQuotaDoc(db)
+        await update(Properties.APPS, 3)
 
-    let usageDoc = await getUsageQuotaDoc(db)
-    expect(usageDoc.usageQuota.apps).toEqual(3)
+        let usageDoc = await getUsageQuotaDoc(db)
+        expect(usageDoc.usageQuota.apps).toEqual(3)
 
-    // create an extra app to test the migration
-    await config.createApp("quota-test")
+        // create an extra app to test the migration
+        await config.createApp("quota-test")
 
-    // migrate
-    await syncApps.run()
+        // migrate
+        await syncApps.run()
 
-    // assert the migration worked 
-    usageDoc = await getUsageQuotaDoc(db)
-    expect(usageDoc.usageQuota.apps).toEqual(2)
-  })
+        // assert the migration worked
+        usageDoc = await getUsageQuotaDoc(db)
+        expect(usageDoc.usageQuota.apps).toEqual(2)
+      })
+    })
 })
 

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.js

@@ -1,5 +1,6 @@
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
 const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { TENANT_ID } = require("../../../../tests/utilities/structures")
 const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
 const syncRows = require("../syncRows")
 const env = require("../../../../environment")
@@ -15,29 +16,30 @@ describe("syncRows", () => {
   afterAll(config.end)  
 
   it("runs successfully", async () => {
-    // create the usage quota doc and mock usages
-    const db = getGlobalDB()
-    await getUsageQuotaDoc(db)
-    await update(Properties.ROW, 300)
-
-    let usageDoc = await getUsageQuotaDoc(db)
-    expect(usageDoc.usageQuota.rows).toEqual(300)
-
-    // app 1
-    await config.createTable()
-    await config.createRow()
-    // app 2
-    await config.createApp("second-app")
-    await config.createTable()
-    await config.createRow()
-    await config.createRow()
-    
-    // migrate
-    await syncRows.run()
-
-    // assert the migration worked 
-    usageDoc = await getUsageQuotaDoc(db)
-    expect(usageDoc.usageQuota.rows).toEqual(3)
+    await doInTenant(TENANT_ID, async () => {
+      const db = getGlobalDB()
+      await getUsageQuotaDoc(db)
+      await update(Properties.ROW, 300)
+
+      let usageDoc = await getUsageQuotaDoc(db)
+      expect(usageDoc.usageQuota.rows).toEqual(300)
+
+      // app 1
+      await config.createTable()
+      await config.createRow()
+      // app 2
+      await config.createApp("second-app")
+      await config.createTable()
+      await config.createRow()
+      await config.createRow()
+
+      // migrate
+      await syncRows.run()
+
+      // assert the migration worked
+      usageDoc = await getUsageQuotaDoc(db)
+      expect(usageDoc.usageQuota.rows).toEqual(3)
+    })
   })
 })
 

package/src/migrations/index.ts

@@ -1,4 +1,3 @@
-import CouchDB from "../db"
 const {
   MIGRATION_TYPES,
   runMigrations,
@@ -52,5 +51,5 @@ export const MIGRATIONS: Migration[] = [
 ]
 
 export const migrate = async (options?: MigrationOptions) => {
-  await runMigrations(CouchDB, MIGRATIONS, options)
+  await runMigrations(MIGRATIONS, options)
 }

package/src/module.d.ts

@@ -1,3 +1,11 @@
 declare module "@budibase/backend-core"
 declare module "@budibase/backend-core/tenancy"
 declare module "@budibase/backend-core/db"
+declare module "@budibase/backend-core/context"
+declare module "@budibase/backend-core/cache"
+declare module "@budibase/backend-core/permissions"
+declare module "@budibase/backend-core/roles"
+declare module "@budibase/backend-core/constants"
+declare module "@budibase/backend-core/auth"
+declare module "@budibase/backend-core/sessions"
+declare module "@budibase/backend-core/encryption"