@budibase/server 1.0.80 → 1.0.81-alpha.2

package/src/integrations/postgres.ts

@@ -103,30 +103,11 @@ module PostgresModule {
     },
   }
 
-  async function internalQuery(client: any, query: SqlQuery) {
-    // need to handle a specific issue with json data types in postgres,
-    // new lines inside the JSON data will break it
-    if (query && query.sql) {
-      const matches = query.sql.match(JSON_REGEX)
-      if (matches && matches.length > 0) {
-        for (let match of matches) {
-          const escaped = escapeDangerousCharacters(match)
-          query.sql = query.sql.replace(match, escaped)
-        }
-      }
-    }
-    try {
-      return await client.query(query.sql, query.bindings || [])
-    } catch (err) {
-      // @ts-ignore
-      throw new Error(err)
-    }
-  }
-
   class PostgresIntegration extends Sql implements DatasourcePlus {
     static pool: any
     private readonly client: any
     private readonly config: PostgresConfig
+    private index: number = 1
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 
@@ -163,6 +144,32 @@ module PostgresModule {
       this.setSchema()
     }
 
+    getBindingIdentifier(): string {
+      return `$${this.index++}`
+    }
+
+    async internalQuery(query: SqlQuery) {
+      const client = this.client
+      this.index = 1
+      // need to handle a specific issue with json data types in postgres,
+      // new lines inside the JSON data will break it
+      if (query && query.sql) {
+        const matches = query.sql.match(JSON_REGEX)
+        if (matches && matches.length > 0) {
+          for (let match of matches) {
+            const escaped = escapeDangerousCharacters(match)
+            query.sql = query.sql.replace(match, escaped)
+          }
+        }
+      }
+      try {
+        return await client.query(query.sql, query.bindings || [])
+      } catch (err) {
+        // @ts-ignore
+        throw new Error(err)
+      }
+    }
+
     setSchema() {
       if (!this.config.schema) {
         this.config.schema = "public"
@@ -241,22 +248,22 @@ module PostgresModule {
     }
 
     async create(query: SqlQuery | string) {
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows.length ? response.rows : [{ created: true }]
     }
 
     async read(query: SqlQuery | string) {
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows
     }
 
     async update(query: SqlQuery | string) {
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows.length ? response.rows : [{ updated: true }]
     }
 
     async delete(query: SqlQuery | string) {
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows.length ? response.rows : [{ deleted: true }]
     }
 
@@ -266,11 +273,11 @@ module PostgresModule {
       if (Array.isArray(input)) {
         const responses = []
         for (let query of input) {
-          responses.push(await internalQuery(this.client, query))
+          responses.push(await this.internalQuery(query))
         }
         return responses
       } else {
-        const response = await internalQuery(this.client, input)
+        const response = await this.internalQuery(input)
         return response.rows.length ? response.rows : [{ [operation]: true }]
       }
     }

package/src/integrations/utils.ts

@@ -6,35 +6,63 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 
-const SQL_TYPE_MAP = {
-  text: FieldTypes.LONGFORM,
-  varchar: FieldTypes.STRING,
+const SQL_NUMBER_TYPE_MAP = {
   integer: FieldTypes.NUMBER,
+  int: FieldTypes.NUMBER,
   bigint: FieldTypes.NUMBER,
   decimal: FieldTypes.NUMBER,
   smallint: FieldTypes.NUMBER,
   real: FieldTypes.NUMBER,
-  "double precision": FieldTypes.NUMBER,
-  timestamp: FieldTypes.DATETIME,
-  time: FieldTypes.DATETIME,
-  boolean: FieldTypes.BOOLEAN,
-  json: FieldTypes.JSON,
-  date: FieldTypes.DATETIME,
-  blob: FieldTypes.LONGFORM,
-  enum: FieldTypes.STRING,
   float: FieldTypes.NUMBER,
-  int: FieldTypes.NUMBER,
   numeric: FieldTypes.NUMBER,
   mediumint: FieldTypes.NUMBER,
   dec: FieldTypes.NUMBER,
   double: FieldTypes.NUMBER,
   fixed: FieldTypes.NUMBER,
-  datetime: FieldTypes.DATETIME,
-  tinyint: FieldTypes.BOOLEAN,
-  long: FieldTypes.LONGFORM,
+  "double precision": FieldTypes.NUMBER,
   number: FieldTypes.NUMBER,
   binary_float: FieldTypes.NUMBER,
   binary_double: FieldTypes.NUMBER,
+  money: FieldTypes.NUMBER,
+  smallmoney: FieldTypes.NUMBER,
+}
+
+const SQL_DATE_TYPE_MAP = {
+  timestamp: FieldTypes.DATETIME,
+  time: FieldTypes.DATETIME,
+  datetime: FieldTypes.DATETIME,
+  smalldatetime: FieldTypes.DATETIME,
+  date: FieldTypes.DATETIME,
+}
+
+const SQL_STRING_TYPE_MAP = {
+  varchar: FieldTypes.STRING,
+  char: FieldTypes.STRING,
+  nchar: FieldTypes.STRING,
+  nvarchar: FieldTypes.STRING,
+  ntext: FieldTypes.STRING,
+  enum: FieldTypes.STRING,
+  blob: FieldTypes.LONGFORM,
+  long: FieldTypes.LONGFORM,
+  text: FieldTypes.LONGFORM,
+}
+
+const SQL_BOOLEAN_TYPE_MAP = {
+  boolean: FieldTypes.BOOLEAN,
+  bit: FieldTypes.BOOLEAN,
+  tinyint: FieldTypes.BOOLEAN,
+}
+
+const SQL_MISC_TYPE_MAP = {
+  json: FieldTypes.JSON,
+}
+
+const SQL_TYPE_MAP = {
+  ...SQL_NUMBER_TYPE_MAP,
+  ...SQL_DATE_TYPE_MAP,
+  ...SQL_STRING_TYPE_MAP,
+  ...SQL_BOOLEAN_TYPE_MAP,
+  ...SQL_MISC_TYPE_MAP,
 }
 
 export enum SqlClients {

package/src/tests/utilities/TestConfiguration.js

@@ -25,6 +25,8 @@ const { createASession } = require("@budibase/backend-core/sessions")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const newid = require("../../db/newid")
 const context = require("@budibase/backend-core/context")
+const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
+const { encrypt } = require("@budibase/backend-core/encryption")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -47,6 +49,10 @@ class TestConfiguration {
     return this.request
   }
 
+  getApp() {
+    return this.app
+  }
+
   getAppId() {
     return this.appId
   }
@@ -83,6 +89,20 @@ class TestConfiguration {
     }
   }
 
+  async generateApiKey(userId = GLOBAL_USER_ID) {
+    const db = getGlobalDB(TENANT_ID)
+    const id = generateDevInfoID(userId)
+    let devInfo
+    try {
+      devInfo = await db.get(id)
+    } catch (err) {
+      devInfo = { _id: id, userId }
+    }
+    devInfo.apiKey = encrypt(`${TENANT_ID}${SEPARATOR}${newid()}`)
+    await db.put(devInfo)
+    return devInfo.apiKey
+  }
+
   async globalUser({
     id = GLOBAL_USER_ID,
     builder = true,
@@ -135,7 +155,7 @@ class TestConfiguration {
     cleanup(this.allApps.map(app => app.appId))
   }
 
-  defaultHeaders() {
+  defaultHeaders(extras = {}) {
     const auth = {
       userId: GLOBAL_USER_ID,
       sessionId: "sessionid",
@@ -154,6 +174,7 @@ class TestConfiguration {
         `${Cookies.CurrentApp}=${appToken}`,
       ],
       [Headers.CSRF_TOKEN]: CSRF_TOKEN,
+      ...extras,
     }
     if (this.appId) {
       headers[Headers.APP_ID] = this.appId
@@ -226,7 +247,7 @@ class TestConfiguration {
 
   async getTable(tableId = null) {
     tableId = tableId || this.table._id
-    return this._req(null, { id: tableId }, controllers.table.find)
+    return this._req(null, { tableId }, controllers.table.find)
   }
 
   async createLinkedTable(relationshipType = null, links = ["link"]) {

package/src/threads/query.js

@@ -2,8 +2,12 @@ const threadUtils = require("./utils")
 threadUtils.threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
-const { processStringSync } = require("@budibase/string-templates")
+const {
+  processStringSync,
+  findHBSBlocks,
+} = require("@budibase/string-templates")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
+const { isSQL } = require("../integrations/utils")
 
 class QueryRunner {
   constructor(input, flags = { noRecursiveQuery: false }) {
@@ -23,23 +27,47 @@ class QueryRunner {
     this.hasRerun = false
   }
 
+  interpolateSQL(fields, parameters, integration) {
+    let sql = fields.sql
+    if (!sql) {
+      return fields
+    }
+    const bindings = findHBSBlocks(sql)
+    let variables = []
+    for (let binding of bindings) {
+      let variable = integration.getBindingIdentifier()
+      variables.push(binding)
+      sql = sql.replace(binding, variable)
+    }
+    // replicate the knex structure
+    fields.sql = sql
+    fields.bindings = this.enrichQueryFields(variables, parameters)
+    return fields
+  }
+
   async execute() {
     let { datasource, fields, queryVerb, transformer } = this
+    const Integration = integrations[datasource.source]
+    if (!Integration) {
+      throw "Integration type does not exist."
+    }
+    const integration = new Integration(datasource.config)
+
     // pre-query, make sure datasource variables are added to parameters
     const parameters = await this.addDatasourceVariables()
-    let query = this.enrichQueryFields(fields, parameters)
+    let query
+    // handle SQL injections by interpolating the variables
+    if (isSQL(datasource)) {
+      query = this.interpolateSQL(fields, parameters, integration)
+    } else {
+      query = this.enrichQueryFields(fields, parameters)
+    }
 
     // Add pagination values for REST queries
     if (this.pagination) {
       query.paginationValues = this.pagination
     }
 
-    const Integration = integrations[datasource.source]
-    if (!Integration) {
-      throw "Integration type does not exist."
-    }
-    const integration = new Integration(datasource.config)
-
     let output = threadUtils.formatResponse(await integration[queryVerb](query))
     let rows = output,
       info = undefined,
@@ -179,7 +207,7 @@ class QueryRunner {
   }
 
   enrichQueryFields(fields, parameters = {}) {
-    const enrichedQuery = {}
+    const enrichedQuery = Array.isArray(fields) ? [] : {}
 
     // enrich the fields with dynamic parameters
     for (let key of Object.keys(fields)) {

package/src/utilities/workerRequests.js

@@ -26,11 +26,31 @@ function request(ctx, request) {
     delete request.body
   }
   if (ctx && ctx.headers) {
-    request.headers.cookie = ctx.headers.cookie
+    request.headers = ctx.headers
   }
   return request
 }
 
+async function checkResponse(response, errorMsg, { ctx } = {}) {
+  if (response.status !== 200) {
+    let error
+    try {
+      error = await response.json()
+    } catch (err) {
+      error = await response.text()
+    }
+    const msg = `Unable to ${errorMsg} - ${
+      error.message ? error.message : error
+    }`
+    if (ctx) {
+      ctx.throw(400, msg)
+    } else {
+      throw msg
+    }
+  }
+  return response.json()
+}
+
 exports.request = request
 
 // have to pass in the tenant ID as this could be coming from an automation
@@ -50,25 +70,17 @@ exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
       },
     })
   )
-
-  if (response.status !== 200) {
-    const error = await response.text()
-    throw `Unable to send email - ${error}`
-  }
-  return response.json()
+  return checkResponse(response, "send email")
 }
 
 exports.getGlobalSelf = async (ctx, appId = null) => {
-  const endpoint = `/api/global/users/self`
+  const endpoint = `/api/global/self`
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + endpoint),
     // we don't want to use API key when getting self
     request(ctx, { method: "GET" })
   )
-  if (response.status !== 200) {
-    ctx.throw(400, "Unable to get self globally.")
-  }
-  let json = await response.json()
+  let json = await checkResponse(response, "get self globally", { ctx })
   if (appId) {
     json = updateAppRole(json)
   }
@@ -83,8 +95,45 @@ exports.removeAppFromUserRoles = async (ctx, appId) => {
       method: "DELETE",
     })
   )
-  if (response.status !== 200) {
-    throw "Unable to remove app role"
-  }
-  return response.json()
+  return checkResponse(response, "remove app role")
+}
+
+exports.allGlobalUsers = async ctx => {
+  const response = await fetch(
+    checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
+    // we don't want to use API key when getting self
+    request(ctx, { method: "GET" })
+  )
+  return checkResponse(response, "get users", { ctx })
+}
+
+exports.saveGlobalUser = async ctx => {
+  const response = await fetch(
+    checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
+    // we don't want to use API key when getting self
+    request(ctx, { method: "POST", body: ctx.request.body })
+  )
+  return checkResponse(response, "save user", { ctx })
+}
+
+exports.deleteGlobalUser = async ctx => {
+  const response = await fetch(
+    checkSlashesInUrl(
+      env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
+    ),
+    // we don't want to use API key when getting self
+    request(ctx, { method: "DELETE" })
+  )
+  return checkResponse(response, "delete user", { ctx, body: ctx.request.body })
+}
+
+exports.readGlobalUser = async ctx => {
+  const response = await fetch(
+    checkSlashesInUrl(
+      env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
+    ),
+    // we don't want to use API key when getting self
+    request(ctx, { method: "GET" })
+  )
+  return checkResponse(response, "get user", { ctx })
 }