@budibase/server 1.2.57 → 1.2.58-alpha.2

package/src/integrations/base/sql.ts

@@ -10,6 +10,7 @@ import { QueryOptions } from "../../definitions/datasource"
 import { isIsoDateString, SqlClient } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import environment from "../../environment"
+import { removeKeyNumbering } from "./utils"
 
 const envLimit = environment.SQL_MAX_ROWS
   ? parseInt(environment.SQL_MAX_ROWS)
@@ -133,12 +134,13 @@ class InternalBuilder {
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const isRelationshipField = key.includes(".")
+        const updatedKey = removeKeyNumbering(key)
+        const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${key}`, value)
+          fn(`${opts.tableName}.${updatedKey}`, value)
         }
         if (opts.relationship && isRelationshipField) {
-          fn(key, value)
+          fn(updatedKey, value)
         }
       }
     }
@@ -157,6 +159,61 @@ class InternalBuilder {
       }
     }
 
+    const contains = (mode: object, any: boolean = false) => {
+      const fnc = allOr ? "orWhere" : "where"
+      const rawFnc = `${fnc}Raw`
+      const not = mode === filters?.notContains ? "NOT " : ""
+      function stringifyArray(value: Array<any>, quoteStyle = '"'): string {
+        for (let i in value) {
+          if (typeof value[i] === "string") {
+            value[i] = `${quoteStyle}${value[i]}${quoteStyle}`
+          }
+        }
+        return `[${value.join(",")}]`
+      }
+      if (this.client === SqlClient.POSTGRES) {
+        iterate(mode, (key: string, value: Array<any>) => {
+          const wrap = any ? "" : "'"
+          const containsOp = any ? "\\?| array" : "@>"
+          const fieldNames = key.split(/\./g)
+          const tableName = fieldNames[0]
+          const columnName = fieldNames[1]
+          // @ts-ignore
+          query = query[rawFnc](
+            `${not}"${tableName}"."${columnName}"::jsonb ${containsOp} ${wrap}${stringifyArray(
+              value,
+              any ? "'" : '"'
+            )}${wrap}`
+          )
+        })
+      } else if (this.client === SqlClient.MY_SQL) {
+        const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
+        iterate(mode, (key: string, value: Array<any>) => {
+          // @ts-ignore
+          query = query[rawFnc](
+            `${not}${jsonFnc}(${key}, '${stringifyArray(value)}')`
+          )
+        })
+      } else {
+        const andOr = mode === filters?.containsAny ? " OR " : " AND "
+        iterate(mode, (key: string, value: Array<any>) => {
+          let statement = ""
+          for (let i in value) {
+            if (typeof value[i] === "string") {
+              value[i] = `%"${value[i]}"%`
+            } else {
+              value[i] = `%${value[i]}%`
+            }
+            statement +=
+              (statement ? andOr : "") +
+              `LOWER(${likeKey(this.client, key)}) LIKE ?`
+          }
+          // @ts-ignore
+          query = query[rawFnc](`${not}(${statement})`, value)
+        })
+      }
+    }
+
     if (!filters) {
       return query
     }
@@ -227,32 +284,13 @@ class InternalBuilder {
       })
     }
     if (filters.contains) {
-      const fnc = allOr ? "orWhere" : "where"
-      const rawFnc = `${fnc}Raw`
-      if (this.client === SqlClient.POSTGRES) {
-        iterate(filters.contains, (key: string, value: any) => {
-          const fieldNames = key.split(/\./g)
-          const tableName = fieldNames[0]
-          const columnName = fieldNames[1]
-          if (typeof value === "string") {
-            value = `"${value}"`
-          }
-          // @ts-ignore
-          query = query[rawFnc](
-            `"${tableName}"."${columnName}"::jsonb @> '[${value}]'`
-          )
-        })
-      } else if (this.client === SqlClient.MY_SQL) {
-        iterate(filters.contains, (key: string, value: any) => {
-          if (typeof value === "string") {
-            value = `"${value}"`
-          }
-          // @ts-ignore
-          query = query[rawFnc](`JSON_CONTAINS(${key}, '${value}')`)
-        })
-      } else {
-        iterate(filters.contains, like)
-      }
+      contains(filters.contains)
+    }
+    if (filters.notContains) {
+      contains(filters.notContains)
+    }
+    if (filters.containsAny) {
+      contains(filters.containsAny, true)
     }
     return query
   }
@@ -582,4 +620,3 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 }
 
 export default SqlQueryBuilder
-module.exports = SqlQueryBuilder

package/src/integrations/base/utils.ts

@@ -1,21 +1,12 @@
-import { QueryJson, Datasource } from "@budibase/types"
+const QUERY_START_REGEX = /\d[0-9]*:/g
 
-module DatasourceUtils {
-  const { integrations } = require("../index")
-
-  export async function makeExternalQuery(
-    datasource: Datasource,
-    json: QueryJson
-  ) {
-    const Integration = integrations[datasource.source]
-    // query is the opinionated function
-    if (Integration.prototype.query) {
-      const integration = new Integration(datasource.config)
-      return integration.query(json)
-    } else {
-      throw "Datasource does not support query."
-    }
+export function removeKeyNumbering(key: any): string {
+  if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
+    const parts = key.split(":")
+    // remove the number
+    parts.shift()
+    return parts.join(":")
+  } else {
+    return key
   }
-
-  module.exports.makeExternalQuery = makeExternalQuery
 }

package/src/integrations/microsoftSqlServer.ts

@@ -16,10 +16,10 @@ import {
   finaliseExternalTables,
   SqlClient,
 } from "./utils"
+import Sql from "./base/sql"
 
 module MSSQLModule {
   const sqlServer = require("mssql")
-  const Sql = require("./base/sql")
   const DEFAULT_SCHEMA = "dbo"
 
   interface MSSQLConfig {
@@ -97,7 +97,8 @@ module MSSQLModule {
   class SqlServerIntegration extends Sql implements DatasourcePlus {
     private readonly config: MSSQLConfig
     private index: number = 0
-    static pool: any
+    private readonly pool: any
+    private client: any
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 

package/src/integrations/mongodb.ts

@@ -92,12 +92,15 @@ module MongoDBModule {
           if (json[field] instanceof Object) {
             json[field] = self.createObjectIds(json[field])
           }
-          if (field === "_id" && typeof json[field] === "string") {
-            const id = json["_id"].match(
+          if (
+            typeof json[field] === "string" &&
+            json[field].toLowerCase().startsWith("objectid")
+          ) {
+            const id = json[field].match(
               /(?<=objectid\(['"]).*(?=['"]\))/gi
             )?.[0]
             if (id) {
-              json["_id"] = ObjectID.createFromHexString(id)
+              json[field] = ObjectID.createFromHexString(id)
             }
           }
         }
@@ -114,10 +117,31 @@ module MongoDBModule {
     }
 
     parseQueryParams(params: string, mode: string) {
-      let queryParams = params.split(/(?<=}),[\n\s]*(?={)/g)
-      let group1 = queryParams[0] ? JSON.parse(queryParams[0]) : {}
-      let group2 = queryParams[1] ? JSON.parse(queryParams[1]) : {}
-      let group3 = queryParams[2] ? JSON.parse(queryParams[2]) : {}
+      let queryParams = []
+      let openCount = 0
+      let inQuotes = false
+      let i = 0
+      let startIndex = 0
+      for (let c of params) {
+        if (c === '"' && i > 0 && params[i - 1] !== "\\") {
+          inQuotes = !inQuotes
+        }
+        if (c === "{" && !inQuotes) {
+          openCount++
+          if (openCount === 1) {
+            startIndex = i
+          }
+        } else if (c === "}" && !inQuotes) {
+          if (openCount === 1) {
+            queryParams.push(JSON.parse(params.substring(startIndex, i + 1)))
+          }
+          openCount--
+        }
+        i++
+      }
+      let group1 = queryParams[0] ?? {}
+      let group2 = queryParams[1] ?? {}
+      let group3 = queryParams[2] ?? {}
       if (mode === "update") {
         return {
           filter: group1,
@@ -176,7 +200,10 @@ module MongoDBModule {
             return await collection.findOne(json)
           }
           case "findOneAndUpdate": {
-            let findAndUpdateJson = json as {
+            if (typeof query.json === "string") {
+              json = this.parseQueryParams(query.json, "update")
+            }
+            let findAndUpdateJson = this.createObjectIds(json) as {
               filter: FilterQuery<any>
               update: UpdateQuery<any>
               options: FindOneAndUpdateOption<any>

package/src/integrations/mysql.ts

@@ -17,10 +17,10 @@ import {
 } from "./utils"
 import dayjs from "dayjs"
 const { NUMBER_REGEX } = require("../utilities")
+import Sql from "./base/sql"
 
 module MySQLModule {
   const mysql = require("mysql2/promise")
-  const Sql = require("./base/sql")
 
   interface MySQLConfig {
     host: string

package/src/integrations/postgres.ts

@@ -14,10 +14,10 @@ import {
   finaliseExternalTables,
   SqlClient,
 } from "./utils"
+import Sql from "./base/sql"
 
 module PostgresModule {
   const { Client, types } = require("pg")
-  const Sql = require("./base/sql")
   const { escapeDangerousCharacters } = require("../utilities")
 
   // Return "date" and "timestamp" types as plain strings.
@@ -117,6 +117,7 @@ module PostgresModule {
     private readonly client: any
     private readonly config: PostgresConfig
     private index: number = 1
+    private open: boolean
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 

package/src/integrations/tests/mongo.spec.js

@@ -102,4 +102,222 @@ describe("MongoDB Integration", () => {
     expect(error).toBeDefined()
     restore()
   })
+
+  it("creates ObjectIds if the field contains a match on ObjectId", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: "ObjectId('ACBD12345678ABCD12345678')",
+          name: "ObjectId('BBBB12345678ABCD12345678')"
+        },
+        update: {
+          _id: "ObjectId('FFFF12345678ABCD12345678')",
+          name: "ObjectId('CCCC12345678ABCD12345678')",
+        },
+        options: {
+          upsert: false,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      name: mongo.ObjectID.createFromHexString("BBBB12345678ABCD12345678"),
+    })
+    expect(args[1]).toEqual({
+      _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
+      name: mongo.ObjectID.createFromHexString("CCCC12345678ABCD12345678"),
+    })
+    expect(args[2]).toEqual({
+      upsert: false
+    })
+  })
+
+  it("creates ObjectIds if the $ operator fields contains a match on ObjectId", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: {
+            $eq: "ObjectId('ACBD12345678ABCD12345678')",
+          }
+        },
+        update: {
+          $set: {
+            _id: "ObjectId('FFFF12345678ABCD12345678')",
+          },
+        },
+        options: {
+          upsert: true,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
+      }
+    })
+    expect(args[2]).toEqual({
+      upsert: true
+    })
+  })
+
+  it("supports findOneAndUpdate", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: {
+            $eq: "ObjectId('ACBD12345678ABCD12345678')",
+          }
+        },
+        update: {
+          $set: {
+            name: "UPDATED",
+            age: 99
+          },
+        },
+        options: {
+          upsert: false,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" },
+    }
+    await config.integration.read(query)
+    expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled()
+    
+    const args = config.integration.client.findOneAndUpdate.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        name: "UPDATED",
+        age: 99
+      }
+    })
+    expect(args[2]).toEqual({
+      upsert: false
+    })
+  })
+
+  it("can parse nested objects with arrays", async () => {
+    const query = {
+      json: `{
+          "_id": {
+            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
+          }
+        },
+        {
+          "$set": {
+            "value": {
+              "data": [
+                { "cid": 1 },
+                { "cid": 2 },
+                { "nested": {
+                  "name": "test"
+                }}
+              ]
+            }
+          }
+        },
+        {
+          "upsert": true
+        }`,
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        value: {
+          data: [
+            { cid: 1 },
+            { cid: 2 },
+            { nested: {
+              name: "test"
+            }}
+          ]
+        },
+      },
+    })
+    expect(args[2]).toEqual({
+      upsert: true
+    })
+  })
+
+  it("ignores braces within strings when parsing nested objects", async () => {
+    const query = {
+      json: `{
+          "_id": {
+            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
+          }
+        },
+        {
+          "$set": {
+            "value": {
+              "data": [
+                { "cid": 1 },
+                { "cid": 2 },
+                { "nested": {
+                  "name": "te}st"
+                }}
+              ]
+            }
+          }
+        },
+        {
+          "upsert": true,
+          "extra": "ad\\"{\\"d"
+        }`,
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        value: {
+          data: [
+            { cid: 1 },
+            { cid: 2 },
+            { nested: {
+              name: "te}st"
+            }}
+          ]
+        },
+      },
+    })
+    expect(args[2]).toEqual({
+      upsert: true,
+      extra: "ad\"{\"d"
+    })
+  })
 })

package/src/integrations/tests/sql.spec.js

@@ -1,4 +1,4 @@
-const Sql = require("../base/sql")
+const Sql = require("../base/sql").default
 const { SqlClient } = require("../utils")
 
 const TABLE_NAME = "test"
@@ -240,18 +240,18 @@ describe("SQL query builder", () => {
     })
   })
 
-  it("should use like expression for MS-SQL when filter is contains", () => {
+  it("should use AND like expression for MS-SQL when filter is contains", () => {
     const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
       filters: {
         contains: {
-          age: 20,
-          name: "John"
+          age: [20, 25],
+          name: ["John", "Mary"]
         }
       }
     }))
     expect(query).toEqual({
-      bindings: [10, "%20%", "%John%"],
-      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where LOWER(${TABLE_NAME}.age) LIKE @p1 and LOWER(${TABLE_NAME}.name) LIKE @p2) as [${TABLE_NAME}]`
+      bindings: [10, "%20%", "%25%", `%"John"%`, `%"Mary"%`],
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER(${TABLE_NAME}.age) LIKE @p1 AND LOWER(${TABLE_NAME}.age) LIKE @p2) and (LOWER(${TABLE_NAME}.name) LIKE @p3 AND LOWER(${TABLE_NAME}.name) LIKE @p4)) as [${TABLE_NAME}]`
     })
   })
 
@@ -259,14 +259,14 @@ describe("SQL query builder", () => {
     const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
       filters: {
         contains: {
-          age: 20,
-          name: "John"
+          age: [20],
+          name: ["John"]
         }
       }
     }))
     expect(query).toEqual({
       bindings: [10],
-      sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_CONTAINS(${TABLE_NAME}.age, '20') and JSON_CONTAINS(${TABLE_NAME}.name, '"John"') limit ?) as \`${TABLE_NAME}\``
+      sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``
     })
   })
 
@@ -274,8 +274,8 @@ describe("SQL query builder", () => {
     const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
       filters: {
         contains: {
-          age: 20,
-          name: "John"
+          age: [20],
+          name: ["John"]
         }
       }
     }))
@@ -284,4 +284,94 @@ describe("SQL query builder", () => {
       sql: `select * from (select * from \"${TABLE_NAME}\" where \"${TABLE_NAME}\".\"age\"::jsonb @> '[20]' and \"${TABLE_NAME}\".\"name\"::jsonb @> '["John"]' limit $1) as \"${TABLE_NAME}\"`
     })
   })
+
+  it("should use NOT like expression for MS-SQL when filter is notContains", () => {
+    const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
+      filters: {
+        notContains: {
+          age: [20],
+          name: ["John"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10, "%20%", `%"John"%`],
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where NOT (LOWER(${TABLE_NAME}.age) LIKE @p1) and NOT (LOWER(${TABLE_NAME}.name) LIKE @p2)) as [${TABLE_NAME}]`
+    })
+  })
+
+  it("should use NOT JSON_CONTAINS expression for MySQL when filter is notContains", () => {
+    const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
+      filters: {
+        notContains: {
+          age: [20],
+          name: ["John"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10],
+      sql: `select * from (select * from \`${TABLE_NAME}\` where NOT JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and NOT JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``
+    })
+  })
+
+  it("should use jsonb operator NOT expression for PostgreSQL when filter is notContains", () => {
+    const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
+      filters: {
+        notContains: {
+          age: [20],
+          name: ["John"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10],
+      sql: `select * from (select * from \"${TABLE_NAME}\" where NOT \"${TABLE_NAME}\".\"age\"::jsonb @> '[20]' and NOT \"${TABLE_NAME}\".\"name\"::jsonb @> '["John"]' limit $1) as \"${TABLE_NAME}\"`
+    })
+  })
+
+  it("should use OR like expression for MS-SQL when filter is containsAny", () => {
+    const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
+      filters: {
+        containsAny: {
+          age: [20, 25],
+          name: ["John", "Mary"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10, "%20%", "%25%", `%"John"%`, `%"Mary"%`],
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER(${TABLE_NAME}.age) LIKE @p1 OR LOWER(${TABLE_NAME}.age) LIKE @p2) and (LOWER(${TABLE_NAME}.name) LIKE @p3 OR LOWER(${TABLE_NAME}.name) LIKE @p4)) as [${TABLE_NAME}]`
+    })
+  })
+
+  it("should use JSON_OVERLAPS expression for MySQL when filter is containsAny", () => {
+    const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
+      filters: {
+        containsAny: {
+          age: [20, 25],
+          name: ["John", "Mary"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10],
+      sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_OVERLAPS(${TABLE_NAME}.age, '[20,25]') and JSON_OVERLAPS(${TABLE_NAME}.name, '["John","Mary"]') limit ?) as \`${TABLE_NAME}\``
+    })
+  })
+
+  it("should use ?| operator expression for PostgreSQL when filter is containsAny", () => {
+    const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
+      filters: {
+        containsAny: {
+          age: [20, 25],
+          name: ["John", "Mary"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10],
+      sql: `select * from (select * from \"${TABLE_NAME}\" where \"${TABLE_NAME}\".\"age\"::jsonb ?| array [20,25] and \"${TABLE_NAME}\".\"name\"::jsonb ?| array ['John','Mary'] limit $1) as \"${TABLE_NAME}\"`
+    })
+  })
 })

package/src/middleware/joi-validator.js

@@ -13,10 +13,13 @@ function validate(schema, property) {
       params = ctx.request[property]
     }
 
-    schema = schema.append({
-      createdAt: Joi.any().optional(),
-      updatedAt: Joi.any().optional(),
-    })
+    // not all schemas have the append property e.g. array schemas
+    if (schema.append) {
+      schema = schema.append({
+        createdAt: Joi.any().optional(),
+        updatedAt: Joi.any().optional(),
+      })
+    }
 
     const { error } = schema.validate(params)
     if (error) {

package/src/threads/index.ts

@@ -106,5 +106,6 @@ export class Thread {
 
   static async shutdown() {
     await Thread.stopThreads()
+    console.log("Threads shutdown")
   }
 }