@budibase/server 1.2.58 → 1.2.59-alpha.0

package/src/integrations/tests/mongo.spec.js

@@ -102,4 +102,222 @@ describe("MongoDB Integration", () => {
     expect(error).toBeDefined()
     restore()
   })
+
+  it("creates ObjectIds if the field contains a match on ObjectId", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: "ObjectId('ACBD12345678ABCD12345678')",
+          name: "ObjectId('BBBB12345678ABCD12345678')"
+        },
+        update: {
+          _id: "ObjectId('FFFF12345678ABCD12345678')",
+          name: "ObjectId('CCCC12345678ABCD12345678')",
+        },
+        options: {
+          upsert: false,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      name: mongo.ObjectID.createFromHexString("BBBB12345678ABCD12345678"),
+    })
+    expect(args[1]).toEqual({
+      _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
+      name: mongo.ObjectID.createFromHexString("CCCC12345678ABCD12345678"),
+    })
+    expect(args[2]).toEqual({
+      upsert: false
+    })
+  })
+
+  it("creates ObjectIds if the $ operator fields contains a match on ObjectId", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: {
+            $eq: "ObjectId('ACBD12345678ABCD12345678')",
+          }
+        },
+        update: {
+          $set: {
+            _id: "ObjectId('FFFF12345678ABCD12345678')",
+          },
+        },
+        options: {
+          upsert: true,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
+      }
+    })
+    expect(args[2]).toEqual({
+      upsert: true
+    })
+  })
+
+  it("supports findOneAndUpdate", async () => {
+    const query = {
+      json: {
+        filter: {
+          _id: {
+            $eq: "ObjectId('ACBD12345678ABCD12345678')",
+          }
+        },
+        update: {
+          $set: {
+            name: "UPDATED",
+            age: 99
+          },
+        },
+        options: {
+          upsert: false,
+        },
+      },
+      extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" },
+    }
+    await config.integration.read(query)
+    expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled()
+    
+    const args = config.integration.client.findOneAndUpdate.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        name: "UPDATED",
+        age: 99
+      }
+    })
+    expect(args[2]).toEqual({
+      upsert: false
+    })
+  })
+
+  it("can parse nested objects with arrays", async () => {
+    const query = {
+      json: `{
+          "_id": {
+            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
+          }
+        },
+        {
+          "$set": {
+            "value": {
+              "data": [
+                { "cid": 1 },
+                { "cid": 2 },
+                { "nested": {
+                  "name": "test"
+                }}
+              ]
+            }
+          }
+        },
+        {
+          "upsert": true
+        }`,
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        value: {
+          data: [
+            { cid: 1 },
+            { cid: 2 },
+            { nested: {
+              name: "test"
+            }}
+          ]
+        },
+      },
+    })
+    expect(args[2]).toEqual({
+      upsert: true
+    })
+  })
+
+  it("ignores braces within strings when parsing nested objects", async () => {
+    const query = {
+      json: `{
+          "_id": {
+            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
+          }
+        },
+        {
+          "$set": {
+            "value": {
+              "data": [
+                { "cid": 1 },
+                { "cid": 2 },
+                { "nested": {
+                  "name": "te}st"
+                }}
+              ]
+            }
+          }
+        },
+        {
+          "upsert": true,
+          "extra": "ad\\"{\\"d"
+        }`,
+      extra: { collection: "testCollection", actionTypes: "updateOne" },
+    }
+    await config.integration.update(query)
+    expect(config.integration.client.updateOne).toHaveBeenCalled()
+    
+    const args = config.integration.client.updateOne.mock.calls[0]
+    expect(args[0]).toEqual({
+      _id: {
+        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+      }
+    })
+    expect(args[1]).toEqual({
+      $set: {
+        value: {
+          data: [
+            { cid: 1 },
+            { cid: 2 },
+            { nested: {
+              name: "te}st"
+            }}
+          ]
+        },
+      },
+    })
+    expect(args[2]).toEqual({
+      upsert: true,
+      extra: "ad\"{\"d"
+    })
+  })
 })

package/src/integrations/tests/sql.spec.js

@@ -1,4 +1,4 @@
-const Sql = require("../base/sql")
+const Sql = require("../base/sql").default
 const { SqlClient } = require("../utils")
 
 const TABLE_NAME = "test"
@@ -240,18 +240,18 @@ describe("SQL query builder", () => {
     })
   })
 
-  it("should use like expression for MS-SQL when filter is contains", () => {
+  it("should use AND like expression for MS-SQL when filter is contains", () => {
     const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
       filters: {
         contains: {
-          age: 20,
-          name: "John"
+          age: [20, 25],
+          name: ["John", "Mary"]
         }
       }
     }))
     expect(query).toEqual({
-      bindings: [10, "%20%", "%John%"],
-      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where LOWER(${TABLE_NAME}.age) LIKE @p1 and LOWER(${TABLE_NAME}.name) LIKE @p2) as [${TABLE_NAME}]`
+      bindings: [10, "%20%", "%25%", `%"John"%`, `%"Mary"%`],
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER(${TABLE_NAME}.age) LIKE @p1 AND LOWER(${TABLE_NAME}.age) LIKE @p2) and (LOWER(${TABLE_NAME}.name) LIKE @p3 AND LOWER(${TABLE_NAME}.name) LIKE @p4)) as [${TABLE_NAME}]`
     })
   })
 
@@ -259,14 +259,14 @@ describe("SQL query builder", () => {
     const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
       filters: {
         contains: {
-          age: 20,
-          name: "John"
+          age: [20],
+          name: ["John"]
         }
       }
     }))
     expect(query).toEqual({
       bindings: [10],
-      sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_CONTAINS(${TABLE_NAME}.age, '20') and JSON_CONTAINS(${TABLE_NAME}.name, '"John"') limit ?) as \`${TABLE_NAME}\``
+      sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``
     })
   })
 
@@ -274,8 +274,8 @@ describe("SQL query builder", () => {
     const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
       filters: {
         contains: {
-          age: 20,
-          name: "John"
+          age: [20],
+          name: ["John"]
         }
       }
     }))
@@ -284,4 +284,94 @@ describe("SQL query builder", () => {
       sql: `select * from (select * from \"${TABLE_NAME}\" where \"${TABLE_NAME}\".\"age\"::jsonb @> '[20]' and \"${TABLE_NAME}\".\"name\"::jsonb @> '["John"]' limit $1) as \"${TABLE_NAME}\"`
     })
   })
+
+  it("should use NOT like expression for MS-SQL when filter is notContains", () => {
+    const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
+      filters: {
+        notContains: {
+          age: [20],
+          name: ["John"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10, "%20%", `%"John"%`],
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where NOT (LOWER(${TABLE_NAME}.age) LIKE @p1) and NOT (LOWER(${TABLE_NAME}.name) LIKE @p2)) as [${TABLE_NAME}]`
+    })
+  })
+
+  it("should use NOT JSON_CONTAINS expression for MySQL when filter is notContains", () => {
+    const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
+      filters: {
+        notContains: {
+          age: [20],
+          name: ["John"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10],
+      sql: `select * from (select * from \`${TABLE_NAME}\` where NOT JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and NOT JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``
+    })
+  })
+
+  it("should use jsonb operator NOT expression for PostgreSQL when filter is notContains", () => {
+    const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
+      filters: {
+        notContains: {
+          age: [20],
+          name: ["John"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10],
+      sql: `select * from (select * from \"${TABLE_NAME}\" where NOT \"${TABLE_NAME}\".\"age\"::jsonb @> '[20]' and NOT \"${TABLE_NAME}\".\"name\"::jsonb @> '["John"]' limit $1) as \"${TABLE_NAME}\"`
+    })
+  })
+
+  it("should use OR like expression for MS-SQL when filter is containsAny", () => {
+    const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
+      filters: {
+        containsAny: {
+          age: [20, 25],
+          name: ["John", "Mary"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10, "%20%", "%25%", `%"John"%`, `%"Mary"%`],
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER(${TABLE_NAME}.age) LIKE @p1 OR LOWER(${TABLE_NAME}.age) LIKE @p2) and (LOWER(${TABLE_NAME}.name) LIKE @p3 OR LOWER(${TABLE_NAME}.name) LIKE @p4)) as [${TABLE_NAME}]`
+    })
+  })
+
+  it("should use JSON_OVERLAPS expression for MySQL when filter is containsAny", () => {
+    const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
+      filters: {
+        containsAny: {
+          age: [20, 25],
+          name: ["John", "Mary"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10],
+      sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_OVERLAPS(${TABLE_NAME}.age, '[20,25]') and JSON_OVERLAPS(${TABLE_NAME}.name, '["John","Mary"]') limit ?) as \`${TABLE_NAME}\``
+    })
+  })
+
+  it("should use ?| operator expression for PostgreSQL when filter is containsAny", () => {
+    const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
+      filters: {
+        containsAny: {
+          age: [20, 25],
+          name: ["John", "Mary"]
+        }
+      }
+    }))
+    expect(query).toEqual({
+      bindings: [10],
+      sql: `select * from (select * from \"${TABLE_NAME}\" where \"${TABLE_NAME}\".\"age\"::jsonb ?| array [20,25] and \"${TABLE_NAME}\".\"name\"::jsonb ?| array ['John','Mary'] limit $1) as \"${TABLE_NAME}\"`
+    })
+  })
 })

package/src/middleware/joi-validator.js

@@ -13,10 +13,13 @@ function validate(schema, property) {
       params = ctx.request[property]
     }
 
-    schema = schema.append({
-      createdAt: Joi.any().optional(),
-      updatedAt: Joi.any().optional(),
-    })
+    // not all schemas have the append property e.g. array schemas
+    if (schema.append) {
+      schema = schema.append({
+        createdAt: Joi.any().optional(),
+        updatedAt: Joi.any().optional(),
+      })
+    }
 
     const { error } = schema.validate(params)
     if (error) {

package/src/threads/index.ts

@@ -106,5 +106,6 @@ export class Thread {
 
   static async shutdown() {
     await Thread.stopThreads()
+    console.log("Threads shutdown")
   }
 }

package/src/utilities/fileSystem/index.js

@@ -15,6 +15,7 @@ const {
   streamUpload,
   deleteFolder,
   downloadTarball,
+  deleteFiles,
 } = require("./utilities")
 const { updateClientLibrary } = require("./clientLibrary")
 const env = require("../../environment")
@@ -332,5 +333,6 @@ exports.cleanup = appIds => {
 exports.upload = upload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
+exports.deleteFiles = deleteFiles
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH

package/src/utilities/redis.js

@@ -20,6 +20,7 @@ exports.shutdown = async () => {
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
+  console.log("Redis shutdown")
 }
 
 exports.doesUserHaveLock = async (devAppId, user) => {