@budibase/server 1.2.44-alpha.0 → 1.2.46

package/src/automations/listeners.ts

@@ -0,0 +1,78 @@
+import { Queue, Job, JobId } from "bull"
+import { AutomationEvent } from "../definitions/automations"
+import * as automation from "../threads/automation"
+
+export const addListeners = (queue: Queue) => {
+  logging(queue)
+  // handleStalled(queue)
+}
+
+const handleStalled = (queue: Queue) => {
+  queue.on("active", async (job: Job) => {
+    await automation.removeStalled(job as AutomationEvent)
+  })
+}
+
+const logging = (queue: Queue) => {
+  if (process.env.NODE_DEBUG?.includes("bull")) {
+    queue
+      .on("error", (error: any) => {
+        // An error occurred.
+        console.error(`automation-event=error error=${JSON.stringify(error)}`)
+      })
+      .on("waiting", (jobId: JobId) => {
+        // A Job is waiting to be processed as soon as a worker is idling.
+        console.log(`automation-event=waiting jobId=${jobId}`)
+      })
+      .on("active", (job: Job, jobPromise: any) => {
+        // A job has started. You can use `jobPromise.cancel()`` to abort it.
+        console.log(`automation-event=active jobId=${job.id}`)
+      })
+      .on("stalled", (job: Job) => {
+        // A job has been marked as stalled. This is useful for debugging job
+        // workers that crash or pause the event loop.
+        console.error(
+          `automation-event=stalled jobId=${job.id} job=${JSON.stringify(job)}`
+        )
+      })
+      .on("progress", (job: Job, progress: any) => {
+        // A job's progress was updated!
+        console.log(
+          `automation-event=progress jobId=${job.id} progress=${progress}`
+        )
+      })
+      .on("completed", (job: Job, result) => {
+        // A job successfully completed with a `result`.
+        console.log(
+          `automation-event=completed jobId=${job.id} result=${result}`
+        )
+      })
+      .on("failed", (job, err: any) => {
+        // A job failed with reason `err`!
+        console.log(`automation-event=failed jobId=${job.id} error=${err}`)
+      })
+      .on("paused", () => {
+        // The queue has been paused.
+        console.log(`automation-event=paused`)
+      })
+      .on("resumed", (job: Job) => {
+        // The queue has been resumed.
+        console.log(`automation-event=paused jobId=${job.id}`)
+      })
+      .on("cleaned", (jobs: Job[], type: string) => {
+        // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned
+        // jobs, and `type` is the type of jobs cleaned.
+        console.log(
+          `automation-event=cleaned length=${jobs.length} type=${type}`
+        )
+      })
+      .on("drained", () => {
+        // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed)
+        console.log(`automation-event=drained`)
+      })
+      .on("removed", (job: Job) => {
+        // A job successfully removed.
+        console.log(`automation-event=removed jobId=${job.id}`)
+      })
+  }
+}

package/src/automations/logging/index.ts

@@ -23,9 +23,6 @@ export async function checkAppMetadata(apps: App[]) {
     for (let [key, errors] of Object.entries(metadata.automationErrors)) {
       const updated = []
       for (let error of errors) {
-        if (!error) {
-          continue
-        }
         const startDate = error.split(dbUtils.SEPARATOR)[2]
         if (startDate > maxStartDate) {
           updated.push(error)

package/src/automations/utils.ts

@@ -21,11 +21,13 @@ const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)
 
+const jobMessage = (job: any, message: string) => {
+  return `app=${job.data.event.appId} automation=${job.data.automation._id} jobId=${job.id} trigger=${job.data.automation.definition.trigger.event} : ${message}`
+}
+
 export async function processEvent(job: any) {
   try {
-    console.log(
-      `${job.data.automation.appId} automation ${job.data.automation._id} running`
-    )
+    console.log(jobMessage(job, "running"))
     // need to actually await these so that an error can be captured properly
     const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId)
     return await tenancy.doInTenant(tenantId, async () => {
@@ -34,9 +36,7 @@ export async function processEvent(job: any) {
     })
   } catch (err) {
     const errJson = JSON.stringify(err)
-    console.error(
-      `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${errJson}`
-    )
+    console.error(jobMessage(job, `was unable to run - ${errJson}`))
     console.trace(err)
     return { err }
   }
@@ -91,6 +91,7 @@ export async function disableAllCrons(appId: any) {
 export async function disableCron(jobId: string, jobKey: string) {
   await queue.removeRepeatableByKey(jobKey)
   await queue.removeJobs(jobId)
+  console.log(`jobId=${jobId} disabled`)
 }
 
 export async function clearMetadata() {

package/src/integrations/base/sql.ts

@@ -10,7 +10,6 @@ import { QueryOptions } from "../../definitions/datasource"
 import { isIsoDateString, SqlClient } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import environment from "../../environment"
-import { removeKeyNumbering } from "./utils"
 
 const envLimit = environment.SQL_MAX_ROWS
   ? parseInt(environment.SQL_MAX_ROWS)
@@ -134,13 +133,12 @@ class InternalBuilder {
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const updatedKey = removeKeyNumbering(key)
-        const isRelationshipField = updatedKey.includes(".")
+        const isRelationshipField = key.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${updatedKey}`, value)
+          fn(`${opts.tableName}.${key}`, value)
         }
         if (opts.relationship && isRelationshipField) {
-          fn(updatedKey, value)
+          fn(key, value)
         }
       }
     }
@@ -159,61 +157,6 @@ class InternalBuilder {
       }
     }
 
-    const contains = (mode: object, any: boolean = false) => {
-      const fnc = allOr ? "orWhere" : "where"
-      const rawFnc = `${fnc}Raw`
-      const not = mode === filters?.notContains ? "NOT " : ""
-      function stringifyArray(value: Array<any>, quoteStyle = '"'): string {
-        for (let i in value) {
-          if (typeof value[i] === "string") {
-            value[i] = `${quoteStyle}${value[i]}${quoteStyle}`
-          }
-        }
-        return `[${value.join(",")}]`
-      }
-      if (this.client === SqlClient.POSTGRES) {
-        iterate(mode, (key: string, value: Array<any>) => {
-          const wrap = any ? "" : "'"
-          const containsOp = any ? "\\?| array" : "@>"
-          const fieldNames = key.split(/\./g)
-          const tableName = fieldNames[0]
-          const columnName = fieldNames[1]
-          // @ts-ignore
-          query = query[rawFnc](
-            `${not}"${tableName}"."${columnName}"::jsonb ${containsOp} ${wrap}${stringifyArray(
-              value,
-              any ? "'" : '"'
-            )}${wrap}`
-          )
-        })
-      } else if (this.client === SqlClient.MY_SQL) {
-        const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
-        iterate(mode, (key: string, value: Array<any>) => {
-          // @ts-ignore
-          query = query[rawFnc](
-            `${not}${jsonFnc}(${key}, '${stringifyArray(value)}')`
-          )
-        })
-      } else {
-        const andOr = mode === filters?.containsAny ? " OR " : " AND "
-        iterate(mode, (key: string, value: Array<any>) => {
-          let statement = ""
-          for (let i in value) {
-            if (typeof value[i] === "string") {
-              value[i] = `%"${value[i]}"%`
-            } else {
-              value[i] = `%${value[i]}%`
-            }
-            statement +=
-              (statement ? andOr : "") +
-              `LOWER(${likeKey(this.client, key)}) LIKE ?`
-          }
-          // @ts-ignore
-          query = query[rawFnc](`${not}(${statement})`, value)
-        })
-      }
-    }
-
     if (!filters) {
       return query
     }
@@ -284,13 +227,32 @@ class InternalBuilder {
       })
     }
     if (filters.contains) {
-      contains(filters.contains)
-    }
-    if (filters.notContains) {
-      contains(filters.notContains)
-    }
-    if (filters.containsAny) {
-      contains(filters.containsAny, true)
+      const fnc = allOr ? "orWhere" : "where"
+      const rawFnc = `${fnc}Raw`
+      if (this.client === SqlClient.POSTGRES) {
+        iterate(filters.contains, (key: string, value: any) => {
+          const fieldNames = key.split(/\./g)
+          const tableName = fieldNames[0]
+          const columnName = fieldNames[1]
+          if (typeof value === "string") {
+            value = `"${value}"`
+          }
+          // @ts-ignore
+          query = query[rawFnc](
+            `"${tableName}"."${columnName}"::jsonb @> '[${value}]'`
+          )
+        })
+      } else if (this.client === SqlClient.MY_SQL) {
+        iterate(filters.contains, (key: string, value: any) => {
+          if (typeof value === "string") {
+            value = `"${value}"`
+          }
+          // @ts-ignore
+          query = query[rawFnc](`JSON_CONTAINS(${key}, '${value}')`)
+        })
+      } else {
+        iterate(filters.contains, like)
+      }
     }
     return query
   }
@@ -620,3 +582,4 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 }
 
 export default SqlQueryBuilder
+module.exports = SqlQueryBuilder

package/src/integrations/base/utils.ts

@@ -1,12 +1,21 @@
-const QUERY_START_REGEX = /\d[0-9]*:/g
+import { QueryJson, Datasource } from "@budibase/types"
 
-export function removeKeyNumbering(key: any): string {
-  if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
-    const parts = key.split(":")
-    // remove the number
-    parts.shift()
-    return parts.join(":")
-  } else {
-    return key
+module DatasourceUtils {
+  const { integrations } = require("../index")
+
+  export async function makeExternalQuery(
+    datasource: Datasource,
+    json: QueryJson
+  ) {
+    const Integration = integrations[datasource.source]
+    // query is the opinionated function
+    if (Integration.prototype.query) {
+      const integration = new Integration(datasource.config)
+      return integration.query(json)
+    } else {
+      throw "Datasource does not support query."
+    }
   }
+
+  module.exports.makeExternalQuery = makeExternalQuery
 }

package/src/integrations/microsoftSqlServer.ts

@@ -16,10 +16,10 @@ import {
   finaliseExternalTables,
   SqlClient,
 } from "./utils"
-import Sql from "./base/sql"
 
 module MSSQLModule {
   const sqlServer = require("mssql")
+  const Sql = require("./base/sql")
   const DEFAULT_SCHEMA = "dbo"
 
   interface MSSQLConfig {
@@ -97,8 +97,7 @@ module MSSQLModule {
   class SqlServerIntegration extends Sql implements DatasourcePlus {
     private readonly config: MSSQLConfig
     private index: number = 0
-    private readonly pool: any
-    private client: any
+    static pool: any
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 

package/src/integrations/mongodb.ts

@@ -92,15 +92,12 @@ module MongoDBModule {
           if (json[field] instanceof Object) {
             json[field] = self.createObjectIds(json[field])
           }
-          if (
-            (field === "_id" || field?.startsWith("$")) &&
-            typeof json[field] === "string"
-          ) {
-            const id = json[field].match(
+          if (field === "_id" && typeof json[field] === "string") {
+            const id = json["_id"].match(
               /(?<=objectid\(['"]).*(?=['"]\))/gi
             )?.[0]
             if (id) {
-              json[field] = ObjectID.createFromHexString(id)
+              json["_id"] = ObjectID.createFromHexString(id)
             }
           }
         }
@@ -117,31 +114,10 @@ module MongoDBModule {
     }
 
     parseQueryParams(params: string, mode: string) {
-      let queryParams = []
-      let openCount = 0
-      let inQuotes = false
-      let i = 0
-      let startIndex = 0
-      for (let c of params) {
-        if (c === '"' && i > 0 && params[i - 1] !== "\\") {
-          inQuotes = !inQuotes
-        }
-        if (c === "{" && !inQuotes) {
-          openCount++
-          if (openCount === 1) {
-            startIndex = i
-          }
-        } else if (c === "}" && !inQuotes) {
-          if (openCount === 1) {
-            queryParams.push(JSON.parse(params.substring(startIndex, i + 1)))
-          }
-          openCount--
-        }
-        i++
-      }
-      let group1 = queryParams[0] ?? {}
-      let group2 = queryParams[1] ?? {}
-      let group3 = queryParams[2] ?? {}
+      let queryParams = params.split(/(?<=}),[\n\s]*(?={)/g)
+      let group1 = queryParams[0] ? JSON.parse(queryParams[0]) : {}
+      let group2 = queryParams[1] ? JSON.parse(queryParams[1]) : {}
+      let group3 = queryParams[2] ? JSON.parse(queryParams[2]) : {}
       if (mode === "update") {
         return {
           filter: group1,
@@ -200,10 +176,7 @@ module MongoDBModule {
             return await collection.findOne(json)
           }
           case "findOneAndUpdate": {
-            if (typeof query.json === "string") {
-              json = this.parseQueryParams(query.json, "update")
-            }
-            let findAndUpdateJson = this.createObjectIds(json) as {
+            let findAndUpdateJson = json as {
               filter: FilterQuery<any>
               update: UpdateQuery<any>
               options: FindOneAndUpdateOption<any>

package/src/integrations/mysql.ts

@@ -17,10 +17,10 @@ import {
 } from "./utils"
 import dayjs from "dayjs"
 const { NUMBER_REGEX } = require("../utilities")
-import Sql from "./base/sql"
 
 module MySQLModule {
   const mysql = require("mysql2/promise")
+  const Sql = require("./base/sql")
 
   interface MySQLConfig {
     host: string

package/src/integrations/postgres.ts

@@ -14,10 +14,10 @@ import {
   finaliseExternalTables,
   SqlClient,
 } from "./utils"
-import Sql from "./base/sql"
 
 module PostgresModule {
   const { Client, types } = require("pg")
+  const Sql = require("./base/sql")
   const { escapeDangerousCharacters } = require("../utilities")
 
   // Return "date" and "timestamp" types as plain strings.
@@ -117,7 +117,6 @@ module PostgresModule {
     private readonly client: any
     private readonly config: PostgresConfig
     private index: number = 1
-    private open: boolean
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 

package/src/integrations/tests/mongo.spec.js

@@ -102,222 +102,4 @@ describe("MongoDB Integration", () => {
     expect(error).toBeDefined()
     restore()
   })
-
-  it("creates ObjectIds if the _id fields contains a match on ObjectId", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: "ObjectId('ACBD12345678ABCD12345678')",
-          name: "ObjectId('name')"
-        },
-        update: {
-          _id: "ObjectId('FFFF12345678ABCD12345678')",
-          name: "ObjectId('updatedName')",
-        },
-        options: {
-          upsert: false,
-        },
-      },
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-    
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      name: "ObjectId('name')",
-    })
-    expect(args[1]).toEqual({
-      _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
-      name: "ObjectId('updatedName')",
-    })
-    expect(args[2]).toEqual({
-      upsert: false
-    })
-  })
-
-  it("creates ObjectIds if the $ operator fields contains a match on ObjectId", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: {
-            $eq: "ObjectId('ACBD12345678ABCD12345678')",
-          }
-        },
-        update: {
-          $set: {
-            _id: "ObjectId('FFFF12345678ABCD12345678')",
-          },
-        },
-        options: {
-          upsert: true,
-        },
-      },
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-    
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      }
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
-      }
-    })
-    expect(args[2]).toEqual({
-      upsert: true
-    })
-  })
-
-  it("supports findOneAndUpdate", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: {
-            $eq: "ObjectId('ACBD12345678ABCD12345678')",
-          }
-        },
-        update: {
-          $set: {
-            name: "UPDATED",
-            age: 99
-          },
-        },
-        options: {
-          upsert: false,
-        },
-      },
-      extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" },
-    }
-    await config.integration.read(query)
-    expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled()
-    
-    const args = config.integration.client.findOneAndUpdate.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      }
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        name: "UPDATED",
-        age: 99
-      }
-    })
-    expect(args[2]).toEqual({
-      upsert: false
-    })
-  })
-
-  it("can parse nested objects with arrays", async () => {
-    const query = {
-      json: `{
-          "_id": {
-            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
-          }
-        },
-        {
-          "$set": {
-            "value": {
-              "data": [
-                { "cid": 1 },
-                { "cid": 2 },
-                { "nested": {
-                  "name": "test"
-                }}
-              ]
-            }
-          }
-        },
-        {
-          "upsert": true
-        }`,
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-    
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      }
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        value: {
-          data: [
-            { cid: 1 },
-            { cid: 2 },
-            { nested: {
-              name: "test"
-            }}
-          ]
-        },
-      },
-    })
-    expect(args[2]).toEqual({
-      upsert: true
-    })
-  })
-
-  it("ignores braces within strings when parsing nested objects", async () => {
-    const query = {
-      json: `{
-          "_id": {
-            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
-          }
-        },
-        {
-          "$set": {
-            "value": {
-              "data": [
-                { "cid": 1 },
-                { "cid": 2 },
-                { "nested": {
-                  "name": "te}st"
-                }}
-              ]
-            }
-          }
-        },
-        {
-          "upsert": true,
-          "extra": "ad\\"{\\"d"
-        }`,
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-    
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      }
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        value: {
-          data: [
-            { cid: 1 },
-            { cid: 2 },
-            { nested: {
-              name: "te}st"
-            }}
-          ]
-        },
-      },
-    })
-    expect(args[2]).toEqual({
-      upsert: true,
-      extra: "ad\"{\"d"
-    })
-  })
 })