@budibase/server 1.2.33-alpha.1 → 1.2.35

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.2.33-alpha.1",
+  "version": "1.2.35",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.2.33-alpha.1",
-    "@budibase/client": "1.2.33-alpha.1",
-    "@budibase/pro": "1.2.33-alpha.0",
-    "@budibase/string-templates": "1.2.33-alpha.1",
-    "@budibase/types": "1.2.33-alpha.1",
+    "@budibase/backend-core": "^1.2.35",
+    "@budibase/client": "^1.2.35",
+    "@budibase/pro": "1.2.34",
+    "@budibase/string-templates": "^1.2.35",
+    "@budibase/types": "^1.2.35",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -106,7 +106,7 @@
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
     "jimp": "0.16.1",
-    "joi": "17.6.0",
+    "joi": "17.2.1",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
     "knex": "0.95.15",
@@ -195,5 +195,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "8e7169e0647d7f5c133e9b49a37bd7428eca743a"
+  "gitHead": "a1c1185fc5dcdaf0c363565a97436b6933bb174c"
 }

package/src/api/controllers/row/internalSearch.js

@@ -1,5 +1,4 @@
 const { SearchIndexes } = require("../../../db/utils")
-const { removeKeyNumbering } = require("./utils")
 const fetch = require("node-fetch")
 const { getCouchInfo } = require("@budibase/backend-core/db")
 const { getAppId } = require("@budibase/backend-core/context")
@@ -198,8 +197,6 @@ class QueryBuilder {
 
     function build(structure, queryFn) {
       for (let [key, value] of Object.entries(structure)) {
-        // check for new format - remove numbering if needed
-        key = removeKeyNumbering(key)
         key = builder.preprocess(key.replace(/ /g, "_"), {
           escape: true,
         })
@@ -313,9 +310,6 @@ class QueryBuilder {
   }
 }
 
-// exported for unit testing
-exports.QueryBuilder = QueryBuilder
-
 /**
  * Executes a lucene search query.
  * @param url The query URL

package/src/api/controllers/row/utils.js

@@ -3,9 +3,8 @@ const { cloneDeep } = require("lodash/fp")
 const { InternalTables } = require("../../../db/utils")
 const userController = require("../user")
 const { FieldTypes } = require("../../../constants")
+const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
-const { makeExternalQuery } = require("../../../integrations/base/query")
-const { removeKeyNumbering } = require("../../../integrations/base/utils")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value) {
@@ -17,8 +16,6 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-exports.removeKeyNumbering = removeKeyNumbering
-
 exports.getDatasourceAndQuery = async json => {
   const datasourceId = json.endpoint.datasourceId
   const db = getAppDB()

package/src/api/controllers/table/external.js

@@ -14,7 +14,7 @@ const {
   FieldTypes,
   RelationshipTypes,
 } = require("../../../constants")
-const { makeExternalQuery } = require("../../../integrations/base/query")
+const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { cloneDeep } = require("lodash/fp")
 const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")

package/src/automations/logging/index.ts

@@ -23,9 +23,6 @@ export async function checkAppMetadata(apps: App[]) {
     for (let [key, errors] of Object.entries(metadata.automationErrors)) {
       const updated = []
       for (let error of errors) {
-        if (!error) {
-          continue
-        }
         const startDate = error.split(dbUtils.SEPARATOR)[2]
         if (startDate > maxStartDate) {
           updated.push(error)

package/src/automations/steps/queryRows.js

@@ -14,6 +14,16 @@ const SortOrdersPretty = {
   [SortOrders.DESCENDING]: "Descending",
 }
 
+const EmptyFilterOptions = {
+  RETURN_ALL: "all",
+  RETURN_NONE: "none",
+}
+
+const EmptyFilterOptionsPretty = {
+  [EmptyFilterOptions.RETURN_ALL]: "Return all table rows",
+  [EmptyFilterOptions.RETURN_NONE]: "Return no rows",
+}
+
 exports.definition = {
   description: "Query rows from the database",
   icon: "Search",
@@ -52,6 +62,12 @@ exports.definition = {
           title: "Limit",
           customType: "queryLimit",
         },
+        onEmptyFilter: {
+          pretty: Object.values(EmptyFilterOptionsPretty),
+          enum: Object.values(EmptyFilterOptions),
+          type: "string",
+          title: "When Filter Empty",
+        },
       },
       required: ["tableId"],
     },
@@ -103,6 +119,10 @@ function typeCoercion(filters, table) {
   return filters
 }
 
+const hasNullFilters = filters =>
+  filters.length === 0 ||
+  filters.some(filter => filter.value === null || filter.value === "")
+
 exports.run = async function ({ inputs, appId }) {
   const { tableId, filters, sortColumn, sortOrder, limit } = inputs
   const table = await getTable(appId, tableId)
@@ -127,9 +147,21 @@ exports.run = async function ({ inputs, appId }) {
     version: "1",
   })
   try {
-    await rowController.search(ctx)
+    let rows
+
+    if (
+      inputs.onEmptyFilter === EmptyFilterOptions.RETURN_NONE &&
+      inputs["filters-def"] &&
+      hasNullFilters(inputs["filters-def"])
+    ) {
+      rows = []
+    } else {
+      await rowController.search(ctx)
+      rows = ctx.body ? ctx.body.rows : []
+    }
+
     return {
-      rows: ctx.body ? ctx.body.rows : [],
+      rows,
       success: ctx.status === 200,
     }
   } catch (err) {

package/src/automations/tests/queryRows.spec.js

@@ -16,7 +16,7 @@ describe("Test a query step automation", () => {
   let table
   let config = setup.getConfig()
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
     table = await config.createTable()
     const row = {
@@ -48,4 +48,70 @@ describe("Test a query step automation", () => {
     expect(res.rows.length).toBe(2)
     expect(res.rows[0].name).toBe(NAME)
   })
+
+  it("Returns all rows when onEmptyFilter has no value and no filters are passed", async () => {
+    const inputs = {
+      tableId: table._id,
+      filters: {},
+      sortColumn: "name",
+      sortOrder: "ascending",
+      limit: 10,
+    }
+    const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
+    expect(res.success).toBe(true)
+    expect(res.rows).toBeDefined()
+    expect(res.rows.length).toBe(2)
+    expect(res.rows[0].name).toBe(NAME)
+  })
+
+  it("Returns no rows when onEmptyFilter is RETURN_NONE and theres no filters", async () => {
+    const inputs = {
+      tableId: table._id,
+      filters: {},
+      "filters-def": [],
+      sortColumn: "name",
+      sortOrder: "ascending",
+      limit: 10,
+      onEmptyFilter: "none",
+    }
+    const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
+    expect(res.success).toBe(false)
+    expect(res.rows).toBeDefined()
+    expect(res.rows.length).toBe(0)
+  })
+
+  it("Returns no rows when onEmptyFilters RETURN_NONE and a filter is passed with a null value", async () => {
+    const inputs = {
+      tableId: table._id,
+      onEmptyFilter: "none",
+      filters: {},
+      "filters-def": [
+        {
+          value: null
+        }
+      ],
+      sortColumn: "name",
+      sortOrder: "ascending",
+      limit: 10,
+    }
+    const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
+    expect(res.success).toBe(false)
+    expect(res.rows).toBeDefined()
+    expect(res.rows.length).toBe(0)
+  })
+
+  it("Returns rows when onEmptyFilter is RETURN_ALL and no filter is passed", async () => {
+    const inputs = {
+      tableId: table._id,
+      onEmptyFilter: "all",
+      filters: {},
+      sortColumn: "name",
+      sortOrder: "ascending",
+      limit: 10,
+    }
+    const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
+    expect(res.success).toBe(true)
+    expect(res.rows).toBeDefined()
+    expect(res.rows.length).toBe(2)
+  })
 })

package/src/integrations/base/sql.ts

@@ -10,7 +10,6 @@ import {
 import { isIsoDateString, SqlClients } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import environment from "../../environment"
-import { removeKeyNumbering } from "./utils"
 
 const envLimit = environment.SQL_MAX_ROWS
   ? parseInt(environment.SQL_MAX_ROWS)
@@ -134,13 +133,12 @@ class InternalBuilder {
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const updatedKey = removeKeyNumbering(key)
-        const isRelationshipField = updatedKey.includes(".")
+        const isRelationshipField = key.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${updatedKey}`, value)
+          fn(`${opts.tableName}.${key}`, value)
         }
         if (opts.relationship && isRelationshipField) {
-          fn(updatedKey, value)
+          fn(key, value)
         }
       }
     }
@@ -584,3 +582,4 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 }
 
 export default SqlQueryBuilder
+module.exports = SqlQueryBuilder

package/src/integrations/base/utils.ts

@@ -1,12 +1,22 @@
-const QUERY_START_REGEX = /\d[0-9]*:/g
+import { QueryJson } from "../../definitions/datasource"
+import { Datasource } from "../../definitions/common"
 
-export function removeKeyNumbering(key: any): string {
-  if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
-    const parts = key.split(":")
-    // remove the number
-    parts.shift()
-    return parts.join(":")
-  } else {
-    return key
+module DatasourceUtils {
+  const { integrations } = require("../index")
+
+  export async function makeExternalQuery(
+    datasource: Datasource,
+    json: QueryJson
+  ) {
+    const Integration = integrations[datasource.source]
+    // query is the opinionated function
+    if (Integration.prototype.query) {
+      const integration = new Integration(datasource.config)
+      return integration.query(json)
+    } else {
+      throw "Datasource does not support query."
+    }
   }
+
+  module.exports.makeExternalQuery = makeExternalQuery
 }

package/src/integrations/microsoftSqlServer.ts

@@ -15,10 +15,10 @@ import {
 } from "./utils"
 import { DatasourcePlus } from "./base/datasourcePlus"
 import { Table, TableSchema } from "../definitions/common"
-import Sql from "./base/sql"
 
 module MSSQLModule {
   const sqlServer = require("mssql")
+  const Sql = require("./base/sql")
   const DEFAULT_SCHEMA = "dbo"
 
   interface MSSQLConfig {
@@ -96,8 +96,7 @@ module MSSQLModule {
   class SqlServerIntegration extends Sql implements DatasourcePlus {
     private readonly config: MSSQLConfig
     private index: number = 0
-    private readonly pool: any
-    private client: any
+    static pool: any
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 

package/src/integrations/mongodb.ts

@@ -92,15 +92,12 @@ module MongoDBModule {
           if (json[field] instanceof Object) {
             json[field] = self.createObjectIds(json[field])
           }
-          if (
-            (field === "_id" || field?.startsWith("$")) &&
-            typeof json[field] === "string"
-          ) {
-            const id = json[field].match(
+          if (field === "_id" && typeof json[field] === "string") {
+            const id = json["_id"].match(
               /(?<=objectid\(['"]).*(?=['"]\))/gi
             )?.[0]
             if (id) {
-              json[field] = ObjectID.createFromHexString(id)
+              json["_id"] = ObjectID.createFromHexString(id)
             }
           }
         }
@@ -117,31 +114,10 @@ module MongoDBModule {
     }
 
     parseQueryParams(params: string, mode: string) {
-      let queryParams = []
-      let openCount = 0
-      let inQuotes = false
-      let i = 0
-      let startIndex = 0
-      for (let c of params) {
-        if (c === '"' && i > 0 && params[i - 1] !== "\\") {
-          inQuotes = !inQuotes
-        }
-        if (c === "{" && !inQuotes) {
-          openCount++
-          if (openCount === 1) {
-            startIndex = i
-          }
-        } else if (c === "}" && !inQuotes) {
-          if (openCount === 1) {
-            queryParams.push(JSON.parse(params.substring(startIndex, i + 1)))
-          }
-          openCount--
-        }
-        i++
-      }
-      let group1 = queryParams[0] ?? {}
-      let group2 = queryParams[1] ?? {}
-      let group3 = queryParams[2] ?? {}
+      let queryParams = params.split(/(?<=}),[\n\s]*(?={)/g)
+      let group1 = queryParams[0] ? JSON.parse(queryParams[0]) : {}
+      let group2 = queryParams[1] ? JSON.parse(queryParams[1]) : {}
+      let group3 = queryParams[2] ? JSON.parse(queryParams[2]) : {}
       if (mode === "update") {
         return {
           filter: group1,
@@ -200,10 +176,7 @@ module MongoDBModule {
             return await collection.findOne(json)
           }
           case "findOneAndUpdate": {
-            if (typeof query.json === "string") {
-              json = this.parseQueryParams(query.json, "update")
-            }
-            let findAndUpdateJson = this.createObjectIds(json) as {
+            let findAndUpdateJson = json as {
               filter: FilterQuery<any>
               update: UpdateQuery<any>
               options: FindOneAndUpdateOption<any>

package/src/integrations/mysql.ts

@@ -16,10 +16,10 @@ import {
 import { DatasourcePlus } from "./base/datasourcePlus"
 import dayjs from "dayjs"
 const { NUMBER_REGEX } = require("../utilities")
-import Sql from "./base/sql"
 
 module MySQLModule {
   const mysql = require("mysql2/promise")
+  const Sql = require("./base/sql")
 
   interface MySQLConfig {
     host: string

package/src/integrations/postgres.ts

@@ -14,10 +14,10 @@ import {
   SqlClients,
 } from "./utils"
 import { DatasourcePlus } from "./base/datasourcePlus"
-import Sql from "./base/sql"
 
 module PostgresModule {
   const { Client, types } = require("pg")
+  const Sql = require("./base/sql")
   const { escapeDangerousCharacters } = require("../utilities")
 
   // Return "date" and "timestamp" types as plain strings.
@@ -117,7 +117,6 @@ module PostgresModule {
     private readonly client: any
     private readonly config: PostgresConfig
     private index: number = 1
-    private open: boolean
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 

package/src/integrations/queries/sql.ts

@@ -1,4 +1,5 @@
 import { findHBSBlocks, processStringSync } from "@budibase/string-templates"
+import { Integration } from "../../definitions/datasource"
 import { DatasourcePlus } from "../base/datasourcePlus"
 
 const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")

package/src/integrations/tests/mongo.spec.js

@@ -102,222 +102,4 @@ describe("MongoDB Integration", () => {
     expect(error).toBeDefined()
     restore()
   })
-
-  it("creates ObjectIds if the _id fields contains a match on ObjectId", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: "ObjectId('ACBD12345678ABCD12345678')",
-          name: "ObjectId('name')"
-        },
-        update: {
-          _id: "ObjectId('FFFF12345678ABCD12345678')",
-          name: "ObjectId('updatedName')",
-        },
-        options: {
-          upsert: false,
-        },
-      },
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-    
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      name: "ObjectId('name')",
-    })
-    expect(args[1]).toEqual({
-      _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
-      name: "ObjectId('updatedName')",
-    })
-    expect(args[2]).toEqual({
-      upsert: false
-    })
-  })
-
-  it("creates ObjectIds if the $ operator fields contains a match on ObjectId", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: {
-            $eq: "ObjectId('ACBD12345678ABCD12345678')",
-          }
-        },
-        update: {
-          $set: {
-            _id: "ObjectId('FFFF12345678ABCD12345678')",
-          },
-        },
-        options: {
-          upsert: true,
-        },
-      },
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-    
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      }
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
-      }
-    })
-    expect(args[2]).toEqual({
-      upsert: true
-    })
-  })
-
-  it("supports findOneAndUpdate", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: {
-            $eq: "ObjectId('ACBD12345678ABCD12345678')",
-          }
-        },
-        update: {
-          $set: {
-            name: "UPDATED",
-            age: 99
-          },
-        },
-        options: {
-          upsert: false,
-        },
-      },
-      extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" },
-    }
-    await config.integration.read(query)
-    expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled()
-    
-    const args = config.integration.client.findOneAndUpdate.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      }
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        name: "UPDATED",
-        age: 99
-      }
-    })
-    expect(args[2]).toEqual({
-      upsert: false
-    })
-  })
-
-  it("can parse nested objects with arrays", async () => {
-    const query = {
-      json: `{
-          "_id": {
-            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
-          }
-        },
-        {
-          "$set": {
-            "value": {
-              "data": [
-                { "cid": 1 },
-                { "cid": 2 },
-                { "nested": {
-                  "name": "test"
-                }}
-              ]
-            }
-          }
-        },
-        {
-          "upsert": true
-        }`,
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-    
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      }
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        value: {
-          data: [
-            { cid: 1 },
-            { cid: 2 },
-            { nested: {
-              name: "test"
-            }}
-          ]
-        },
-      },
-    })
-    expect(args[2]).toEqual({
-      upsert: true
-    })
-  })
-
-  it("ignores braces within strings when parsing nested objects", async () => {
-    const query = {
-      json: `{
-          "_id": {
-            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
-          }
-        },
-        {
-          "$set": {
-            "value": {
-              "data": [
-                { "cid": 1 },
-                { "cid": 2 },
-                { "nested": {
-                  "name": "te}st"
-                }}
-              ]
-            }
-          }
-        },
-        {
-          "upsert": true,
-          "extra": "ad\\"{\\"d"
-        }`,
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-    
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      }
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        value: {
-          data: [
-            { cid: 1 },
-            { cid: 2 },
-            { nested: {
-              name: "te}st"
-            }}
-          ]
-        },
-      },
-    })
-    expect(args[2]).toEqual({
-      upsert: true,
-      extra: "ad\"{\"d"
-    })
-  })
 })

package/src/integrations/tests/sql.spec.js

@@ -1,4 +1,4 @@
-const Sql = require("../base/sql").default
+const Sql = require("../base/sql")
 const { SqlClients } = require("../utils")
 
 const TABLE_NAME = "test"