@budibase/server 1.2.41-alpha.4 → 1.2.42

package/dist/utilities/fileSystem/index.js

@@ -14,7 +14,7 @@ const { join } = require("path");
 const uuid = require("uuid/v4");
 const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
 const env = require("../../environment");
 const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
@@ -289,6 +289,5 @@ exports.cleanup = appIds => {
 exports.upload = upload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
-exports.deleteFiles = deleteFiles;
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.2.41-alpha.4",
+  "version": "1.2.42",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.2.41-alpha.4",
-    "@budibase/client": "1.2.41-alpha.4",
-    "@budibase/pro": "1.2.41-alpha.3",
-    "@budibase/string-templates": "1.2.41-alpha.4",
-    "@budibase/types": "1.2.41-alpha.4",
+    "@budibase/backend-core": "^1.2.42",
+    "@budibase/client": "^1.2.42",
+    "@budibase/pro": "1.2.41",
+    "@budibase/string-templates": "^1.2.42",
+    "@budibase/types": "^1.2.42",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -106,7 +106,7 @@
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
     "jimp": "0.16.1",
-    "joi": "17.6.0",
+    "joi": "17.2.1",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
     "knex": "0.95.15",
@@ -195,5 +195,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "1d79a9f3506ed748ab167afe925a85065d845437"
+  "gitHead": "111226abbd5e491027931e272d2267f25057fb13"
 }

package/src/api/controllers/row/internalSearch.js

@@ -1,5 +1,4 @@
 const { SearchIndexes } = require("../../../db/utils")
-const { removeKeyNumbering } = require("./utils")
 const fetch = require("node-fetch")
 const { getCouchInfo } = require("@budibase/backend-core/db")
 const { getAppId } = require("@budibase/backend-core/context")
@@ -21,8 +20,6 @@ class QueryBuilder {
       notEmpty: {},
       oneOf: {},
       contains: {},
-      notContains: {},
-      containsAny: {},
       ...base,
     }
     this.limit = 50
@@ -128,16 +125,6 @@ class QueryBuilder {
     return this
   }
 
-  addNotContains(key, value) {
-    this.query.notContains[key] = value
-    return this
-  }
-
-  addContainsAny(key, value) {
-    this.query.containsAny[key] = value
-    return this
-  }
-
   /**
    * Preprocesses a value before going into a lucene search.
    * Transforms strings to lowercase and wraps strings and bools in quotes.
@@ -183,29 +170,11 @@ class QueryBuilder {
       return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
     }
 
-    const contains = (key, value, mode = "AND") => {
-      if (Array.isArray(value) && value.length === 0) {
+    const contains = (key, value) => {
+      if (!value && value !== 0) {
         return null
       }
-      if (!Array.isArray(value)) {
-        return `${key}:${value}`
-      }
-      let statement = `${builder.preprocess(value[0], { escape: true })}`
-      for (let i = 1; i < value.length; i++) {
-        statement += ` ${mode} ${builder.preprocess(value[i], {
-          escape: true,
-        })}`
-      }
-      return `${key}:(${statement})`
-    }
-
-    const notContains = (key, value) => {
-      const allPrefix = allOr === "" ? "*:* AND" : ""
-      return allPrefix + "NOT " + contains(key, value)
-    }
-
-    const containsAny = (key, value) => {
-      return contains(key, value, "OR")
+      return `${key}:${builder.preprocess(value, { escape: true })}`
     }
 
     const oneOf = (key, value) => {
@@ -228,8 +197,6 @@ class QueryBuilder {
 
     function build(structure, queryFn) {
       for (let [key, value] of Object.entries(structure)) {
-        // check for new format - remove numbering if needed
-        key = removeKeyNumbering(key)
         key = builder.preprocess(key.replace(/ /g, "_"), {
           escape: true,
         })
@@ -308,12 +275,6 @@ class QueryBuilder {
     if (this.query.contains) {
       build(this.query.contains, contains)
     }
-    if (this.query.notContains) {
-      build(this.query.notContains, notContains)
-    }
-    if (this.query.containsAny) {
-      build(this.query.containsAny, containsAny)
-    }
     // make sure table ID is always added as an AND
     if (tableId) {
       query = `(${query})`
@@ -349,9 +310,6 @@ class QueryBuilder {
   }
 }
 
-// exported for unit testing
-exports.QueryBuilder = QueryBuilder
-
 /**
  * Executes a lucene search query.
  * @param url The query URL

package/src/api/controllers/row/utils.js

@@ -3,9 +3,8 @@ const { cloneDeep } = require("lodash/fp")
 const { InternalTables } = require("../../../db/utils")
 const userController = require("../user")
 const { FieldTypes } = require("../../../constants")
+const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
-const { makeExternalQuery } = require("../../../integrations/base/query")
-const { removeKeyNumbering } = require("../../../integrations/base/utils")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value) {
@@ -17,8 +16,6 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-exports.removeKeyNumbering = removeKeyNumbering
-
 exports.getDatasourceAndQuery = async json => {
   const datasourceId = json.endpoint.datasourceId
   const db = getAppDB()

package/src/api/controllers/static/index.ts

@@ -12,7 +12,7 @@ const {
 } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
 const { clientLibraryPath } = require("../../../utilities")
-const { upload, deleteFiles } = require("../../../utilities/fileSystem")
+const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentType } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
@@ -97,10 +97,6 @@ export const uploadFile = async function (ctx: any) {
   ctx.body = await Promise.all(uploads)
 }
 
-export const deleteObjects = async function (ctx: any) {
-  ctx.body = await deleteFiles(ObjectStoreBuckets.APPS, ctx.request.body.keys)
-}
-
 export const serveApp = async function (ctx: any) {
   const db = getAppDB({ skip_setup: true })
   const appInfo = await db.get(DocumentType.APP_METADATA)

package/src/api/controllers/table/external.js

@@ -14,7 +14,7 @@ const {
   FieldTypes,
   RelationshipTypes,
 } = require("../../../constants")
-const { makeExternalQuery } = require("../../../integrations/base/query")
+const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { cloneDeep } = require("lodash/fp")
 const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")

package/src/api/routes/static.ts

@@ -38,11 +38,6 @@ router
   // TODO: for now this builder endpoint is not authorized/secured, will need to be
   .get("/builder/:file*", controller.serveBuilder)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
-  .post(
-    "/api/attachments/delete",
-    authorized(BUILDER),
-    controller.deleteObjects
-  )
   .post("/api/beta/:feature", controller.toggleBetaUiFeature)
   .post(
     "/api/attachments/:tableId/upload",
@@ -50,12 +45,6 @@ router
     authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
     controller.uploadFile
   )
-  .post(
-    "/api/attachments/:tableId/delete",
-    paramResource("tableId"),
-    authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
-    controller.deleteObjects
-  )
   .get("/:appId/:path*", controller.serveApp)
   .get("/app/:appUrl/:path*", controller.serveApp)
   .post(

package/src/automations/logging/index.ts

@@ -23,9 +23,6 @@ export async function checkAppMetadata(apps: App[]) {
     for (let [key, errors] of Object.entries(metadata.automationErrors)) {
       const updated = []
       for (let error of errors) {
-        if (!error) {
-          continue
-        }
         const startDate = error.split(dbUtils.SEPARATOR)[2]
         if (startDate > maxStartDate) {
           updated.push(error)

package/src/integrations/base/sql.ts

@@ -10,7 +10,6 @@ import { QueryOptions } from "../../definitions/datasource"
 import { isIsoDateString, SqlClient } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import environment from "../../environment"
-import { removeKeyNumbering } from "./utils"
 
 const envLimit = environment.SQL_MAX_ROWS
   ? parseInt(environment.SQL_MAX_ROWS)
@@ -134,13 +133,12 @@ class InternalBuilder {
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const updatedKey = removeKeyNumbering(key)
-        const isRelationshipField = updatedKey.includes(".")
+        const isRelationshipField = key.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${updatedKey}`, value)
+          fn(`${opts.tableName}.${key}`, value)
         }
         if (opts.relationship && isRelationshipField) {
-          fn(updatedKey, value)
+          fn(key, value)
         }
       }
     }
@@ -159,61 +157,6 @@ class InternalBuilder {
       }
     }
 
-    const contains = (mode: object, any: boolean = false) => {
-      const fnc = allOr ? "orWhere" : "where"
-      const rawFnc = `${fnc}Raw`
-      const not = mode === filters?.notContains ? "NOT " : ""
-      function stringifyArray(value: Array<any>, quoteStyle = '"'): string {
-        for (let i in value) {
-          if (typeof value[i] === "string") {
-            value[i] = `${quoteStyle}${value[i]}${quoteStyle}`
-          }
-        }
-        return `[${value.join(",")}]`
-      }
-      if (this.client === SqlClient.POSTGRES) {
-        iterate(mode, (key: string, value: Array<any>) => {
-          const wrap = any ? "" : "'"
-          const containsOp = any ? "\\?| array" : "@>"
-          const fieldNames = key.split(/\./g)
-          const tableName = fieldNames[0]
-          const columnName = fieldNames[1]
-          // @ts-ignore
-          query = query[rawFnc](
-            `${not}"${tableName}"."${columnName}"::jsonb ${containsOp} ${wrap}${stringifyArray(
-              value,
-              any ? "'" : '"'
-            )}${wrap}`
-          )
-        })
-      } else if (this.client === SqlClient.MY_SQL) {
-        const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
-        iterate(mode, (key: string, value: Array<any>) => {
-          // @ts-ignore
-          query = query[rawFnc](
-            `${not}${jsonFnc}(${key}, '${stringifyArray(value)}')`
-          )
-        })
-      } else {
-        const andOr = mode === filters?.containsAny ? " OR " : " AND "
-        iterate(mode, (key: string, value: Array<any>) => {
-          let statement = ""
-          for (let i in value) {
-            if (typeof value[i] === "string") {
-              value[i] = `%"${value[i]}"%`
-            } else {
-              value[i] = `%${value[i]}%`
-            }
-            statement +=
-              (statement ? andOr : "") +
-              `LOWER(${likeKey(this.client, key)}) LIKE ?`
-          }
-          // @ts-ignore
-          query = query[rawFnc](`${not}(${statement})`, value)
-        })
-      }
-    }
-
     if (!filters) {
       return query
     }
@@ -284,13 +227,32 @@ class InternalBuilder {
       })
     }
     if (filters.contains) {
-      contains(filters.contains)
-    }
-    if (filters.notContains) {
-      contains(filters.notContains)
-    }
-    if (filters.containsAny) {
-      contains(filters.containsAny, true)
+      const fnc = allOr ? "orWhere" : "where"
+      const rawFnc = `${fnc}Raw`
+      if (this.client === SqlClient.POSTGRES) {
+        iterate(filters.contains, (key: string, value: any) => {
+          const fieldNames = key.split(/\./g)
+          const tableName = fieldNames[0]
+          const columnName = fieldNames[1]
+          if (typeof value === "string") {
+            value = `"${value}"`
+          }
+          // @ts-ignore
+          query = query[rawFnc](
+            `"${tableName}"."${columnName}"::jsonb @> '[${value}]'`
+          )
+        })
+      } else if (this.client === SqlClient.MY_SQL) {
+        iterate(filters.contains, (key: string, value: any) => {
+          if (typeof value === "string") {
+            value = `"${value}"`
+          }
+          // @ts-ignore
+          query = query[rawFnc](`JSON_CONTAINS(${key}, '${value}')`)
+        })
+      } else {
+        iterate(filters.contains, like)
+      }
     }
     return query
   }
@@ -620,3 +582,4 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 }
 
 export default SqlQueryBuilder
+module.exports = SqlQueryBuilder

package/src/integrations/base/utils.ts

@@ -1,12 +1,21 @@
-const QUERY_START_REGEX = /\d[0-9]*:/g
+import { QueryJson, Datasource } from "@budibase/types"
 
-export function removeKeyNumbering(key: any): string {
-  if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
-    const parts = key.split(":")
-    // remove the number
-    parts.shift()
-    return parts.join(":")
-  } else {
-    return key
+module DatasourceUtils {
+  const { integrations } = require("../index")
+
+  export async function makeExternalQuery(
+    datasource: Datasource,
+    json: QueryJson
+  ) {
+    const Integration = integrations[datasource.source]
+    // query is the opinionated function
+    if (Integration.prototype.query) {
+      const integration = new Integration(datasource.config)
+      return integration.query(json)
+    } else {
+      throw "Datasource does not support query."
+    }
   }
+
+  module.exports.makeExternalQuery = makeExternalQuery
 }

package/src/integrations/microsoftSqlServer.ts

@@ -16,10 +16,10 @@ import {
   finaliseExternalTables,
   SqlClient,
 } from "./utils"
-import Sql from "./base/sql"
 
 module MSSQLModule {
   const sqlServer = require("mssql")
+  const Sql = require("./base/sql")
   const DEFAULT_SCHEMA = "dbo"
 
   interface MSSQLConfig {
@@ -97,8 +97,7 @@ module MSSQLModule {
   class SqlServerIntegration extends Sql implements DatasourcePlus {
     private readonly config: MSSQLConfig
     private index: number = 0
-    private readonly pool: any
-    private client: any
+    static pool: any
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 

package/src/integrations/mongodb.ts

@@ -92,15 +92,12 @@ module MongoDBModule {
           if (json[field] instanceof Object) {
             json[field] = self.createObjectIds(json[field])
           }
-          if (
-            (field === "_id" || field?.startsWith("$")) &&
-            typeof json[field] === "string"
-          ) {
-            const id = json[field].match(
+          if (field === "_id" && typeof json[field] === "string") {
+            const id = json["_id"].match(
               /(?<=objectid\(['"]).*(?=['"]\))/gi
             )?.[0]
             if (id) {
-              json[field] = ObjectID.createFromHexString(id)
+              json["_id"] = ObjectID.createFromHexString(id)
             }
           }
         }
@@ -117,31 +114,10 @@ module MongoDBModule {
     }
 
     parseQueryParams(params: string, mode: string) {
-      let queryParams = []
-      let openCount = 0
-      let inQuotes = false
-      let i = 0
-      let startIndex = 0
-      for (let c of params) {
-        if (c === '"' && i > 0 && params[i - 1] !== "\\") {
-          inQuotes = !inQuotes
-        }
-        if (c === "{" && !inQuotes) {
-          openCount++
-          if (openCount === 1) {
-            startIndex = i
-          }
-        } else if (c === "}" && !inQuotes) {
-          if (openCount === 1) {
-            queryParams.push(JSON.parse(params.substring(startIndex, i + 1)))
-          }
-          openCount--
-        }
-        i++
-      }
-      let group1 = queryParams[0] ?? {}
-      let group2 = queryParams[1] ?? {}
-      let group3 = queryParams[2] ?? {}
+      let queryParams = params.split(/(?<=}),[\n\s]*(?={)/g)
+      let group1 = queryParams[0] ? JSON.parse(queryParams[0]) : {}
+      let group2 = queryParams[1] ? JSON.parse(queryParams[1]) : {}
+      let group3 = queryParams[2] ? JSON.parse(queryParams[2]) : {}
       if (mode === "update") {
         return {
           filter: group1,
@@ -200,10 +176,7 @@ module MongoDBModule {
             return await collection.findOne(json)
           }
           case "findOneAndUpdate": {
-            if (typeof query.json === "string") {
-              json = this.parseQueryParams(query.json, "update")
-            }
-            let findAndUpdateJson = this.createObjectIds(json) as {
+            let findAndUpdateJson = json as {
               filter: FilterQuery<any>
               update: UpdateQuery<any>
               options: FindOneAndUpdateOption<any>

package/src/integrations/mysql.ts

@@ -17,10 +17,10 @@ import {
 } from "./utils"
 import dayjs from "dayjs"
 const { NUMBER_REGEX } = require("../utilities")
-import Sql from "./base/sql"
 
 module MySQLModule {
   const mysql = require("mysql2/promise")
+  const Sql = require("./base/sql")
 
   interface MySQLConfig {
     host: string

package/src/integrations/postgres.ts

@@ -14,10 +14,10 @@ import {
   finaliseExternalTables,
   SqlClient,
 } from "./utils"
-import Sql from "./base/sql"
 
 module PostgresModule {
   const { Client, types } = require("pg")
+  const Sql = require("./base/sql")
   const { escapeDangerousCharacters } = require("../utilities")
 
   // Return "date" and "timestamp" types as plain strings.
@@ -117,7 +117,6 @@ module PostgresModule {
     private readonly client: any
     private readonly config: PostgresConfig
     private index: number = 1
-    private open: boolean
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}