@budibase/server 1.0.136 → 1.0.139

package/dist/utilities/fileSystem/index.js

@@ -12,7 +12,7 @@ const { budibaseTempDir } = require("../budibaseDir");
 const fs = require("fs");
 const { join } = require("path");
 const uuid = require("uuid/v4");
-const { doWithDB } = require("@budibase/backend-core/db");
+const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
 const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
@@ -135,13 +135,17 @@ exports.streamBackup = (appId) => __awaiter(void 0, void 0, void 0, function* ()
  * @return {*} either a readable stream or a string
  */
 exports.exportDB = (dbName, { stream, filter, exportName } = {}) => __awaiter(void 0, void 0, void 0, function* () {
+    // streaming a DB dump is a bit more complicated, can't close DB
+    if (stream) {
+        const db = dangerousGetDB(dbName);
+        const memStream = new MemoryStream();
+        memStream.on("end", () => __awaiter(void 0, void 0, void 0, function* () {
+            yield closeDB(db);
+        }));
+        db.dump(memStream, { filter });
+        return memStream;
+    }
     return doWithDB(dbName, (db) => __awaiter(void 0, void 0, void 0, function* () {
-        // Stream the dump if required
-        if (stream) {
-            const memStream = new MemoryStream();
-            db.dump(memStream, { filter });
-            return memStream;
-        }
         // Write the dump to file if required
         if (exportName) {
             const path = join(budibaseTempDir(), exportName);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.136",
+  "version": "1.0.139",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -68,10 +68,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.136",
-    "@budibase/client": "^1.0.136",
-    "@budibase/pro": "1.0.135",
-    "@budibase/string-templates": "^1.0.136",
+    "@budibase/backend-core": "^1.0.139",
+    "@budibase/client": "^1.0.139",
+    "@budibase/pro": "1.0.138",
+    "@budibase/string-templates": "^1.0.139",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -180,5 +180,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "6b93b7cdd2a28851863c2e8c2caddda0d7bd45b1"
+  "gitHead": "075cd99f94b0a9a696aeab48ae0ba1a386d2a322"
 }

package/src/automations/automationUtils.js

@@ -86,3 +86,15 @@ exports.substituteLoopStep = (hbsString, substitute) => {
 
   return hbsString
 }
+
+exports.stringSplit = value => {
+  if (value == null) {
+    return []
+  }
+  if (value.split("\n").length > 1) {
+    value = value.split("\n")
+  } else {
+    value = value.split(",")
+  }
+  return value
+}

package/src/integrations/microsoftSqlServer.ts

@@ -242,12 +242,10 @@ module MSSQLModule {
           if (typeof name !== "string") {
             continue
           }
-          const type: string = convertSqlType(def.DATA_TYPE)
-
           schema[name] = {
             autocolumn: !!autoColumns.find((col: string) => col === name),
             name: name,
-            type,
+            ...convertSqlType(def.DATA_TYPE),
           }
         }
         tables[tableName] = {

package/src/integrations/mysql.ts

@@ -15,6 +15,7 @@ import {
 } from "./utils"
 import { DatasourcePlus } from "./base/datasourcePlus"
 import dayjs from "dayjs"
+import { FieldTypes } from "../constants"
 const { NUMBER_REGEX } = require("../utilities")
 
 module MySQLModule {
@@ -101,7 +102,7 @@ module MySQLModule {
       }
       // if not a number, see if it is a date - important to do in this order as any
       // integer will be considered a valid date
-      else if (dayjs(binding).isValid()) {
+      else if (/^\d/.test(binding) && dayjs(binding).isValid()) {
         bindings[i] = dayjs(binding).toDate()
       }
     }
@@ -151,20 +152,24 @@ module MySQLModule {
 
     async internalQuery(
       query: SqlQuery,
-      connect: boolean = true
+      opts: { connect?: boolean; disableCoercion?: boolean } = {
+        connect: true,
+        disableCoercion: false,
+      }
     ): Promise<any[] | any> {
       try {
-        if (connect) {
+        if (opts?.connect) {
           await this.connect()
         }
+        const baseBindings = query.bindings || []
+        const bindings = opts?.disableCoercion
+          ? baseBindings
+          : bindingTypeCoerce(baseBindings)
         // Node MySQL is callback based, so we must wrap our call in a promise
-        const response = await this.client.query(
-          query.sql,
-          bindingTypeCoerce(query.bindings || [])
-        )
+        const response = await this.client.query(query.sql, bindings)
         return response[0]
       } finally {
-        if (connect) {
+        if (opts?.connect) {
           await this.disconnect()
         }
       }
@@ -179,7 +184,7 @@ module MySQLModule {
         // get the tables first
         const tablesResp = await this.internalQuery(
           { sql: "SHOW TABLES;" },
-          false
+          { connect: false }
         )
         const tableNames = tablesResp.map(
           (obj: any) =>
@@ -191,7 +196,7 @@ module MySQLModule {
           const schema: TableSchema = {}
           const descResp = await this.internalQuery(
             { sql: `DESCRIBE \`${tableName}\`;` },
-            false
+            { connect: false }
           )
           for (let column of descResp) {
             const columnName = column.Field
@@ -211,8 +216,8 @@ module MySQLModule {
             schema[columnName] = {
               name: columnName,
               autocolumn: isAuto,
-              type: convertSqlType(column.Type),
               constraints,
+              ...convertSqlType(column.Type),
             }
           }
           if (!tables[tableName]) {
@@ -254,7 +259,8 @@ module MySQLModule {
     async query(json: QueryJson) {
       await this.connect()
       try {
-        const queryFn = (query: any) => this.internalQuery(query, false)
+        const queryFn = (query: any) =>
+          this.internalQuery(query, { connect: false, disableCoercion: true })
         return await this.queryWithReturning(json, queryFn)
       } finally {
         await this.disconnect()

package/src/integrations/oracle.ts

@@ -279,9 +279,9 @@ module OracleModule {
       )
     }
 
-    private internalConvertType(column: OracleColumn): string {
+    private internalConvertType(column: OracleColumn): { type: string } {
       if (this.isBooleanType(column)) {
-        return FieldTypes.BOOLEAN
+        return { type: FieldTypes.BOOLEAN }
       }
 
       return convertSqlType(column.type)
@@ -328,7 +328,7 @@ module OracleModule {
               fieldSchema = {
                 autocolumn: OracleIntegration.isAutoColumn(oracleColumn),
                 name: columnName,
-                type: this.internalConvertType(oracleColumn),
+                ...this.internalConvertType(oracleColumn),
               }
               table.schema[columnName] = fieldSchema
             }

package/src/integrations/postgres.ts

@@ -227,7 +227,6 @@ module PostgresModule {
           }
         }
 
-        const type: string = convertSqlType(column.data_type)
         const identity = !!(
           column.identity_generation ||
           column.identity_start ||
@@ -242,7 +241,7 @@ module PostgresModule {
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,
-          type,
+          ...convertSqlType(column.data_type),
         }
       }
 

package/src/integrations/utils.ts

@@ -35,6 +35,9 @@ const SQL_DATE_TYPE_MAP = {
   date: FieldTypes.DATETIME,
 }
 
+const SQL_DATE_ONLY_TYPES = ["date"]
+const SQL_TIME_ONLY_TYPES = ["time"]
+
 const SQL_STRING_TYPE_MAP = {
   varchar: FieldTypes.STRING,
   char: FieldTypes.STRING,
@@ -85,9 +88,9 @@ export function breakExternalTableId(tableId: string | undefined) {
     return {}
   }
   const parts = tableId.split(DOUBLE_SEPARATOR)
-  let tableName = parts.pop()
+  let datasourceId = parts.shift()
   // if they need joined
-  let datasourceId = parts.join(DOUBLE_SEPARATOR)
+  let tableName = parts.join(DOUBLE_SEPARATOR)
   return { datasourceId, tableName }
 }
 
@@ -137,12 +140,20 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
 }
 
 export function convertSqlType(type: string) {
+  let foundType = FieldTypes.STRING
+  const lcType = type.toLowerCase()
   for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
-    if (type.toLowerCase().includes(external)) {
-      return internal
+    if (lcType.includes(external)) {
+      foundType = internal
+      break
     }
   }
-  return FieldTypes.STRING
+  const schema: any = { type: foundType }
+  if (foundType === FieldTypes.DATETIME) {
+    schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType)
+    schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType)
+  }
+  return schema
 }
 
 export function getSqlQuery(query: SqlQuery | string): SqlQuery {

package/src/threads/automation.js

@@ -100,10 +100,10 @@ class Orchestrator {
     let automation = this._automation
     const app = await this.getApp()
     let stopped = false
-    let loopStep
+    let loopStep = null
 
     let stepCount = 0
-    let loopStepNumber
+    let loopStepNumber = null
     let loopSteps = []
     for (let step of automation.definition.steps) {
       stepCount++
@@ -117,15 +117,17 @@ class Orchestrator {
       if (loopStep) {
         input = await processObject(loopStep.inputs, this._context)
       }
-      let iterations = loopStep ? input.binding.length : 1
+      let iterations = loopStep
+        ? Array.isArray(input.binding)
+          ? input.binding.length
+          : automationUtils.stringSplit(input.binding).length
+        : 1
       let iterationCount = 0
       for (let index = 0; index < iterations; index++) {
         let originalStepInput = cloneDeep(step.inputs)
 
         // Handle if the user has set a max iteration count or if it reaches the max limit set by us
         if (loopStep) {
-          // lets first of all handle the input
-          // if the input is array then use it, if it is a string then split it on every new line
           let newInput = await processObject(
             loopStep.inputs,
             cloneDeep(this._context)
@@ -134,9 +136,6 @@ class Orchestrator {
             newInput,
             loopStep.schema.inputs
           )
-          this._context.steps[loopStepNumber] = {
-            currentItem: newInput.binding[index],
-          }
 
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           if (
@@ -154,6 +153,20 @@ class Orchestrator {
             break
           }
 
+          let item
+          if (
+            typeof loopStep.inputs.binding === "string" &&
+            loopStep.inputs.option === "String"
+          ) {
+            item = automationUtils.stringSplit(newInput.binding)
+          } else {
+            item = loopStep.inputs.binding
+          }
+
+          this._context.steps[loopStepNumber] = {
+            currentItem: item[index],
+          }
+
           // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
           // Pretty hacky because we need to account for the row object
           for (let [key, value] of Object.entries(originalStepInput)) {
@@ -286,18 +299,16 @@ class Orchestrator {
 
 module.exports = (input, callback) => {
   const appId = input.data.event.appId
-  doInAppContext(appId, () => {
+  doInAppContext(appId, async () => {
     const automationOrchestrator = new Orchestrator(
       input.data.automation,
       input.data.event
     )
-    automationOrchestrator
-      .execute()
-      .then(response => {
-        callback(null, response)
-      })
-      .catch(err => {
-        callback(err)
-      })
+    try {
+      const response = await automationOrchestrator.execute()
+      callback(null, response)
+    } catch (err) {
+      callback(err)
+    }
   })
 }

package/src/threads/query.js

@@ -191,14 +191,13 @@ class QueryRunner {
 }
 
 module.exports = (input, callback) => {
-  doInAppContext(input.appId, () => {
+  doInAppContext(input.appId, async () => {
     const Runner = new QueryRunner(input)
-    Runner.execute()
-      .then(response => {
-        callback(null, response)
-      })
-      .catch(err => {
-        callback(err)
-      })
+    try {
+      const response = await Runner.execute()
+      callback(null, response)
+    } catch (err) {
+      callback(err)
+    }
   })
 }

package/src/utilities/fileSystem/index.js

@@ -2,7 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
 const uuid = require("uuid/v4")
-const { doWithDB } = require("@budibase/backend-core/db")
+const {
+  doWithDB,
+  dangerousGetDB,
+  closeDB,
+} = require("@budibase/backend-core/db")
 const { ObjectStoreBuckets } = require("../../constants")
 const {
   upload,
@@ -151,14 +155,18 @@ exports.streamBackup = async appId => {
  * @return {*} either a readable stream or a string
  */
 exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => {
-  return doWithDB(dbName, async db => {
-    // Stream the dump if required
-    if (stream) {
-      const memStream = new MemoryStream()
-      db.dump(memStream, { filter })
-      return memStream
-    }
+  // streaming a DB dump is a bit more complicated, can't close DB
+  if (stream) {
+    const db = dangerousGetDB(dbName)
+    const memStream = new MemoryStream()
+    memStream.on("end", async () => {
+      await closeDB(db)
+    })
+    db.dump(memStream, { filter })
+    return memStream
+  }
 
+  return doWithDB(dbName, async db => {
     // Write the dump to file if required
     if (exportName) {
       const path = join(budibaseTempDir(), exportName)