@budibase/server 1.0.219 → 1.0.220-alpha.0

package/src/environment.js

@@ -74,10 +74,13 @@ module.exports = {
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
+  BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
+  BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
+  DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,

package/src/integrations/airtable.ts

@@ -18,6 +18,7 @@ module AirtableModule {
     description:
       "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.",
     friendlyName: "Airtable",
+    type: "Spreadsheet",
     datasource: {
       apiKey: {
         type: DatasourceFieldTypes.PASSWORD,

package/src/integrations/arangodb.ts

@@ -19,6 +19,7 @@ module ArangoModule {
   const SCHEMA: Integration = {
     docs: "https://github.com/arangodb/arangojs",
     friendlyName: "ArangoDB",
+    type: "Non-relational",
     description:
       "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
     datasource: {

package/src/integrations/base/sql.ts

@@ -89,6 +89,32 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
   return filters
 }
 
+function generateSelectStatement(
+  json: QueryJson,
+  knex: Knex
+): (string | Knex.Raw)[] {
+  const { resource, meta } = json
+  const schema = meta?.table?.schema
+  return resource.fields.map(field => {
+    const fieldNames = field.split(/\./g)
+    const tableName = fieldNames[0]
+    const columnName = fieldNames[1]
+    if (
+      columnName &&
+      schema?.[columnName] &&
+      knex.client.config.client === SqlClients.POSTGRES
+    ) {
+      const externalType = schema[columnName].externalType
+      if (externalType?.includes("money")) {
+        return knex.raw(
+          `"${tableName}"."${columnName}"::money::numeric as "${field}"`
+        )
+      }
+    }
+    return `${field} as ${field}`
+  })
+}
+
 class InternalBuilder {
   private readonly client: string
 
@@ -216,9 +242,7 @@ class InternalBuilder {
   }
 
   addRelationships(
-    knex: Knex,
     query: KnexQuery,
-    fields: string | string[],
     fromTable: string,
     relationships: RelationshipsJson[] | undefined
   ): KnexQuery {
@@ -323,12 +347,12 @@ class InternalBuilder {
     if (!resource) {
       resource = { fields: [] }
     }
-    let selectStatement: string | string[] = "*"
+    let selectStatement: string | (string | Knex.Raw)[] = "*"
     // handle select
     if (resource.fields && resource.fields.length > 0) {
       // select the resources as the format "table.columnName" - this is what is provided
       // by the resource builder further up
-      selectStatement = resource.fields.map(field => `${field} as ${field}`)
+      selectStatement = generateSelectStatement(json, knex)
     }
     let foundLimit = limit || BASE_LIMIT
     // handle pagination
@@ -363,13 +387,7 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    query = this.addRelationships(
-      knex,
-      preQuery,
-      selectStatement,
-      tableName,
-      relationships
-    )
+    query = this.addRelationships(preQuery, tableName, relationships)
     return this.addFilters(query, filters, { relationship: true })
   }
 

package/src/integrations/couchdb.ts

@@ -16,6 +16,7 @@ module CouchDBModule {
   const SCHEMA: Integration = {
     docs: "https://docs.couchdb.org/en/stable/",
     friendlyName: "CouchDB",
+    type: "Non-relational",
     description:
       "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
     datasource: {

package/src/integrations/dynamodb.ts

@@ -21,6 +21,7 @@ module DynamoModule {
     description:
       "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
     friendlyName: "DynamoDB",
+    type: "Non-relational",
     datasource: {
       region: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/elasticsearch.ts

@@ -17,6 +17,7 @@ module ElasticsearchModule {
     description:
       "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
     friendlyName: "ElasticSearch",
+    type: "Non-relational",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,

package/src/integrations/firebase.ts

@@ -16,6 +16,7 @@ module Firebase {
   const SCHEMA: Integration = {
     docs: "https://firebase.google.com/docs/firestore/quickstart",
     friendlyName: "Firestore",
+    type: "Non-relational",
     description:
       "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
     datasource: {

package/src/integrations/googlesheets.ts

@@ -49,6 +49,7 @@ module GoogleSheetsModule {
     description:
       "Create and collaborate on online spreadsheets in real-time and from any device. ",
     friendlyName: "Google Sheets",
+    type: "Spreadsheet",
     datasource: {
       spreadsheetId: {
         display: "Google Sheet URL",

package/src/integrations/index.ts

@@ -12,6 +12,7 @@ const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const firebase = require("./firebase")
 const redis = require("./redis")
+const snowflake = require("./snowflake")
 const { SourceNames } = require("../definitions/datasource")
 const environment = require("../environment")
 
@@ -29,6 +30,7 @@ const DEFINITIONS = {
   [SourceNames.REST]: rest.schema,
   [SourceNames.FIRESTORE]: firebase.schema,
   [SourceNames.REDIS]: redis.schema,
+  [SourceNames.SNOWFLAKE]: snowflake.schema,
 }
 
 const INTEGRATIONS = {
@@ -46,6 +48,8 @@ const INTEGRATIONS = {
   [SourceNames.FIRESTORE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
   [SourceNames.REDIS]: redis.integration,
+  [SourceNames.FIREBASE]: firebase.integration,
+  [SourceNames.SNOWFLAKE]: snowflake.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/microsoftSqlServer.ts

@@ -44,6 +44,7 @@ module MSSQLModule {
     description:
       "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
     friendlyName: "MS SQL Server",
+    type: "Relational",
     datasource: {
       user: {
         type: DatasourceFieldTypes.STRING,
@@ -246,6 +247,7 @@ module MSSQLModule {
             autocolumn: !!autoColumns.find((col: string) => col === name),
             name: name,
             ...convertSqlType(def.DATA_TYPE),
+            externalType: def.DATA_TYPE,
           }
         }
         tables[tableName] = {

package/src/integrations/mongodb.ts

@@ -24,6 +24,7 @@ module MongoDBModule {
   const SCHEMA: Integration = {
     docs: "https://github.com/mongodb/node-mongodb-native",
     friendlyName: "MongoDB",
+    type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
     datasource: {

package/src/integrations/mysql.ts

@@ -36,6 +36,7 @@ module MySQLModule {
     docs: "https://github.com/sidorares/node-mysql2",
     plus: true,
     friendlyName: "MySQL",
+    type: "Relational",
     description:
       "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
     datasource: {
@@ -232,6 +233,7 @@ module MySQLModule {
               autocolumn: isAuto,
               constraints,
               ...convertSqlType(column.Type),
+              externalType: column.Type,
             }
           }
           if (!tables[tableName]) {

package/src/integrations/oracle.ts

@@ -40,6 +40,7 @@ module OracleModule {
     docs: "https://github.com/oracle/node-oracledb",
     plus: true,
     friendlyName: "Oracle",
+    type: "Relational",
     description:
       "Oracle Database is an object-relational database management system developed by Oracle Corporation",
     datasource: {

package/src/integrations/postgres.ts

@@ -47,6 +47,7 @@ module PostgresModule {
     docs: "https://node-postgres.com",
     plus: true,
     friendlyName: "PostgreSQL",
+    type: "Relational",
     description:
       "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
     datasource: {
@@ -271,6 +272,7 @@ module PostgresModule {
             autocolumn: isAuto,
             name: columnName,
             ...convertSqlType(column.data_type),
+            externalType: column.data_type,
           }
         }
 

package/src/integrations/queries/sql.ts

@@ -9,7 +9,9 @@ export function enrichQueryFields(
   parameters = {}
 ) {
   const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {}
-
+  if (!fields || !parameters) {
+    return enrichedQuery
+  }
   // enrich the fields with dynamic parameters
   for (let key of Object.keys(fields)) {
     if (fields[key] == null) {

package/src/integrations/redis.ts

@@ -17,6 +17,7 @@ module RedisModule {
     docs: "https://redis.io/docs/",
     description: "",
     friendlyName: "Redis",
+    type: "Non-relational",
     datasource: {
       host: {
         type: "string",

package/src/integrations/rest.ts

@@ -64,6 +64,7 @@ module RestModule {
     description:
       "With the REST API datasource, you can connect, query and pull data from multiple REST APIs. You can then use the retrieved data to build apps.",
     friendlyName: "REST API",
+    type: "API",
     datasource: {
       url: {
         type: DatasourceFieldTypes.STRING,
@@ -286,7 +287,7 @@ module RestModule {
           input.body = form
           break
         case BodyTypes.XML:
-          if (object != null) {
+          if (object != null && Object.keys(object).length) {
             string = new XmlBuilder().buildObject(object)
           }
           input.body = string

package/src/integrations/s3.ts

@@ -17,6 +17,7 @@ module S3Module {
     description:
       "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
     friendlyName: "Amazon S3",
+    type: "Object store",
     datasource: {
       region: {
         type: "string",

package/src/integrations/snowflake.ts

@@ -0,0 +1,99 @@
+import { Integration, QueryTypes, SqlQuery } from "../definitions/datasource"
+import { Snowflake } from "snowflake-promise"
+
+module SnowflakeModule {
+  interface SnowflakeConfig {
+    account: string
+    username: string
+    password: string
+    warehouse: string
+    database: string
+    schema: string
+  }
+
+  const SCHEMA: Integration = {
+    docs: "https://developers.snowflake.com/",
+    description:
+      "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
+    friendlyName: "Snowflake",
+    type: "Relational",
+    datasource: {
+      account: {
+        type: "string",
+        required: true,
+      },
+      username: {
+        type: "string",
+        required: true,
+      },
+      password: {
+        type: "password",
+        required: true,
+      },
+      warehouse: {
+        type: "string",
+        required: true,
+      },
+      database: {
+        type: "string",
+        required: true,
+      },
+      schema: {
+        type: "string",
+        required: true,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.SQL,
+      },
+      read: {
+        type: QueryTypes.SQL,
+      },
+      update: {
+        type: QueryTypes.SQL,
+      },
+      delete: {
+        type: QueryTypes.SQL,
+      },
+    },
+  }
+
+  class SnowflakeIntegration {
+    private client: Snowflake
+
+    constructor(config: SnowflakeConfig) {
+      this.client = new Snowflake(config)
+    }
+
+    async internalQuery(query: SqlQuery) {
+      await this.client.connect()
+      try {
+        return await this.client.execute(query.sql)
+      } catch (err: any) {
+        throw err?.message.split(":")[1] || err?.message
+      }
+    }
+
+    async create(query: SqlQuery) {
+      return this.internalQuery(query)
+    }
+
+    async read(query: SqlQuery) {
+      return this.internalQuery(query)
+    }
+
+    async update(query: SqlQuery) {
+      return this.internalQuery(query)
+    }
+
+    async delete(query: SqlQuery) {
+      return this.internalQuery(query)
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: SnowflakeIntegration,
+  }
+}

package/src/integrations/tests/rest.spec.js

@@ -155,12 +155,27 @@ describe("REST Integration", () => {
       expect(output.headers["Content-Type"]).toEqual("application/json")
     })
 
-    it("should allow XML", () => {
+    it("should allow raw XML", () => {
+      const output = config.integration.addBody("xml", "<a>1</a><b>2</b>", {})
+      expect(output.body.includes("<a>1</a>")).toEqual(true)
+      expect(output.body.includes("<b>2</b>")).toEqual(true)
+      expect(output.headers["Content-Type"]).toEqual("application/xml")
+    })
+
+    it("should allow a valid js object and parse the contents to xml", () => {
       const output = config.integration.addBody("xml", input, {})
       expect(output.body.includes("<a>1</a>")).toEqual(true)
       expect(output.body.includes("<b>2</b>")).toEqual(true)
       expect(output.headers["Content-Type"]).toEqual("application/xml")
     })
+
+    it("should allow a valid json string and parse the contents to xml", () => {
+      const output = config.integration.addBody("xml", JSON.stringify(input), {})
+      expect(output.body.includes("<a>1</a>")).toEqual(true)
+      expect(output.body.includes("<b>2</b>")).toEqual(true)
+      expect(output.headers["Content-Type"]).toEqual("application/xml")
+    })
+
   })
 
   describe("response", () => {

package/src/migrations/tests/index.spec.ts

@@ -93,8 +93,24 @@ describe("migrations", () => {
       await clearMigrations()
       const appId = config.prodAppId
       const roles = { [appId]: "role_12345" }
-      await config.createUser(undefined, undefined, false, true, roles) // admin only
-      await config.createUser(undefined, undefined, false, false, roles) // non admin non builder
+      await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        true,
+        roles
+      ) // admin only
+      await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        false,
+        roles
+      ) // non admin non builder
       await config.createTable()
       await config.createRow()
       await config.createRow()

package/src/module.d.ts

@@ -8,3 +8,5 @@ declare module "@budibase/backend-core/constants"
 declare module "@budibase/backend-core/auth"
 declare module "@budibase/backend-core/sessions"
 declare module "@budibase/backend-core/encryption"
+declare module "@budibase/backend-core/utils"
+declare module "@budibase/backend-core/redis"

package/src/tests/utilities/TestConfiguration.js

@@ -28,6 +28,8 @@ const { encrypt } = require("@budibase/backend-core/encryption")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
+const FIRSTNAME = "Barbara"
+const LASTNAME = "Barbington"
 const CSRF_TOKEN = "e3727778-7af0-4226-b5eb-f43cbe60a306"
 
 class TestConfiguration {
@@ -59,6 +61,15 @@ class TestConfiguration {
     return this.prodAppId
   }
 
+  getUserDetails() {
+    return {
+      globalId: GLOBAL_USER_ID,
+      email: EMAIL,
+      firstName: FIRSTNAME,
+      lastName: LASTNAME,
+    }
+  }
+
   async doInContext(appId, task) {
     if (!appId) {
       appId = this.appId
@@ -118,6 +129,8 @@ class TestConfiguration {
   // USER / AUTH
   async globalUser({
     id = GLOBAL_USER_ID,
+    firstName = FIRSTNAME,
+    lastName = LASTNAME,
     builder = true,
     admin = false,
     email = EMAIL,
@@ -135,6 +148,8 @@ class TestConfiguration {
         ...existing,
         roles: roles || {},
         tenantId: TENANT_ID,
+        firstName,
+        lastName,
       }
       await createASession(id, {
         sessionId: "sessionid",
@@ -161,6 +176,8 @@ class TestConfiguration {
 
   async createUser(
     id = null,
+    firstName = FIRSTNAME,
+    lastName = LASTNAME,
     email = EMAIL,
     builder = true,
     admin = false,
@@ -169,6 +186,8 @@ class TestConfiguration {
     const globalId = !id ? `us_${Math.random()}` : `us_${id}`
     const resp = await this.globalUser({
       id: globalId,
+      firstName,
+      lastName,
       email,
       builder,
       admin,
@@ -520,14 +539,14 @@ class TestConfiguration {
 
   // QUERY
 
-  async previewQuery(request, config, datasource, fields) {
+  async previewQuery(request, config, datasource, fields, params, verb) {
     return request
       .post(`/api/queries/preview`)
       .send({
         datasourceId: datasource._id,
-        parameters: {},
+        parameters: params || {},
         fields,
-        queryVerb: "read",
+        queryVerb: verb || "read",
         name: datasource.name,
       })
       .set(config.defaultHeaders())

package/src/threads/automation.js

@@ -9,11 +9,12 @@ const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const { AutomationErrors, LoopStepTypes } = require("../constants")
+const { storeLog } = require("../automations/logging")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 
 const CRON_STEP_ID = triggerDefs.CRON.stepId
-const STOPPED_STATUS = { success: false, status: "STOPPED" }
+const STOPPED_STATUS = { success: true, status: "STOPPED" }
 const { cloneDeep } = require("lodash/fp")
 const env = require("../environment")
 
@@ -218,7 +219,7 @@ class Orchestrator {
             }
           }
           if (
-            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
+            index === env.AUTOMATION_MAX_ITERATION ||
             index === parseInt(loopStep.inputs.iterations)
           ) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
@@ -275,7 +276,7 @@ class Orchestrator {
           this._context.steps[stepCount] = outputs
           // if filter causes us to stop execution don't break the loop, set a var
           // so that we can finish iterating through the steps and record that it stopped
-          if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+          if (step.stepId === FILTER_STEP_ID && !outputs.result) {
             stopped = true
             this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
               ...outputs,
@@ -325,6 +326,8 @@ class Orchestrator {
       }
     }
 
+    // store the logs for the automation run
+    await storeLog(this._automation, this.executionOutput)
     return this.executionOutput
   }
 }

package/src/threads/index.ts

@@ -77,7 +77,7 @@ export class Thread {
     })
   }
 
-  static shutdown() {
+  static stopThreads() {
     return new Promise<void>(resolve => {
       if (Thread.workerRefs.length === 0) {
         resolve()
@@ -95,4 +95,8 @@ export class Thread {
       Thread.workerRefs = []
     })
   }
+
+  static async shutdown() {
+    await Thread.stopThreads()
+  }
 }