@budibase/server 0.9.185-alpha.2 → 0.9.185-alpha.20

package/src/automations/bullboard.js

@@ -1,6 +1,6 @@
-const { createBullBoard } = require("bull-board")
-const { BullAdapter } = require("bull-board/bullAdapter")
-const express = require("express")
+const { createBullBoard } = require("@bull-board/api")
+const { BullAdapter } = require("@bull-board/api/bullAdapter")
+const { KoaAdapter } = require("@bull-board/koa")
 const env = require("../environment")
 const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
@@ -9,23 +9,40 @@ const { JobQueues } = require("../constants")
 const { utils } = require("@budibase/auth/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
-const redisConfig = redisProtocolUrl || { redis: opts }
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, redisConfig)
+const CLEANUP_PERIOD_MS = 60 * 1000
+const queueConfig = redisProtocolUrl || { redis: opts }
+let cleanupInternal = null
 
-exports.pathPrefix = "/bulladmin"
+let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
+
+async function cleanup() {
+  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
+}
+
+const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  const expressApp = express()
+  // cleanup the events every 5 minutes
+  if (!cleanupInternal) {
+    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
+    // fire off an initial cleanup
+    cleanup().catch(err => {
+      console.error(`Unable to cleanup automation queue initially - ${err}`)
+    })
+  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
+  const serverAdapter = new KoaAdapter()
   for (let queue of queues) {
     adapters.push(new BullAdapter(queue))
   }
-  const { router } = createBullBoard(adapters)
-
-  expressApp.use(exports.pathPrefix, router)
-  return expressApp
+  createBullBoard({
+    queues: adapters,
+    serverAdapter,
+  })
+  serverAdapter.setBasePath(PATH_PREFIX)
+  return serverAdapter.registerPlugin()
 }
 
 exports.queue = automationQueue

package/src/automations/tests/automation.spec.js

@@ -1,5 +1,5 @@
 jest.mock("../../utilities/usageQuota")
-jest.mock("../thread")
+jest.mock("../../threads/automation")
 jest.mock("../../utilities/redis", () => ({
   init: jest.fn(),
   checkTestFlag: () => {
@@ -11,8 +11,7 @@ jest.spyOn(global.console, "error")
 
 require("../../environment")
 const automation = require("../index")
-const usageQuota = require("../../utilities/usageQuota")
-const thread = require("../thread")
+const thread = require("../../threads/automation")
 const triggers = require("../triggers")
 const { basicAutomation } = require("../../tests/utilities/structures")
 const { wait } = require("../../utilities")
@@ -62,7 +61,7 @@ describe("Run through some parts of the automations system", () => {
           }
         }
       }
-    }))
+    }), expect.any(Function))
   })
 
   it("should be able to clean inputs with the utilities", () => {

package/src/automations/triggers.js

@@ -11,6 +11,10 @@ const utils = require("./utils")
 const env = require("../environment")
 
 const TRIGGER_DEFINITIONS = definitions
+const JOB_OPTS = {
+  removeOnComplete: true,
+  removeOnFail: true,
+}
 
 async function queueRelevantRowAutomations(event, eventType) {
   if (event.appId == null) {
@@ -47,7 +51,7 @@ async function queueRelevantRowAutomations(event, eventType) {
       automationTrigger.inputs &&
       automationTrigger.inputs.tableId === event.row.tableId
     ) {
-      await queue.add({ automation, event })
+      await queue.add({ automation, event }, JOB_OPTS)
     }
   }
 }
@@ -86,7 +90,7 @@ exports.externalTrigger = async function (
     automation.definition.trigger != null &&
     automation.definition.trigger.stepId === definitions.APP.stepId &&
     automation.definition.trigger.stepId === "APP" &&
-    !checkTestFlag(automation._id)
+    !(await checkTestFlag(automation._id))
   ) {
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
@@ -100,7 +104,7 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data)
+    return queue.add(data, JOB_OPTS)
   }
 }
 

package/src/automations/utils.js

@@ -1,4 +1,4 @@
-const runner = require("./thread")
+const { Thread, ThreadType } = require("../threads")
 const { definitions } = require("./triggerInfo")
 const webhooks = require("../api/controllers/webhook")
 const CouchDB = require("../db")
@@ -10,11 +10,12 @@ const { getDeployedAppID } = require("@budibase/auth/db")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
+const Runner = new Thread(ThreadType.AUTOMATION)
 
 exports.processEvent = async job => {
   try {
     // need to actually await these so that an error can be captured properly
-    return await runner(job)
+    return await Runner.run(job)
   } catch (err) {
     console.error(
       `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
@@ -162,3 +163,12 @@ exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
   }
   return newAuto
 }
+
+/**
+ * When removing an app/unpublishing it need to make sure automations are cleaned up (cron).
+ * @param appId {string} the app that is being removed.
+ * @return {Promise<void>} clean is complete if this succeeds.
+ */
+exports.cleanupAutomations = async appId => {
+  await exports.disableAllCrons(appId)
+}

package/src/constants/index.js

@@ -163,8 +163,15 @@ exports.MetadataTypes = {
   AUTOMATION_TEST_HISTORY: "automationTestHistory",
 }
 
+exports.InvalidColumns = {
+  ID: "_id",
+  REV: "_rev",
+  TABLE_ID: "tableId",
+}
+
 exports.BuildSchemaErrors = {
   NO_KEY: "no_key",
+  INVALID_COLUMN: "invalid_column",
 }
 
 // pass through the list from the auth/core lib

package/src/definitions/datasource.ts

@@ -46,6 +46,7 @@ export enum SourceNames {
   MYSQL = "MYSQL",
   ARANGODB = "ARANGODB",
   REST = "REST",
+  ORACLE = "ORACLE",
 }
 
 export enum IncludeRelationships {

package/src/environment.js

@@ -23,6 +23,8 @@ if (!LOADED && isDev() && !isTest()) {
   LOADED = true
 }
 
+let inThread = false
+
 module.exports = {
   // important
   PORT: process.env.PORT,
@@ -62,6 +64,7 @@ module.exports = {
   USERID_API_KEY: process.env.USERID_API_KEY,
   DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
+  DISABLE_THREADING: process.env.DISABLE_THREADING,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value
@@ -72,6 +75,18 @@ module.exports = {
   isProd: () => {
     return !isDev()
   },
+  // used to check if already in a thread, don't thread further
+  setInThread: () => {
+    inThread = true
+  },
+  isInThread: () => {
+    return inThread
+  },
+}
+
+// threading can cause memory issues with node-ts in development
+if (isDev() && module.exports.DISABLE_THREADING == null) {
+  module.exports._set("DISABLE_THREADING", "1")
 }
 
 // clean up any environment variable edge cases

package/src/integrations/airtable.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module AirtableModule {
   const Airtable = require("airtable")
@@ -73,7 +74,7 @@ module AirtableModule {
     },
   }
 
-  class AirtableIntegration {
+  class AirtableIntegration implements IntegrationBase {
     private config: AirtableConfig
     private client: any
 

package/src/integrations/arangodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module ArangoModule {
   const { Database, aql } = require("arangojs")
@@ -55,7 +56,7 @@ module ArangoModule {
     },
   }
 
-  class ArangoDBIntegration {
+  class ArangoDBIntegration implements IntegrationBase {
     private config: ArangodbConfig
     private client: any
 

package/src/integrations/base/IntegrationBase.ts

@@ -0,0 +1,6 @@
+export interface IntegrationBase {
+  create?(query: any): Promise<any[]>
+  read?(query: any): Promise<any[]>
+  update?(query: any): Promise<any[]>
+  delete?(query: any): Promise<any[]>
+}

package/src/integrations/base/datasourcePlus.ts

@@ -1,6 +1,7 @@
 import { Table } from "../../definitions/common"
+import { IntegrationBase } from "./IntegrationBase"
 
-export interface DatasourcePlus {
+export interface DatasourcePlus extends IntegrationBase {
   tables: Record<string, Table>
   schemaErrors: Record<string, string>
 

package/src/integrations/base/sql.ts

@@ -13,22 +13,50 @@ import SqlTableQueryBuilder from "./sqlTable"
 const BASE_LIMIT = 5000
 
 type KnexQuery = Knex.QueryBuilder | Knex
+// these are invalid dates sent by the client, need to convert them to a real max date
+const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z"
+const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z"
+
+function parse(input: any) {
+  if (Array.isArray(input)) {
+    return JSON.stringify(input)
+  }
+  if (typeof input !== "string") {
+    return input
+  }
+  if (input === MAX_ISO_DATE) {
+    return new Date(8640000000000000)
+  }
+  if (input === MIN_ISO_DATE) {
+    return new Date(-8640000000000000)
+  }
+  if (isIsoDateString(input)) {
+    return new Date(input)
+  }
+  return input
+}
 
 function parseBody(body: any) {
   for (let [key, value] of Object.entries(body)) {
-    if (Array.isArray(value)) {
-      body[key] = JSON.stringify(value)
-    }
-    if (typeof value !== "string") {
-      continue
-    }
-    if (isIsoDateString(value)) {
-      body[key] = new Date(value)
-    }
+    body[key] = parse(value)
   }
   return body
 }
 
+function parseFilters(filters: SearchFilters): SearchFilters {
+  for (let [key, value] of Object.entries(filters)) {
+    let parsed
+    if (typeof value === "object") {
+      parsed = parseFilters(value)
+    } else {
+      parsed = parse(value)
+    }
+    // @ts-ignore
+    filters[key] = parsed
+  }
+  return filters
+}
+
 class InternalBuilder {
   private readonly client: string
 
@@ -53,6 +81,7 @@ class InternalBuilder {
     if (!filters) {
       return query
     }
+    filters = parseFilters(filters)
     // if all or specified in filters, then everything is an or
     const allOr = filters.allOr
     if (filters.oneOf) {
@@ -387,6 +416,8 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         id = results?.[0].id
       } else if (sqlClient === SqlClients.MY_SQL) {
         id = results?.insertId
+      } else if (sqlClient === SqlClients.ORACLE) {
+        id = response.outBinds[0][0]
       }
       row = processFn(
         await this.getReturningRow(queryFn, this.checkLookupKeys(id, json))
@@ -399,4 +430,5 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
   }
 }
 
+export default SqlQueryBuilder
 module.exports = SqlQueryBuilder

package/src/integrations/couchdb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module CouchDBModule {
   const PouchDB = require("pouchdb")
@@ -50,7 +51,7 @@ module CouchDBModule {
     },
   }
 
-  class CouchDBIntegration {
+  class CouchDBIntegration implements IntegrationBase {
     private config: CouchDBConfig
     private client: any
 

package/src/integrations/dynamodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module DynamoModule {
   const AWS = require("aws-sdk")
@@ -113,7 +114,7 @@ module DynamoModule {
     },
   }
 
-  class DynamoDBIntegration {
+  class DynamoDBIntegration implements IntegrationBase {
     private config: DynamoDBConfig
     private client: any
 

package/src/integrations/elasticsearch.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module ElasticsearchModule {
   const { Client } = require("@elastic/elasticsearch")
@@ -74,7 +75,7 @@ module ElasticsearchModule {
     },
   }
 
-  class ElasticSearchIntegration {
+  class ElasticSearchIntegration implements IntegrationBase {
     private config: ElasticsearchConfig
     private client: any
 

package/src/integrations/index.ts

@@ -39,6 +39,13 @@ const INTEGRATIONS = {
   [SourceNames.REST]: rest.integration,
 }
 
+// optionally add oracle integration if the oracle binary can be installed
+if (!(process.arch === 'arm64' && process.platform === 'darwin')) {
+  const oracle = require("./oracle")
+  DEFINITIONS[SourceNames.ORACLE] =  oracle.schema
+  INTEGRATIONS[SourceNames.ORACLE] = oracle.integration
+}
+
 module.exports = {
   definitions: DEFINITIONS,
   integrations: INTEGRATIONS,

package/src/integrations/mongodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module MongoDBModule {
   const { MongoClient } = require("mongodb")
@@ -62,7 +63,7 @@ module MongoDBModule {
     },
   }
 
-  class MongoIntegration {
+  class MongoIntegration implements IntegrationBase {
     private config: MongoDBConfig
     private client: any
 

package/src/integrations/mysql.ts

@@ -184,7 +184,7 @@ module MySQLModule {
       return results.length ? results : [{ created: true }]
     }
 
-    read(query: SqlQuery | string) {
+    async read(query: SqlQuery | string) {
       return internalQuery(this.client, getSqlQuery(query))
     }