@budibase/server 0.9.125-alpha.4 → 0.9.125-alpha.8

package/dist/utilities/redis.js

@@ -11,20 +11,25 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 const { Client, utils } = require("@budibase/auth/redis");
 const { getGlobalIDFromUserMetadataID } = require("../db/utils");
 const APP_DEV_LOCK_SECONDS = 600;
-let devAppClient, debounceClient;
+const AUTOMATION_TEST_FLAG_SECONDS = 60;
+let devAppClient, debounceClient, flagClient;
 // we init this as we want to keep the connection open all the time
 // reduces the performance hit
 exports.init = () => __awaiter(void 0, void 0, void 0, function* () {
     devAppClient = new Client(utils.Databases.DEV_LOCKS);
     debounceClient = new Client(utils.Databases.DEBOUNCE);
+    flagClient = new Client(utils.Databases.FLAGS);
     yield devAppClient.init();
     yield debounceClient.init();
+    yield flagClient.init();
 });
 exports.shutdown = () => __awaiter(void 0, void 0, void 0, function* () {
     if (devAppClient)
         yield devAppClient.finish();
     if (debounceClient)
         yield debounceClient.finish();
+    if (flagClient)
+        yield flagClient.finish();
 });
 exports.doesUserHaveLock = (devAppId, user) => __awaiter(void 0, void 0, void 0, function* () {
     const value = yield devAppClient.get(devAppId);
@@ -66,3 +71,13 @@ exports.checkDebounce = (id) => __awaiter(void 0, void 0, void 0, function* () {
 exports.setDebounce = (id, seconds) => __awaiter(void 0, void 0, void 0, function* () {
     yield debounceClient.store(id, "debouncing", seconds);
 });
+exports.setTestFlag = (id) => __awaiter(void 0, void 0, void 0, function* () {
+    yield flagClient.store(id, { testing: true }, AUTOMATION_TEST_FLAG_SECONDS);
+});
+exports.checkTestFlag = (id) => __awaiter(void 0, void 0, void 0, function* () {
+    const flag = yield flagClient.get(id);
+    return !!(flag && flag.testing);
+});
+exports.clearTestFlag = (id) => __awaiter(void 0, void 0, void 0, function* () {
+    yield devAppClient.delete(id);
+});

package/dist/utilities/rowProcessor/index.js

@@ -190,10 +190,11 @@ exports.inputProcessing = (user = {}, table, row) => {
  * @param {object} ctx the request which is looking for enriched rows.
  * @param {object} table the table from which these rows came from originally, this is used to determine
  * the schema of the rows and then enrich.
- * @param {object[]} rows the rows which are to be enriched.
- * @returns {object[]} the enriched rows will be returned.
+ * @param {object[]|object} rows the rows which are to be enriched.
+ * @param {object} opts used to set some options for the output, such as disabling relationship squashing.
+ * @returns {object[]|object} the enriched rows will be returned.
  */
-exports.outputProcessing = (ctx, table, rows) => __awaiter(void 0, void 0, void 0, function* () {
+exports.outputProcessing = (ctx, table, rows, opts = { squash: true }) => __awaiter(void 0, void 0, void 0, function* () {
     const appId = ctx.appId;
     let wasArray = true;
     if (!(rows instanceof Array)) {
@@ -217,6 +218,8 @@ exports.outputProcessing = (ctx, table, rows) => __awaiter(void 0, void 0, void
             }
         }
     }
-    enriched = yield linkRows.squashLinksToPrimaryDisplay(appId, table, enriched);
+    if (opts.squash) {
+        enriched = yield linkRows.squashLinksToPrimaryDisplay(appId, table, enriched);
+    }
     return wasArray ? enriched : enriched[0];
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.125-alpha.4",
+  "version": "0.9.125-alpha.8",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -24,7 +24,9 @@
     "lint": "eslint --fix src/",
     "lint:fix": "yarn run format && yarn run lint",
     "multi:enable": "node scripts/multiTenancy.js enable",
-    "multi:disable": "node scripts/multiTenancy.js disable"
+    "multi:disable": "node scripts/multiTenancy.js disable",
+    "selfhost:enable": "node scripts/selfhost.js enable",
+    "selfhost:disable": "node scripts/selfhost.js disable"
   },
   "jest": {
     "preset": "ts-jest",
@@ -61,9 +63,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.125-alpha.4",
-    "@budibase/client": "^0.9.125-alpha.4",
-    "@budibase/string-templates": "^0.9.125-alpha.4",
+    "@budibase/auth": "^0.9.125-alpha.8",
+    "@budibase/client": "^0.9.125-alpha.8",
+    "@budibase/string-templates": "^0.9.125-alpha.8",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -109,7 +111,6 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
-    "worker-farm": "1.7.0",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -140,5 +141,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "6748b75efd368d833d3cc2133ec4a5f6489c278c"
+  "gitHead": "108a7836353583a04bb0280891cbc6dac1f8a66b"
 }

package/scripts/selfhost.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+const updateDotEnv = require("update-dotenv")
+
+const arg = process.argv.slice(2)[0]
+
+updateDotEnv({
+  SELF_HOSTED: arg === "enable" ? "1" : "0",
+}).then(() => console.log("Updated server!"))

package/src/api/controllers/automation.js

@@ -1,12 +1,14 @@
 const CouchDB = require("../../db")
 const actions = require("../../automations/actions")
-const logic = require("../../automations/logic")
 const triggers = require("../../automations/triggers")
-const webhooks = require("./webhook")
 const { getAutomationParams, generateAutomationID } = require("../../db/utils")
-
-const WH_STEP_ID = triggers.BUILTIN_DEFINITIONS.WEBHOOK.stepId
-const CRON_STEP_ID = triggers.BUILTIN_DEFINITIONS.CRON.stepId
+const {
+  checkForWebhooks,
+  updateTestHistory,
+} = require("../../automations/utils")
+const { deleteEntityMetadata } = require("../../utilities")
+const { MetadataTypes } = require("../../constants")
+const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
 
 /*************************
  *                       *
@@ -14,6 +16,19 @@ const CRON_STEP_ID = triggers.BUILTIN_DEFINITIONS.CRON.stepId
  *                       *
  *************************/
 
+async function cleanupAutomationMetadata(appId, automationId) {
+  await deleteEntityMetadata(
+    appId,
+    MetadataTypes.AUTOMATION_TEST_INPUT,
+    automationId
+  )
+  await deleteEntityMetadata(
+    appId,
+    MetadataTypes.AUTOMATION_TEST_HISTORY,
+    automationId
+  )
+}
+
 function cleanAutomationInputs(automation) {
   if (automation == null) {
     return automation
@@ -21,6 +36,10 @@ function cleanAutomationInputs(automation) {
   let steps = automation.definition.steps
   let trigger = automation.definition.trigger
   let allSteps = [...steps, trigger]
+  // live is not a property used anymore
+  if (automation.live != null) {
+    delete automation.live
+  }
   for (let step of allSteps) {
     if (step == null) {
       continue
@@ -34,119 +53,6 @@ function cleanAutomationInputs(automation) {
   return automation
 }
 
-/**
- * This function handles checking of any cron jobs need to be created or deleted for automations.
- * @param {string} appId The ID of the app in which we are checking for webhooks
- * @param {object|undefined} oldAuto The old automation object if updating/deleting
- * @param {object|undefined} newAuto The new automation object if creating/updating
- */
-async function checkForCronTriggers({ appId, oldAuto, newAuto }) {
-  const oldTrigger = oldAuto ? oldAuto.definition.trigger : null
-  const newTrigger = newAuto ? newAuto.definition.trigger : null
-  function isCronTrigger(auto) {
-    return (
-      auto &&
-      auto.definition.trigger &&
-      auto.definition.trigger.stepId === CRON_STEP_ID
-    )
-  }
-
-  const isLive = auto => auto && auto.live
-
-  const cronTriggerRemoved =
-    isCronTrigger(oldAuto) && !isCronTrigger(newAuto) && oldTrigger.cronJobId
-  const cronTriggerDeactivated = !isLive(newAuto) && isLive(oldAuto)
-
-  const cronTriggerActivated = isLive(newAuto) && !isLive(oldAuto)
-
-  if (cronTriggerRemoved || (cronTriggerDeactivated && oldTrigger.cronJobId)) {
-    await triggers.automationQueue.removeRepeatableByKey(oldTrigger.cronJobId)
-  }
-  // need to create cron job
-  else if (isCronTrigger(newAuto) && cronTriggerActivated) {
-    const job = await triggers.automationQueue.add(
-      {
-        automation: newAuto,
-        event: { appId, timestamp: Date.now() },
-      },
-      { repeat: { cron: newTrigger.inputs.cron } }
-    )
-    // Assign cron job ID from bull so we can remove it later if the cron trigger is removed
-    newTrigger.cronJobId = job.id
-  }
-  return newAuto
-}
-
-/**
- * This function handles checking if any webhooks need to be created or deleted for automations.
- * @param {string} appId The ID of the app in which we are checking for webhooks
- * @param {object|undefined} oldAuto The old automation object if updating/deleting
- * @param {object|undefined} newAuto The new automation object if creating/updating
- * @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
- * written to DB (this does not write to DB as it would be wasteful to repeat).
- */
-async function checkForWebhooks({ appId, oldAuto, newAuto }) {
-  const oldTrigger = oldAuto ? oldAuto.definition.trigger : null
-  const newTrigger = newAuto ? newAuto.definition.trigger : null
-  const triggerChanged =
-    oldTrigger && newTrigger && oldTrigger.id !== newTrigger.id
-  function isWebhookTrigger(auto) {
-    return (
-      auto &&
-      auto.definition.trigger &&
-      auto.definition.trigger.stepId === WH_STEP_ID
-    )
-  }
-  // need to delete webhook
-  if (
-    isWebhookTrigger(oldAuto) &&
-    (!isWebhookTrigger(newAuto) || triggerChanged) &&
-    oldTrigger.webhookId
-  ) {
-    try {
-      let db = new CouchDB(appId)
-      // need to get the webhook to get the rev
-      const webhook = await db.get(oldTrigger.webhookId)
-      const ctx = {
-        appId,
-        params: { id: webhook._id, rev: webhook._rev },
-      }
-      // might be updating - reset the inputs to remove the URLs
-      if (newTrigger) {
-        delete newTrigger.webhookId
-        newTrigger.inputs = {}
-      }
-      await webhooks.destroy(ctx)
-    } catch (err) {
-      // don't worry about not being able to delete, if it doesn't exist all good
-    }
-  }
-  // need to create webhook
-  if (
-    (!isWebhookTrigger(oldAuto) || triggerChanged) &&
-    isWebhookTrigger(newAuto)
-  ) {
-    const ctx = {
-      appId,
-      request: {
-        body: new webhooks.Webhook(
-          "Automation webhook",
-          webhooks.WebhookType.AUTOMATION,
-          newAuto._id
-        ),
-      },
-    }
-    await webhooks.save(ctx)
-    const id = ctx.body.webhook._id
-    newTrigger.webhookId = id
-    newTrigger.inputs = {
-      schemaUrl: `api/webhooks/schema/${appId}/${id}`,
-      triggerUrl: `api/webhooks/trigger/${appId}/${id}`,
-    }
-  }
-  return newAuto
-}
-
 exports.create = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   let automation = ctx.request.body
@@ -165,10 +71,6 @@ exports.create = async function (ctx) {
     appId: ctx.appId,
     newAuto: automation,
   })
-  automation = await checkForCronTriggers({
-    appId: ctx.appId,
-    newAuto: automation,
-  })
   const response = await db.put(automation)
   automation._rev = response.rev
 
@@ -193,14 +95,26 @@ exports.update = async function (ctx) {
     oldAuto: oldAutomation,
     newAuto: automation,
   })
-  automation = await checkForCronTriggers({
-    appId: ctx.appId,
-    oldAuto: oldAutomation,
-    newAuto: automation,
-  })
   const response = await db.put(automation)
   automation._rev = response.rev
 
+  const oldAutoTrigger =
+    oldAutomation && oldAutomation.definition.trigger
+      ? oldAutomation.definition.trigger
+      : {}
+  const newAutoTrigger =
+    automation && automation.definition.trigger
+      ? automation.definition.trigger
+      : {}
+  // trigger has been updated, remove the test inputs
+  if (oldAutoTrigger.id !== newAutoTrigger.id) {
+    await deleteEntityMetadata(
+      ctx.appId,
+      MetadataTypes.AUTOMATION_TEST_INPUT,
+      automation._id
+    )
+  }
+
   ctx.status = 200
   ctx.body = {
     message: `Automation ${automation._id} updated successfully.`,
@@ -229,35 +143,29 @@ exports.find = async function (ctx) {
 
 exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
-  const oldAutomation = await db.get(ctx.params.id)
+  const automationId = ctx.params.id
+  const oldAutomation = await db.get(automationId)
   await checkForWebhooks({
     appId: ctx.appId,
     oldAuto: oldAutomation,
   })
-  await checkForCronTriggers({
-    appId: ctx.appId,
-    oldAuto: oldAutomation,
-  })
-  ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
+  // delete metadata first
+  await cleanupAutomationMetadata(ctx.appId, automationId)
+  ctx.body = await db.remove(automationId, ctx.params.rev)
 }
 
 exports.getActionList = async function (ctx) {
-  ctx.body = actions.DEFINITIONS
+  ctx.body = actions.ACTION_DEFINITIONS
 }
 
 exports.getTriggerList = async function (ctx) {
-  ctx.body = triggers.BUILTIN_DEFINITIONS
-}
-
-exports.getLogicList = async function (ctx) {
-  ctx.body = logic.BUILTIN_DEFINITIONS
+  ctx.body = triggers.TRIGGER_DEFINITIONS
 }
 
 module.exports.getDefinitionList = async function (ctx) {
   ctx.body = {
-    logic: logic.BUILTIN_DEFINITIONS,
-    trigger: triggers.BUILTIN_DEFINITIONS,
-    action: actions.DEFINITIONS,
+    trigger: triggers.TRIGGER_DEFINITIONS,
+    action: actions.ACTION_DEFINITIONS,
   }
 }
 
@@ -268,15 +176,37 @@ module.exports.getDefinitionList = async function (ctx) {
  *********************/
 
 exports.trigger = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
   let automation = await db.get(ctx.params.id)
   await triggers.externalTrigger(automation, {
     ...ctx.request.body,
-    appId: ctx.appId,
+    appId,
   })
-  ctx.status = 200
   ctx.body = {
     message: `Automation ${automation._id} has been triggered.`,
     automation,
   }
 }
+
+exports.test = async function (ctx) {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  let automation = await db.get(ctx.params.id)
+  await setTestFlag(automation._id)
+  const response = await triggers.externalTrigger(
+    automation,
+    {
+      ...ctx.request.body,
+      appId,
+    },
+    { getResponses: true }
+  )
+  // save a test history run
+  await updateTestHistory(ctx.appId, automation, {
+    ...ctx.request.body,
+    occurredAt: new Date().getTime(),
+  })
+  await clearTestFlag(automation._id)
+  ctx.body = response
+}

package/src/api/controllers/deploy/index.js

@@ -1,7 +1,11 @@
 const CouchDB = require("../../../db")
 const Deployment = require("./Deployment")
 const { Replication } = require("@budibase/auth/db")
-const { DocumentTypes } = require("../../../db/utils")
+const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
+const {
+  disableAllCrons,
+  enableCronTrigger,
+} = require("../../../automations/utils")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -58,6 +62,23 @@ async function storeDeploymentHistory(deployment) {
   return deployment
 }
 
+async function initDeployedApp(prodAppId) {
+  const db = new CouchDB(prodAppId)
+  const automations = (
+    await db.allDocs(
+      getAutomationParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+  const promises = []
+  await disableAllCrons(prodAppId)
+  for (let automation of automations) {
+    promises.push(enableCronTrigger(prodAppId, automation))
+  }
+  await Promise.all(promises)
+}
+
 async function deployApp(deployment) {
   try {
     const productionAppId = deployment.appId.replace("_dev", "")
@@ -85,6 +106,7 @@ async function deployApp(deployment) {
       },
     })
 
+    await initDeployedApp(productionAppId)
     deployment.setStatus(DeploymentStatus.SUCCESS)
     await storeDeploymentHistory(deployment)
   } catch (err) {