@budibase/server 0.9.136 → 0.9.140-alpha.0

package/src/automations/triggers.js

@@ -1,246 +1,22 @@
 const CouchDB = require("../db")
 const emitter = require("../events/index")
-const env = require("../environment")
-const Queue = env.isTest()
-  ? require("../utilities/queue/inMemoryQueue")
-  : require("bull")
 const { getAutomationParams } = require("../db/utils")
 const { coerce } = require("../utilities/rowProcessor")
-const { utils } = require("@budibase/auth/redis")
-const { JobQueues } = require("../constants")
-const {
-  isExternalTable,
-  breakExternalTableId,
-} = require("../integrations/utils")
-const { getExternalTable } = require("../api/controllers/table/utils")
-
-const { opts, redisProtocolUrl } = utils.getRedisOptions()
-
-const redisConfig = redisProtocolUrl || { redis: opts }
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, redisConfig)
-
-const FAKE_STRING = "TEST"
-const FAKE_BOOL = false
-const FAKE_NUMBER = 1
-const FAKE_DATETIME = "1970-01-01T00:00:00.000Z"
+const { definitions } = require("./triggerInfo")
+const { isDevAppID } = require("../db/utils")
+// need this to call directly, so we can get a response
+const { queue } = require("./bullboard")
+const { checkTestFlag } = require("../utilities/redis")
+const utils = require("./utils")
+const env = require("../environment")
 
-const BUILTIN_DEFINITIONS = {
-  ROW_SAVED: {
-    name: "Row Created",
-    event: "row:save",
-    icon: "ri-save-line",
-    tagline: "Row is added to {{inputs.enriched.table.name}}",
-    description: "Fired when a row is added to your database",
-    stepId: "ROW_SAVED",
-    inputs: {},
-    schema: {
-      inputs: {
-        properties: {
-          tableId: {
-            type: "string",
-            customType: "table",
-            title: "Table",
-          },
-        },
-        required: ["tableId"],
-      },
-      outputs: {
-        properties: {
-          row: {
-            type: "object",
-            customType: "row",
-            description: "The new row that was created",
-          },
-          id: {
-            type: "string",
-            description: "Row ID - can be used for updating",
-          },
-          revision: {
-            type: "string",
-            description: "Revision of row",
-          },
-        },
-        required: ["row", "id"],
-      },
-    },
-    type: "TRIGGER",
-  },
-  ROW_UPDATED: {
-    name: "Row Updated",
-    event: "row:update",
-    icon: "ri-refresh-line",
-    tagline: "Row is updated in {{inputs.enriched.table.name}}",
-    description: "Fired when a row is updated in your database",
-    stepId: "ROW_UPDATED",
-    inputs: {},
-    schema: {
-      inputs: {
-        properties: {
-          tableId: {
-            type: "string",
-            customType: "table",
-            title: "Table",
-          },
-        },
-        required: ["tableId"],
-      },
-      outputs: {
-        properties: {
-          row: {
-            type: "object",
-            customType: "row",
-            description: "The row that was updated",
-          },
-          id: {
-            type: "string",
-            description: "Row ID - can be used for updating",
-          },
-          revision: {
-            type: "string",
-            description: "Revision of row",
-          },
-        },
-        required: ["row", "id"],
-      },
-    },
-    type: "TRIGGER",
-  },
-  ROW_DELETED: {
-    name: "Row Deleted",
-    event: "row:delete",
-    icon: "ri-delete-bin-line",
-    tagline: "Row is deleted from {{inputs.enriched.table.name}}",
-    description: "Fired when a row is deleted from your database",
-    stepId: "ROW_DELETED",
-    inputs: {},
-    schema: {
-      inputs: {
-        properties: {
-          tableId: {
-            type: "string",
-            customType: "table",
-            title: "Table",
-          },
-        },
-        required: ["tableId"],
-      },
-      outputs: {
-        properties: {
-          row: {
-            type: "object",
-            customType: "row",
-            description: "The row that was deleted",
-          },
-        },
-        required: ["row"],
-      },
-    },
-    type: "TRIGGER",
-  },
-  WEBHOOK: {
-    name: "Webhook",
-    event: "web:trigger",
-    icon: "ri-global-line",
-    tagline: "Webhook endpoint is hit",
-    description: "Trigger an automation when a HTTP POST webhook is hit",
-    stepId: "WEBHOOK",
-    inputs: {},
-    schema: {
-      inputs: {
-        properties: {
-          schemaUrl: {
-            type: "string",
-            customType: "webhookUrl",
-            title: "Schema URL",
-          },
-          triggerUrl: {
-            type: "string",
-            customType: "webhookUrl",
-            title: "Trigger URL",
-          },
-        },
-        required: ["schemaUrl", "triggerUrl"],
-      },
-      outputs: {
-        properties: {
-          body: {
-            type: "object",
-            description: "Body of the request which hit the webhook",
-          },
-        },
-        required: ["body"],
-      },
-    },
-    type: "TRIGGER",
-  },
-  APP: {
-    name: "App Action",
-    event: "app:trigger",
-    icon: "ri-window-fill",
-    tagline: "Automation fired from the frontend",
-    description: "Trigger an automation from an action inside your app",
-    stepId: "APP",
-    inputs: {},
-    schema: {
-      inputs: {
-        properties: {
-          fields: {
-            type: "object",
-            customType: "triggerSchema",
-            title: "Fields",
-          },
-        },
-        required: [],
-      },
-      outputs: {
-        properties: {
-          fields: {
-            type: "object",
-            description: "Fields submitted from the app frontend",
-          },
-        },
-        required: ["fields"],
-      },
-    },
-    type: "TRIGGER",
-  },
-  CRON: {
-    name: "Cron Trigger",
-    event: "cron:trigger",
-    icon: "ri-timer-line",
-    tagline: "Cron Trigger (<b>{{inputs.cron}}</b>)",
-    description: "Triggers automation on a cron schedule.",
-    stepId: "CRON",
-    inputs: {},
-    schema: {
-      inputs: {
-        properties: {
-          cron: {
-            type: "string",
-            customType: "cron",
-            title: "Expression",
-          },
-        },
-        required: ["cron"],
-      },
-      outputs: {
-        properties: {
-          timestamp: {
-            type: "number",
-            description: "Timestamp the cron was executed",
-          },
-        },
-        required: ["timestamp"],
-      },
-    },
-    type: "TRIGGER",
-  },
-}
+const TRIGGER_DEFINITIONS = definitions
 
 async function queueRelevantRowAutomations(event, eventType) {
   if (event.appId == null) {
     throw `No appId specified for ${eventType} - check event emitters.`
   }
+
   const db = new CouchDB(event.appId)
   let automations = await db.allDocs(
     getAutomationParams(null, { include_docs: true })
@@ -257,14 +33,22 @@ async function queueRelevantRowAutomations(event, eventType) {
   for (let automation of automations) {
     let automationDef = automation.definition
     let automationTrigger = automationDef ? automationDef.trigger : {}
+    // don't queue events which are for dev apps, only way to test automations is
+    // running tests on them, in production the test flag will never
+    // be checked due to lazy evaluation (first always false)
     if (
-      !automation.live ||
-      !automationTrigger.inputs ||
-      automationTrigger.inputs.tableId !== event.row.tableId
+      !env.ALLOW_DEV_AUTOMATIONS &&
+      isDevAppID(event.appId) &&
+      !(await checkTestFlag(automation._id))
     ) {
       continue
     }
-    automationQueue.add({ automation, event })
+    if (
+      automationTrigger.inputs &&
+      automationTrigger.inputs.tableId === event.row.tableId
+    ) {
+      await queue.add({ automation, event })
+    }
   }
 }
 
@@ -292,51 +76,12 @@ emitter.on("row:delete", async function (event) {
   await queueRelevantRowAutomations(event, "row:delete")
 })
 
-async function fillRowOutput(automation, params) {
-  let triggerSchema = automation.definition.trigger
-  let tableId = triggerSchema.inputs.tableId
-  try {
-    let table
-    if (!isExternalTable(tableId)) {
-      const db = new CouchDB(params.appId)
-      table = await db.get(tableId)
-    } else {
-      const { datasourceId, tableName } = breakExternalTableId(tableId)
-      table = await getExternalTable(params.appId, datasourceId, tableName)
-    }
-    let row = {}
-    for (let schemaKey of Object.keys(table.schema)) {
-      const paramValue = params[schemaKey]
-      let propSchema = table.schema[schemaKey]
-      switch (propSchema.constraints.type) {
-        case "string":
-          row[schemaKey] = paramValue || FAKE_STRING
-          break
-        case "boolean":
-          row[schemaKey] = paramValue || FAKE_BOOL
-          break
-        case "number":
-          row[schemaKey] = paramValue || FAKE_NUMBER
-          break
-        case "datetime":
-          row[schemaKey] = paramValue || FAKE_DATETIME
-          break
-      }
-    }
-    params.row = row
-  } catch (err) {
-    /* istanbul ignore next */
-    throw "Failed to find table for trigger"
-  }
-  return params
-}
-
-module.exports.externalTrigger = async function (automation, params) {
-  // TODO: replace this with allowing user in builder to input values in future
+exports.externalTrigger = async function (
+  automation,
+  params,
+  { getResponses } = {}
+) {
   if (automation.definition != null && automation.definition.trigger != null) {
-    if (automation.definition.trigger.inputs.tableId != null) {
-      params = await fillRowOutput(automation, params)
-    }
     if (automation.definition.trigger.stepId === "APP") {
       // values are likely to be submitted as strings, so we shall convert to correct type
       const coercedFields = {}
@@ -347,14 +92,12 @@ module.exports.externalTrigger = async function (automation, params) {
       params.fields = coercedFields
     }
   }
-
-  automationQueue.add({ automation, event: params })
-}
-
-module.exports.getQueues = () => {
-  return [automationQueue]
+  const data = { automation, event: params }
+  if (getResponses) {
+    return utils.processEvent({ data })
+  } else {
+    return queue.add(data)
+  }
 }
-module.exports.fillRowOutput = fillRowOutput
-module.exports.automationQueue = automationQueue
 
-module.exports.BUILTIN_DEFINITIONS = BUILTIN_DEFINITIONS
+exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS

package/src/automations/utils.js

@@ -0,0 +1,159 @@
+const runner = require("./thread")
+const { definitions } = require("./triggerInfo")
+const webhooks = require("../api/controllers/webhook")
+const CouchDB = require("../db")
+const { queue } = require("./bullboard")
+const newid = require("../db/newid")
+const { updateEntityMetadata } = require("../utilities")
+const { MetadataTypes } = require("../constants")
+
+const WH_STEP_ID = definitions.WEBHOOK.stepId
+const CRON_STEP_ID = definitions.CRON.stepId
+
+exports.processEvent = async job => {
+  try {
+    // need to actually await these so that an error can be captured properly
+    return await runner(job)
+  } catch (err) {
+    console.error(
+      `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
+    )
+    return { err }
+  }
+}
+
+exports.updateTestHistory = async (appId, automation, history) => {
+  return updateEntityMetadata(
+    appId,
+    MetadataTypes.AUTOMATION_TEST_HISTORY,
+    automation._id,
+    metadata => {
+      if (metadata && Array.isArray(metadata.history)) {
+        metadata.history.push(history)
+      } else {
+        metadata = {
+          history: [history],
+        }
+      }
+      return metadata
+    }
+  )
+}
+
+// end the repetition and the job itself
+exports.disableAllCrons = async appId => {
+  const promises = []
+  const jobs = await queue.getRepeatableJobs()
+  for (let job of jobs) {
+    if (job.key.includes(`${appId}_cron`)) {
+      promises.push(queue.removeRepeatableByKey(job.key))
+      promises.push(queue.removeJobs(job.id))
+    }
+  }
+  return Promise.all(promises)
+}
+
+/**
+ * This function handles checking of any cron jobs that need to be enabled/updated.
+ * @param {string} appId The ID of the app in which we are checking for webhooks
+ * @param {object|undefined} automation The automation object to be updated.
+ */
+exports.enableCronTrigger = async (appId, automation) => {
+  const trigger = automation ? automation.definition.trigger : null
+  function isCronTrigger(auto) {
+    return (
+      auto &&
+      auto.definition.trigger &&
+      auto.definition.trigger.stepId === CRON_STEP_ID
+    )
+  }
+  // need to create cron job
+  if (isCronTrigger(automation)) {
+    // make a job id rather than letting Bull decide, makes it easier to handle on way out
+    const jobId = `${appId}_cron_${newid()}`
+    const job = await queue.add(
+      {
+        automation,
+        event: { appId, timestamp: Date.now() },
+      },
+      { repeat: { cron: trigger.inputs.cron }, jobId }
+    )
+    // Assign cron job ID from bull so we can remove it later if the cron trigger is removed
+    trigger.cronJobId = job.id
+    const db = new CouchDB(appId)
+    const response = await db.put(automation)
+    automation._id = response.id
+    automation._rev = response.rev
+  }
+  return automation
+}
+
+/**
+ * This function handles checking if any webhooks need to be created or deleted for automations.
+ * @param {string} appId The ID of the app in which we are checking for webhooks
+ * @param {object|undefined} oldAuto The old automation object if updating/deleting
+ * @param {object|undefined} newAuto The new automation object if creating/updating
+ * @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
+ * written to DB (this does not write to DB as it would be wasteful to repeat).
+ */
+exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
+  const oldTrigger = oldAuto ? oldAuto.definition.trigger : null
+  const newTrigger = newAuto ? newAuto.definition.trigger : null
+  const triggerChanged =
+    oldTrigger && newTrigger && oldTrigger.id !== newTrigger.id
+  function isWebhookTrigger(auto) {
+    return (
+      auto &&
+      auto.definition.trigger &&
+      auto.definition.trigger.stepId === WH_STEP_ID
+    )
+  }
+  // need to delete webhook
+  if (
+    isWebhookTrigger(oldAuto) &&
+    (!isWebhookTrigger(newAuto) || triggerChanged) &&
+    oldTrigger.webhookId
+  ) {
+    try {
+      let db = new CouchDB(appId)
+      // need to get the webhook to get the rev
+      const webhook = await db.get(oldTrigger.webhookId)
+      const ctx = {
+        appId,
+        params: { id: webhook._id, rev: webhook._rev },
+      }
+      // might be updating - reset the inputs to remove the URLs
+      if (newTrigger) {
+        delete newTrigger.webhookId
+        newTrigger.inputs = {}
+      }
+      await webhooks.destroy(ctx)
+    } catch (err) {
+      // don't worry about not being able to delete, if it doesn't exist all good
+    }
+  }
+  // need to create webhook
+  if (
+    (!isWebhookTrigger(oldAuto) || triggerChanged) &&
+    isWebhookTrigger(newAuto)
+  ) {
+    const ctx = {
+      appId,
+      request: {
+        body: new webhooks.Webhook(
+          "Automation webhook",
+          webhooks.WebhookType.AUTOMATION,
+          newAuto._id
+        ),
+      },
+    }
+    await webhooks.save(ctx)
+    const id = ctx.body.webhook._id
+    newTrigger.webhookId = id
+    newTrigger.inputs = {
+      schemaUrl: `api/webhooks/schema/${appId}/${id}`,
+      triggerUrl: `api/webhooks/trigger/${appId}/${id}`,
+    }
+  }
+  return newAuto
+}

package/src/constants/index.js

@@ -123,5 +123,10 @@ exports.BaseQueryVerbs = {
   DELETE: "delete",
 }
 
+exports.MetadataTypes = {
+  AUTOMATION_TEST_INPUT: "automationTestInput",
+  AUTOMATION_TEST_HISTORY: "automationTestHistory",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/db/linkedRows/index.js

@@ -36,6 +36,18 @@ exports.IncludeDocs = IncludeDocs
 exports.getLinkDocuments = getLinkDocuments
 exports.createLinkView = createLinkView
 
+function clearRelationshipFields(table, rows) {
+  for (let [key, field] of Object.entries(table.schema)) {
+    if (field.type === FieldTypes.LINK) {
+      rows = rows.map(row => {
+        delete row[key]
+        return row
+      })
+    }
+  }
+  return rows
+}
+
 async function getLinksForRows(appId, rows) {
   const tableIds = [...new Set(rows.map(el => el.tableId))]
   // start by getting all the link values for performance reasons
@@ -126,33 +138,6 @@ exports.updateLinks = async function (args) {
   }
 }
 
-/**
- * Update a row with information about the links that pertain to it.
- * @param {string} appId The instance in which this row has been created.
- * @param {object} rows The row(s) themselves which is to be updated with info (if applicable). This can be
- * a single row object or an array of rows - both will be handled.
- * @returns {Promise<object>} The updated row (this may be the same if no links were found). If an array was input
- * then an array will be output, object input -> object output.
- */
-exports.attachLinkIDs = async (appId, rows) => {
-  const links = await getLinksForRows(appId, rows)
-  // now iterate through the rows and all field information
-  for (let row of rows) {
-    // find anything that matches the row's ID we are searching for and join it
-    links
-      .filter(el => el.thisId === row._id)
-      .forEach(link => {
-        if (row[link.fieldName] == null) {
-          row[link.fieldName] = []
-        }
-        row[link.fieldName].push(link.id)
-      })
-  }
-  // if it was an array when it came in then handle it as an array in response
-  // otherwise return the first element as there was only one input
-  return rows
-}
-
 /**
  * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
  * This is required for formula fields, this may only be utilised internally (for now).
@@ -173,6 +158,9 @@ exports.attachFullLinkedDocs = async (ctx, table, rows) => {
   const links = (await getLinksForRows(appId, rows)).filter(link =>
     rows.some(row => row._id === link.thisId)
   )
+  // clear any existing links that could be dupe'd
+  rows = clearRelationshipFields(table, rows)
+  // now get the docs and combine into the rows
   let linked = await getFullLinkedDocs(ctx, appId, links)
   const linkedTables = []
   for (let row of rows) {
@@ -203,19 +191,17 @@ exports.attachFullLinkedDocs = async (ctx, table, rows) => {
 exports.squashLinksToPrimaryDisplay = async (appId, table, enriched) => {
   const db = new CouchDB(appId)
   // will populate this as we find them
-  const linkedTables = []
-  for (let [column, schema] of Object.entries(table.schema)) {
-    if (schema.type !== FieldTypes.LINK) {
-      continue
-    }
-    for (let row of enriched) {
-      if (!row[column] || !row[column].length) {
+  const linkedTables = [table]
+  for (let row of enriched) {
+    // this only fetches the table if its not already in array
+    const rowTable = await getLinkedTable(db, row.tableId, linkedTables)
+    for (let [column, schema] of Object.entries(rowTable.schema)) {
+      if (schema.type !== FieldTypes.LINK || !Array.isArray(row[column])) {
         continue
       }
       const newLinks = []
       for (let link of row[column]) {
         const linkTblId = link.tableId || getRelatedTableForField(table, column)
-        // this only fetches the table if its not already in array
         const linkedTable = await getLinkedTable(db, linkTblId, linkedTables)
         const obj = { _id: link._id }
         if (link[linkedTable.primaryDisplay]) {

package/src/db/tests/linkTests.spec.js

@@ -59,16 +59,4 @@ describe("test link functionality", () => {
       expect(Array.isArray(output)).toBe(true)
     })
   })
-
-  describe("attachLinkIDs", () => {
-    it("should be able to attach linkIDs", async () => {
-      await config.init()
-      await config.createTable()
-      const table = await config.createLinkedTable()
-      const row = await config.createRow()
-      const linkRow = await config.createRow(basicLinkedRow(table._id, row._id))
-      const attached = await links.attachLinkIDs(config.getAppId(), [linkRow])
-      expect(attached[0].link[0]).toBe(row._id)
-    })
-  })
 })

package/src/db/utils.js

@@ -7,6 +7,8 @@ const {
   APP_PREFIX,
   SEPARATOR,
   StaticDatabases,
+  isDevAppID,
+  isProdAppID,
 } = require("@budibase/auth/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -36,6 +38,7 @@ const DocumentTypes = {
   DATASOURCE_PLUS: "datasource_plus",
   QUERY: "query",
   DEPLOYMENTS: "deployments",
+  METADATA: "metadata",
 }
 
 const ViewNames = {
@@ -63,6 +66,8 @@ const BudibaseInternalDB = {
 
 exports.APP_PREFIX = APP_PREFIX
 exports.APP_DEV_PREFIX = APP_DEV_PREFIX
+exports.isDevAppID = isDevAppID
+exports.isProdAppID = isProdAppID
 exports.USER_METDATA_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.LINK_USER_METADATA_PREFIX = `${DocumentTypes.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.ViewNames = ViewNames
@@ -331,6 +336,18 @@ exports.getQueryParams = (datasourceId = null, otherProps = {}) => {
   )
 }
 
+exports.generateMetadataID = (type, entityId) => {
+  return `${DocumentTypes.METADATA}${SEPARATOR}${type}${SEPARATOR}${entityId}`
+}
+
+exports.getMetadataParams = (type, entityId = null, otherProps = {}) => {
+  let docId = `${type}${SEPARATOR}`
+  if (entityId != null) {
+    docId += entityId
+  }
+  return getDocParams(DocumentTypes.METADATA, docId, otherProps)
+}
+
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */

package/src/environment.js

@@ -26,7 +26,7 @@ module.exports = {
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
-  SELF_HOSTED: process.env.SELF_HOSTED,
+  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
   AWS_REGION: process.env.AWS_REGION,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
@@ -55,6 +55,7 @@ module.exports = {
   BUDIBASE_API_KEY: process.env.BUDIBASE_API_KEY,
   USERID_API_KEY: process.env.USERID_API_KEY,
   DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
+  ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value