@budibase/server 1.0.212-alpha.10 → 1.0.212-alpha.13

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.212-alpha.10",
+  "version": "1.0.212-alpha.13",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.0.212-alpha.10",
-    "@budibase/client": "^1.0.212-alpha.10",
-    "@budibase/pro": "1.0.212-alpha.9",
-    "@budibase/string-templates": "^1.0.212-alpha.10",
-    "@budibase/types": "^1.0.212-alpha.10",
+    "@budibase/backend-core": "^1.0.212-alpha.13",
+    "@budibase/client": "^1.0.212-alpha.13",
+    "@budibase/pro": "1.0.212-alpha.12",
+    "@budibase/string-templates": "^1.0.212-alpha.13",
+    "@budibase/types": "^1.0.212-alpha.13",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -195,5 +195,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "d5a8da563fdf65795bf908de0359f0ea5e9a9b1d"
+  "gitHead": "1b460f45a10f98573cb9fe7ca5269550155c5bed"
 }

package/src/api/controllers/application.ts

@@ -45,10 +45,12 @@ const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 import { syncGlobalUsers } from "./user"
 const { app: appCache } = require("@budibase/backend-core/cache")
 import { cleanupAutomations } from "../../automations/utils"
+import { checkAppMetadata } from "../../automations/logging"
 const {
   getAppDB,
   getProdAppDB,
   updateAppId,
+  doInAppContext,
 } = require("@budibase/backend-core/context")
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
@@ -192,7 +194,7 @@ export const fetch = async (ctx: any) => {
     }
   }
 
-  ctx.body = apps
+  ctx.body = await checkAppMetadata(apps)
 }
 
 export const fetchAppDefinition = async (ctx: any) => {
@@ -548,22 +550,24 @@ export const sync = async (ctx: any, next: any) => {
 }
 
 const updateAppPackage = async (appPackage: any, appId: any) => {
-  const db = getAppDB()
-  const application = await db.get(DocumentTypes.APP_METADATA)
+  return doInAppContext(appId, async () => {
+    const db = getAppDB()
+    const application = await db.get(DocumentTypes.APP_METADATA)
 
-  const newAppPackage = { ...application, ...appPackage }
-  if (appPackage._rev !== application._rev) {
-    newAppPackage._rev = application._rev
-  }
+    const newAppPackage = { ...application, ...appPackage }
+    if (appPackage._rev !== application._rev) {
+      newAppPackage._rev = application._rev
+    }
 
-  // the locked by property is attached by server but generated from
-  // Redis, shouldn't ever store it
-  delete newAppPackage.lockedBy
+    // the locked by property is attached by server but generated from
+    // Redis, shouldn't ever store it
+    delete newAppPackage.lockedBy
 
-  await db.put(newAppPackage)
-  // remove any cached metadata, so that it will be updated
-  await appCache.invalidateAppMetadata(appId)
-  return newAppPackage
+    await db.put(newAppPackage)
+    // remove any cached metadata, so that it will be updated
+    await appCache.invalidateAppMetadata(appId)
+    return newAppPackage
+  })
 }
 
 const createEmptyAppPackage = async (ctx: any, app: any) => {

package/src/api/controllers/automation.js

@@ -1,6 +1,10 @@
 const actions = require("../../automations/actions")
 const triggers = require("../../automations/triggers")
-const { getAutomationParams, generateAutomationID } = require("../../db/utils")
+const {
+  getAutomationParams,
+  generateAutomationID,
+  DocumentTypes,
+} = require("../../db/utils")
 const {
   checkForWebhooks,
   updateTestHistory,
@@ -9,8 +13,14 @@ const {
 const { deleteEntityMetadata } = require("../../utilities")
 const { MetadataTypes } = require("../../constants")
 const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
-const { getAppDB } = require("@budibase/backend-core/context")
+const {
+  getAppDB,
+  getProdAppDB,
+  doInAppContext,
+} = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
+const { app } = require("@budibase/backend-core/cache")
+const { automations } = require("@budibase/pro")
 
 const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
 const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@@ -183,6 +193,29 @@ exports.destroy = async function (ctx) {
   await events.automation.deleted(oldAutomation)
 }
 
+exports.logSearch = async function (ctx) {
+  ctx.body = await automations.logs.logSearch(ctx.request.body)
+}
+
+exports.clearLogError = async function (ctx) {
+  const { automationId, appId } = ctx.request.body
+  await doInAppContext(appId, async () => {
+    const db = getProdAppDB()
+    const metadata = await db.get(DocumentTypes.APP_METADATA)
+    if (!automationId) {
+      delete metadata.automationErrors
+    } else if (
+      metadata.automationErrors &&
+      metadata.automationErrors[automationId]
+    ) {
+      delete metadata.automationErrors[automationId]
+    }
+    await db.put(metadata)
+    await app.invalidateAppMetadata(metadata.appId, metadata)
+    ctx.body = { message: `Error logs cleared.` }
+  })
+}
+
 exports.getActionList = async function (ctx) {
   ctx.body = ACTION_DEFS
 }

package/src/api/index.js

@@ -72,7 +72,10 @@ router.use(async (ctx, next) => {
       error,
     }
     ctx.log.error(err)
-    console.trace(err)
+    // unauthorised errors don't provide a useful trace
+    if (!env.isTest()) {
+      console.trace(err)
+    }
   }
 })
 

package/src/api/routes/automation.js

@@ -51,6 +51,16 @@ router
     automationValidator(false),
     controller.create
   )
+  .post(
+    "/api/automations/logs/search",
+    authorized(BUILDER),
+    controller.logSearch
+  )
+  .delete(
+    "/api/automations/logs",
+    authorized(BUILDER),
+    controller.clearLogError
+  )
   .delete(
     "/api/automations/:id/:rev",
     paramResource("id"),

package/src/automations/logging/index.ts

@@ -0,0 +1,35 @@
+import * as env from "../../environment"
+import { AutomationResults, Automation, App } from "@budibase/types"
+import { automations } from "@budibase/pro"
+import { db as dbUtils } from "@budibase/backend-core"
+
+export async function storeLog(
+  automation: Automation,
+  results: AutomationResults
+) {
+  // can disable this if un-needed in self-host, also only do this for prod apps
+  if (env.DISABLE_AUTOMATION_LOGS) {
+    return
+  }
+  await automations.logs.storeLog(automation, results)
+}
+
+export async function checkAppMetadata(apps: App[]) {
+  const maxStartDate = await automations.logs.oldestLogDate()
+  for (let metadata of apps) {
+    if (!metadata.automationErrors) {
+      continue
+    }
+    for (let [key, errors] of Object.entries(metadata.automationErrors)) {
+      const updated = []
+      for (let error of errors) {
+        const startDate = error.split(dbUtils.SEPARATOR)[2]
+        if (startDate > maxStartDate) {
+          updated.push(error)
+        }
+      }
+      metadata.automationErrors[key] = updated
+    }
+  }
+  return apps
+}

package/src/automations/steps/filter.js

@@ -50,43 +50,51 @@ exports.definition = {
     outputs: {
       properties: {
         success: {
+          type: "boolean",
+          description: "Whether the action was successful",
+        },
+        result: {
           type: "boolean",
           description: "Whether the logic block passed",
         },
       },
-      required: ["success"],
+      required: ["success", "result"],
     },
   },
 }
 
 exports.run = async function filter({ inputs }) {
-  let { field, condition, value } = inputs
-  // coerce types so that we can use them
-  if (!isNaN(value) && !isNaN(field)) {
-    value = parseFloat(value)
-    field = parseFloat(field)
-  } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
-    value = Date.parse(value)
-    field = Date.parse(field)
-  }
-  let success = false
-  if (typeof field !== "object" && typeof value !== "object") {
-    switch (condition) {
-      case FilterConditions.EQUAL:
-        success = field === value
-        break
-      case FilterConditions.NOT_EQUAL:
-        success = field !== value
-        break
-      case FilterConditions.GREATER_THAN:
-        success = field > value
-        break
-      case FilterConditions.LESS_THAN:
-        success = field < value
-        break
+  try {
+    let { field, condition, value } = inputs
+    // coerce types so that we can use them
+    if (!isNaN(value) && !isNaN(field)) {
+      value = parseFloat(value)
+      field = parseFloat(field)
+    } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
+      value = Date.parse(value)
+      field = Date.parse(field)
+    }
+    let result = false
+    if (typeof field !== "object" && typeof value !== "object") {
+      switch (condition) {
+        case FilterConditions.EQUAL:
+          result = field === value
+          break
+        case FilterConditions.NOT_EQUAL:
+          result = field !== value
+          break
+        case FilterConditions.GREATER_THAN:
+          result = field > value
+          break
+        case FilterConditions.LESS_THAN:
+          result = field < value
+          break
+      }
+    } else {
+      result = false
     }
-  } else {
-    success = false
+    return { success: true, result }
+  } catch (err) {
+    return { success: false, result: false }
   }
-  return { success }
 }

package/src/automations/tests/filter.spec.js

@@ -6,7 +6,8 @@ describe("test the filter logic", () => {
     let res = await setup.runStep(setup.actions.FILTER.stepId,
       { field, condition, value }
     )
-    expect(res.success).toEqual(pass)
+    expect(res.result).toEqual(pass)
+    expect(res.success).toEqual(true)
   }
 
   it("should be able test equality", async () => {

package/src/db/linkedRows/index.js

@@ -65,6 +65,7 @@ async function getLinksForRows(rows) {
   // return duplicates, could be querying for both tables in a relation
   return getUniqueByProp(
     responses
+      .filter(el => el != null)
       // create a unique ID which we can use for getting only unique ones
       .map(el => ({ ...el, unique: el.id + el.thisId + el.fieldName })),
     "unique"

package/src/db/utils.js

@@ -11,6 +11,8 @@ const {
   isProdAppID,
   getDevelopmentAppID,
   generateAppID,
+  getQueryIndex,
+  ViewNames,
 } = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -22,11 +24,7 @@ const AppStatus = {
 }
 
 const DocumentTypes = {
-  APP: CoreDocTypes.APP,
-  DEV: CoreDocTypes.DEV,
-  APP_DEV: CoreDocTypes.APP_DEV,
-  APP_METADATA: CoreDocTypes.APP_METADATA,
-  ROLE: CoreDocTypes.ROLE,
+  ...CoreDocTypes,
   TABLE: "ta",
   ROW: "ro",
   USER: "us",
@@ -45,11 +43,6 @@ const DocumentTypes = {
   USER_FLAG: "flag",
 }
 
-const ViewNames = {
-  LINK: "by_link",
-  ROUTING: "screen_routes",
-}
-
 const InternalTables = {
   USER_METADATA: "ta_users",
 }
@@ -89,9 +82,7 @@ exports.generateDevAppID = getDevelopmentAppID
 exports.generateRoleID = generateRoleID
 exports.getRoleParams = getRoleParams
 
-exports.getQueryIndex = viewName => {
-  return `database/${viewName}`
-}
+exports.getQueryIndex = getQueryIndex
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this

package/src/environment.js

@@ -78,6 +78,7 @@ module.exports = {
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
+  DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,

package/src/module.d.ts

@@ -8,3 +8,4 @@ declare module "@budibase/backend-core/constants"
 declare module "@budibase/backend-core/auth"
 declare module "@budibase/backend-core/sessions"
 declare module "@budibase/backend-core/encryption"
+declare module "@budibase/backend-core/redis"

package/src/threads/automation.js

@@ -9,11 +9,12 @@ const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const { AutomationErrors, LoopStepTypes } = require("../constants")
+const { storeLog } = require("../automations/logging")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 
 const CRON_STEP_ID = triggerDefs.CRON.stepId
-const STOPPED_STATUS = { success: false, status: "STOPPED" }
+const STOPPED_STATUS = { success: true, status: "STOPPED" }
 const { cloneDeep } = require("lodash/fp")
 const env = require("../environment")
 
@@ -275,7 +276,7 @@ class Orchestrator {
           this._context.steps[stepCount] = outputs
           // if filter causes us to stop execution don't break the loop, set a var
           // so that we can finish iterating through the steps and record that it stopped
-          if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+          if (step.stepId === FILTER_STEP_ID && !outputs.result) {
             stopped = true
             this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
               ...outputs,
@@ -325,6 +326,8 @@ class Orchestrator {
       }
     }
 
+    // store the logs for the automation run
+    await storeLog(this._automation, this.executionOutput)
     return this.executionOutput
   }
 }

package/src/threads/index.ts

@@ -77,7 +77,7 @@ export class Thread {
     })
   }
 
-  static shutdown() {
+  static stopThreads() {
     return new Promise<void>(resolve => {
       if (Thread.workerRefs.length === 0) {
         resolve()
@@ -95,4 +95,8 @@ export class Thread {
       Thread.workerRefs = []
     })
   }
+
+  static async shutdown() {
+    await Thread.stopThreads()
+  }
 }