@budibase/server 0.0.32 → 0.0.999-alpha.30

package/src/api/routes/utils/validators.ts

@@ -0,0 +1,240 @@
+import { auth, permissions } from "@budibase/backend-core"
+import { DataSourceOperation } from "../../../constants"
+import { WebhookActionType } from "@budibase/types"
+import Joi from "joi"
+
+const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
+const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
+const OPTIONAL_BOOLEAN = Joi.boolean().optional().allow(null)
+const APP_NAME_REGEX = /^[\w\s]+$/
+
+export function tableValidator() {
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    _id: OPTIONAL_STRING,
+    _rev: OPTIONAL_STRING,
+    type: OPTIONAL_STRING.valid("table", "internal", "external"),
+    primaryDisplay: OPTIONAL_STRING,
+    schema: Joi.object().required(),
+    name: Joi.string().required(),
+    views: Joi.object(),
+    rows: Joi.array(),
+  }).unknown(true))
+}
+
+export function nameValidator() {
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    name: OPTIONAL_STRING,
+  }))
+}
+
+export function datasourceValidator() {
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    _id: Joi.string(),
+    _rev: Joi.string(),
+    type: OPTIONAL_STRING.allow("datasource_plus"),
+    relationships: Joi.array().items(Joi.object({
+      from: Joi.string().required(),
+      to: Joi.string().required(),
+      cardinality: Joi.valid("1:N", "1:1", "N:N").required()
+    })),
+  }).unknown(true))
+}
+
+function filterObject() {
+  // prettier-ignore
+  return Joi.object({
+    string: Joi.object().optional(),
+    fuzzy: Joi.object().optional(),
+    range: Joi.object().optional(),
+    equal: Joi.object().optional(),
+    notEqual: Joi.object().optional(),
+    empty: Joi.object().optional(),
+    notEmpty: Joi.object().optional(),
+    oneOf: Joi.object().optional(),
+    contains: Joi.object().optional(),
+    notContains: Joi.object().optional(),
+    allOr: Joi.boolean().optional(),
+  }).unknown(true)
+}
+
+export function internalSearchValidator() {
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    tableId: OPTIONAL_STRING,
+    query: filterObject(),
+    limit: OPTIONAL_NUMBER,
+    sort: OPTIONAL_STRING,
+    sortOrder: OPTIONAL_STRING,
+    sortType: OPTIONAL_STRING,
+    paginate: Joi.boolean(),
+    bookmark: Joi.alternatives().try(OPTIONAL_STRING, OPTIONAL_NUMBER).optional(),
+  }))
+}
+
+export function externalSearchValidator() {
+  return auth.joiValidator.body(
+    Joi.object({
+      query: filterObject(),
+      paginate: Joi.boolean().optional(),
+      bookmark: Joi.alternatives()
+        .try(OPTIONAL_STRING, OPTIONAL_NUMBER)
+        .optional(),
+      limit: OPTIONAL_NUMBER,
+      sort: Joi.object({
+        column: Joi.string(),
+        order: OPTIONAL_STRING.valid("ascending", "descending"),
+        type: OPTIONAL_STRING.valid("string", "number"),
+      }).optional(),
+    })
+  )
+}
+
+export function datasourceQueryValidator() {
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    endpoint: Joi.object({
+      datasourceId: Joi.string().required(),
+      operation: Joi.string().required().valid(...Object.values(DataSourceOperation)),
+      entityId: Joi.string().required(),
+    }).required(),
+    resource: Joi.object({
+      fields: Joi.array().items(Joi.string()).optional(),
+    }).optional(),
+    body: Joi.object().optional(),
+    sort: Joi.object().optional(),
+    filters: filterObject().optional(),
+    paginate: Joi.object({
+      page: Joi.string().alphanum().optional(),
+      limit: Joi.number().optional(),
+    }).optional(),
+  }))
+}
+
+export function webhookValidator() {
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    live: Joi.bool(),
+    _id: OPTIONAL_STRING,
+    _rev: OPTIONAL_STRING,
+    name: Joi.string().required(),
+    bodySchema: Joi.object().optional(),
+    action: Joi.object({
+      type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
+      target: Joi.string().required(),
+    }).required(),
+  }).unknown(true))
+}
+
+export function roleValidator() {
+  const permLevelArray = Object.values(permissions.PermissionLevel)
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    _id: OPTIONAL_STRING,
+    _rev: OPTIONAL_STRING,
+    name: Joi.string().required(),
+    // this is the base permission ID (for now a built in)
+    permissionId: Joi.string().valid(...Object.values(permissions.BuiltinPermissionID)).required(),
+    permissions: Joi.object()
+      .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+      .optional(),
+    inherits: OPTIONAL_STRING,
+  }).unknown(true))
+}
+
+export function permissionValidator() {
+  const permLevelArray = Object.values(permissions.PermissionLevel)
+  // prettier-ignore
+  return auth.joiValidator.params(Joi.object({
+    level: Joi.string().valid(...permLevelArray).required(),
+    resourceId: Joi.string(),
+    roleId: Joi.string(),
+  }).unknown(true))
+}
+
+export function screenValidator() {
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    name: Joi.string().required(),
+    showNavigation: OPTIONAL_BOOLEAN,
+    width: OPTIONAL_STRING,
+    routing: Joi.object({
+      route: Joi.string().required(),
+      roleId: Joi.string().required().allow(""),
+      homeScreen: OPTIONAL_BOOLEAN,
+    }).required().unknown(true),
+    props: Joi.object({
+      _id: Joi.string().required(),
+      _component: Joi.string().required(),
+      _children: Joi.array().required(),
+      _styles: Joi.object().required(),
+      type: OPTIONAL_STRING,
+      table: OPTIONAL_STRING,
+      layoutId: OPTIONAL_STRING,
+    }).required().unknown(true),
+  }).unknown(true))
+}
+
+function generateStepSchema(allowStepTypes: string[]) {
+  // prettier-ignore
+  return Joi.object({
+    stepId: Joi.string().required(),
+    id: Joi.string().required(),
+    description: Joi.string().required(),
+    name: Joi.string().required(),
+    tagline: Joi.string().required(),
+    icon: Joi.string().required(),
+    params: Joi.object(),
+    args: Joi.object(),
+    type: Joi.string().required().valid(...allowStepTypes),
+  }).unknown(true)
+}
+
+export function automationValidator(existing = false) {
+  // prettier-ignore
+  return auth.joiValidator.body(Joi.object({
+    _id: existing ? Joi.string().required() : OPTIONAL_STRING,
+    _rev: existing ? Joi.string().required() : OPTIONAL_STRING,
+    name: Joi.string().required(),
+    type: Joi.string().valid("automation").required(),
+    definition: Joi.object({
+      steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])),
+      trigger: generateStepSchema(["TRIGGER"]).allow(null),
+    }).required().unknown(true),
+  }).unknown(true))
+}
+
+export function applicationValidator(opts = { isCreate: true }) {
+  // prettier-ignore
+  const base: any = {
+    _id: OPTIONAL_STRING,
+    _rev: OPTIONAL_STRING,
+    url: OPTIONAL_STRING,
+    template: Joi.object({
+      templateString: OPTIONAL_STRING,
+    })
+  }
+
+  const appNameValidator = Joi.string()
+    .pattern(new RegExp(APP_NAME_REGEX))
+    .error(new Error("App name must be letters, numbers and spaces only"))
+  if (opts.isCreate) {
+    base.name = appNameValidator.required()
+  } else {
+    base.name = appNameValidator.optional()
+  }
+
+  return auth.joiValidator.body(
+    Joi.object({
+      _id: OPTIONAL_STRING,
+      _rev: OPTIONAL_STRING,
+      name: appNameValidator,
+      url: OPTIONAL_STRING,
+      template: Joi.object({
+        templateString: OPTIONAL_STRING,
+      }).unknown(true),
+    }).unknown(true)
+  )
+}

package/src/api/routes/view.ts

@@ -0,0 +1,34 @@
+import Router from "@koa/router"
+import * as viewController from "../controllers/view"
+import * as rowController from "../controllers/row"
+import authorized from "../../middleware/authorized"
+import { paramResource } from "../../middleware/resourceId"
+import { permissions } from "@budibase/backend-core"
+
+const router: Router = new Router()
+
+router
+  .get(
+    "/api/views/export",
+    authorized(permissions.BUILDER),
+    viewController.exportView
+  )
+  .get(
+    "/api/views/:viewName",
+    paramResource("viewName"),
+    authorized(
+      permissions.PermissionType.VIEW,
+      permissions.PermissionLevel.READ
+    ),
+    rowController.fetchView
+  )
+  .get("/api/views", authorized(permissions.BUILDER), viewController.fetch)
+  .delete(
+    "/api/views/:viewName",
+    paramResource("viewName"),
+    authorized(permissions.BUILDER),
+    viewController.destroy
+  )
+  .post("/api/views", authorized(permissions.BUILDER), viewController.save)
+
+export default router

package/src/api/routes/webhook.ts

@@ -0,0 +1,27 @@
+import Router from "@koa/router"
+import * as controller from "../controllers/webhook"
+import authorized from "../../middleware/authorized"
+import { permissions } from "@budibase/backend-core"
+import { webhookValidator } from "./utils/validators"
+
+const BUILDER = permissions.BUILDER
+const router: Router = new Router()
+
+router
+  .get("/api/webhooks", authorized(BUILDER), controller.fetch)
+  .put(
+    "/api/webhooks",
+    authorized(BUILDER),
+    webhookValidator(),
+    controller.save
+  )
+  .delete("/api/webhooks/:id/:rev", authorized(BUILDER), controller.destroy)
+  .post(
+    "/api/webhooks/schema/:instance/:id",
+    authorized(BUILDER),
+    controller.buildSchema
+  )
+  // this shouldn't have authorisation, right now its always public
+  .post("/api/webhooks/trigger/:instance/:id", controller.trigger)
+
+export default router

package/src/app.ts

@@ -0,0 +1,114 @@
+if (process.env.DD_APM_ENABLED) {
+  require("./ddApm")
+}
+
+// need to load environment first
+import env from "./environment"
+
+import { ExtendableContext } from "koa"
+import * as db from "./db"
+db.init()
+import Koa from "koa"
+import koaBody from "koa-body"
+import http from "http"
+import * as api from "./api"
+import * as automations from "./automations"
+import { Thread } from "./threads"
+import * as redis from "./utilities/redis"
+import { initialise as initialiseWebsockets } from "./websockets"
+import { events, logging, middleware, timers } from "@budibase/backend-core"
+import { startup } from "./startup"
+const Sentry = require("@sentry/node")
+const destroyable = require("server-destroy")
+const { userAgent } = require("koa-useragent")
+
+const app = new Koa()
+
+let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10")
+if (!mbNumber || isNaN(mbNumber)) {
+  mbNumber = 10
+}
+// set up top level koa middleware
+app.use(
+  koaBody({
+    multipart: true,
+    formLimit: `${mbNumber}mb`,
+    jsonLimit: `${mbNumber}mb`,
+    textLimit: `${mbNumber}mb`,
+    // @ts-ignore
+    enableTypes: ["json", "form", "text"],
+    parsedMethods: ["POST", "PUT", "PATCH", "DELETE"],
+  })
+)
+
+app.use(middleware.correlation)
+app.use(middleware.pino)
+app.use(userAgent)
+
+if (env.isProd()) {
+  env._set("NODE_ENV", "production")
+  Sentry.init()
+
+  app.on("error", (err: any, ctx: ExtendableContext) => {
+    Sentry.withScope(function (scope: any) {
+      scope.addEventProcessor(function (event: any) {
+        return Sentry.Handlers.parseRequest(event, ctx.request)
+      })
+      Sentry.captureException(err)
+    })
+  })
+}
+
+const server = http.createServer(app.callback())
+destroyable(server)
+initialiseWebsockets(app, server)
+
+let shuttingDown = false,
+  errCode = 0
+
+server.on("close", async () => {
+  // already in process
+  if (shuttingDown) {
+    return
+  }
+  shuttingDown = true
+  console.log("Server Closed")
+  timers.cleanup()
+  await automations.shutdown()
+  await redis.shutdown()
+  events.shutdown()
+  await Thread.shutdown()
+  api.shutdown()
+  if (!env.isTest()) {
+    process.exit(errCode)
+  }
+})
+
+export default server.listen(env.PORT || 0, async () => {
+  await startup(app, server)
+})
+
+const shutdown = () => {
+  server.close()
+  // @ts-ignore
+  server.destroy()
+}
+
+process.on("uncaughtException", err => {
+  // @ts-ignore
+  // don't worry about this error, comes from zlib isn't important
+  if (err && err["code"] === "ERR_INVALID_CHAR") {
+    return
+  }
+  errCode = -1
+  logging.logAlert("Uncaught exception.", err)
+  shutdown()
+})
+
+process.on("SIGTERM", () => {
+  shutdown()
+})
+
+process.on("SIGINT", () => {
+  shutdown()
+})

package/src/automations/actions.ts

@@ -0,0 +1,109 @@
+import * as sendSmtpEmail from "./steps/sendSmtpEmail"
+import * as createRow from "./steps/createRow"
+import * as updateRow from "./steps/updateRow"
+import * as deleteRow from "./steps/deleteRow"
+import * as executeScript from "./steps/executeScript"
+import * as executeQuery from "./steps/executeQuery"
+import * as outgoingWebhook from "./steps/outgoingWebhook"
+import * as serverLog from "./steps/serverLog"
+import * as discord from "./steps/discord"
+import * as slack from "./steps/slack"
+import * as zapier from "./steps/zapier"
+import * as integromat from "./steps/integromat"
+import * as filter from "./steps/filter"
+import * as delay from "./steps/delay"
+import * as queryRow from "./steps/queryRows"
+import * as loop from "./steps/loop"
+import env from "../environment"
+import {
+  AutomationStepSchema,
+  AutomationStepInput,
+  PluginType,
+  AutomationStep,
+} from "@budibase/types"
+import sdk from "../sdk"
+import { getAutomationPlugin } from "../utilities/fileSystem"
+
+const ACTION_IMPLS: Record<
+  string,
+  (opts: AutomationStepInput) => Promise<any>
+> = {
+  SEND_EMAIL_SMTP: sendSmtpEmail.run,
+  CREATE_ROW: createRow.run,
+  UPDATE_ROW: updateRow.run,
+  DELETE_ROW: deleteRow.run,
+  OUTGOING_WEBHOOK: outgoingWebhook.run,
+  EXECUTE_SCRIPT: executeScript.run,
+  EXECUTE_QUERY: executeQuery.run,
+  SERVER_LOG: serverLog.run,
+  DELAY: delay.run,
+  FILTER: filter.run,
+  QUERY_ROWS: queryRow.run,
+  // these used to be lowercase step IDs, maintain for backwards compat
+  discord: discord.run,
+  slack: slack.run,
+  zapier: zapier.run,
+  integromat: integromat.run,
+}
+export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
+  {
+    SEND_EMAIL_SMTP: sendSmtpEmail.definition,
+    CREATE_ROW: createRow.definition,
+    UPDATE_ROW: updateRow.definition,
+    DELETE_ROW: deleteRow.definition,
+    OUTGOING_WEBHOOK: outgoingWebhook.definition,
+    EXECUTE_SCRIPT: executeScript.definition,
+    EXECUTE_QUERY: executeQuery.definition,
+    SERVER_LOG: serverLog.definition,
+    DELAY: delay.definition,
+    FILTER: filter.definition,
+    QUERY_ROWS: queryRow.definition,
+    LOOP: loop.definition,
+    // these used to be lowercase step IDs, maintain for backwards compat
+    discord: discord.definition,
+    slack: slack.definition,
+    zapier: zapier.definition,
+    integromat: integromat.definition,
+  }
+
+// don't add the bash script/definitions unless in self host
+// the fact this isn't included in any definitions means it cannot be
+// ran at all
+if (env.SELF_HOSTED) {
+  const bash = require("./steps/bash")
+  // @ts-ignore
+  ACTION_IMPLS["EXECUTE_BASH"] = bash.run
+  // @ts-ignore
+  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+}
+
+export async function getActionDefinitions() {
+  const actionDefinitions = BUILTIN_ACTION_DEFINITIONS
+  if (env.SELF_HOSTED) {
+    const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
+    for (let plugin of plugins) {
+      const schema = plugin.schema.schema as AutomationStep
+      actionDefinitions[schema.stepId] = {
+        ...schema,
+        custom: true,
+      }
+    }
+  }
+  return actionDefinitions
+}
+
+/* istanbul ignore next */
+export async function getAction(stepId: string) {
+  if (ACTION_IMPLS[stepId] != null) {
+    return ACTION_IMPLS[stepId]
+  }
+  // must be a plugin
+  if (env.SELF_HOSTED) {
+    const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
+    const found = plugins.find(plugin => plugin.schema.schema.stepId === stepId)
+    if (!found) {
+      throw new Error(`Unable to find action implementation for "${stepId}"`)
+    }
+    return (await getAutomationPlugin(found)).action
+  }
+}

package/src/automations/automationUtils.ts

@@ -0,0 +1,163 @@
+import {
+  decodeJSBinding,
+  isJSBinding,
+  encodeJSBinding,
+} from "@budibase/string-templates"
+import sdk from "../sdk"
+import { Row } from "@budibase/types"
+import { LoopStep, LoopStepType, LoopInput } from "../definitions/automations"
+
+/**
+ * When values are input to the system generally they will be of type string as this is required for template strings.
+ * This can generate some odd scenarios as the Schema of the automation requires a number but the builder might supply
+ * a string with template syntax to get the number from the rest of the context. To support this the server has to
+ * make sure that the post template statement can be cast into the correct type, this function does this for numbers
+ * and booleans.
+ *
+ * @param {object} inputs An object of inputs, please note this will not recurse down into any objects within, it simply
+ * cleanses the top level inputs, however it can be used by recursively calling it deeper into the object structures if
+ * the schema is known.
+ * @param {object} schema The defined schema of the inputs, in the form of JSON schema. The schema definition of an
+ * automation is the likely use case of this, however validate.js syntax can be converted closely enough to use this by
+ * wrapping the schema properties in a top level "properties" object.
+ * @returns {object} The inputs object which has had all the various types supported by this function converted to their
+ * primitive types.
+ */
+export function cleanInputValues(inputs: Record<string, any>, schema?: any) {
+  if (schema == null) {
+    return inputs
+  }
+  for (let inputKey of Object.keys(inputs)) {
+    let input = inputs[inputKey]
+    if (typeof input !== "string") {
+      continue
+    }
+    let propSchema = schema.properties[inputKey]
+    if (!propSchema) {
+      continue
+    }
+    if (propSchema.type === "boolean") {
+      let lcInput = input.toLowerCase()
+      if (lcInput === "true") {
+        inputs[inputKey] = true
+      }
+      if (lcInput === "false") {
+        inputs[inputKey] = false
+      }
+    }
+    if (propSchema.type === "number") {
+      let floatInput = parseFloat(input)
+      if (!isNaN(floatInput)) {
+        inputs[inputKey] = floatInput
+      }
+    }
+  }
+  //Check if input field for Update Row should be a relationship and cast to array
+  for (let key in inputs.row) {
+    if (
+      inputs.schema?.[key]?.type === "link" &&
+      inputs.row[key] &&
+      typeof inputs.row[key] === "string"
+    ) {
+      try {
+        inputs.row[key] = JSON.parse(inputs.row[key])
+      } catch (e) {
+        //Link is not an array or object, so continue
+      }
+    }
+  }
+  return inputs
+}
+
+/**
+ * Given a row input like a save or update row we need to clean the inputs against a schema that is not part of
+ * the automation but is instead part of the Table/Table. This function will get the table schema and use it to instead
+ * perform the cleanInputValues function on the input row.
+ *
+ * @param {string} tableId The ID of the Table/Table which the schema is to be retrieved for.
+ * @param {object} row The input row structure which requires clean-up after having been through template statements.
+ * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
+ */
+export async function cleanUpRow(tableId: string, row: Row) {
+  let table = await sdk.tables.getTable(tableId)
+  return cleanInputValues(row, { properties: table.schema })
+}
+
+export function getError(err: any) {
+  if (err == null) {
+    return "No error provided."
+  }
+  if (
+    typeof err === "object" &&
+    (err.toString == null || err.toString() === "[object Object]")
+  ) {
+    return JSON.stringify(err)
+  }
+  return typeof err !== "string" ? err.toString() : err
+}
+
+export function substituteLoopStep(hbsString: string, substitute: string) {
+  let checkForJS = isJSBinding(hbsString)
+  let substitutedHbsString = ""
+  let open = checkForJS ? `$("` : "{{"
+  let closed = checkForJS ? `")` : "}}"
+  if (checkForJS) {
+    hbsString = decodeJSBinding(hbsString) as string
+  }
+  let pointer = 0,
+    openPointer = 0,
+    closedPointer = 0
+  while (pointer < hbsString?.length) {
+    openPointer = hbsString.indexOf(open, pointer)
+    closedPointer = hbsString.indexOf(closed, pointer) + 2
+    if (openPointer < 0 || closedPointer < 0) {
+      substitutedHbsString += hbsString.substring(pointer)
+      break
+    }
+    let before = hbsString.substring(pointer, openPointer)
+    let block = hbsString
+      .substring(openPointer, closedPointer)
+      .replace(/loop/, substitute)
+    substitutedHbsString += before + block
+    pointer = closedPointer
+  }
+  if (checkForJS) {
+    substitutedHbsString = encodeJSBinding(substitutedHbsString)
+  }
+  return substitutedHbsString
+}
+
+export function stringSplit(value: string | string[]) {
+  if (value == null || Array.isArray(value)) {
+    return value || []
+  }
+  if (value.split("\n").length > 1) {
+    value = value.split("\n")
+  } else {
+    value = value.split(",")
+  }
+  return value
+}
+
+export function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
+  if (!input || !input.binding) {
+    return null
+  }
+  try {
+    switch (loopStep.inputs.option) {
+      case LoopStepType.ARRAY:
+        if (typeof input.binding === "string") {
+          return JSON.parse(input.binding)
+        }
+        break
+      case LoopStepType.STRING:
+        if (Array.isArray(input.binding)) {
+          return input.binding.join(",")
+        }
+        break
+    }
+  } catch (err) {
+    throw new Error("Unable to cast to correct type")
+  }
+  return input.binding
+}