@budibase/backend-core 2.9.19 → 2.9.21-alpha.0

package/src/plugin/tests/validation.spec.ts

@@ -1,83 +0,0 @@
-import { validate } from "../utils"
-import fetch from "node-fetch"
-import { PluginType } from "@budibase/types"
-
-const repoUrl =
-  "https://raw.githubusercontent.com/Budibase/budibase-skeleton/master"
-const automationLink = `${repoUrl}/automation/schema.json.hbs`
-const componentLink = `${repoUrl}/component/schema.json.hbs`
-const datasourceLink = `${repoUrl}/datasource/schema.json.hbs`
-
-async function getSchema(link: string) {
-  const response = await fetch(link)
-  if (response.status > 300) {
-    return
-  }
-  const text = await response.text()
-  return JSON.parse(text)
-}
-
-async function runTest(opts: { link?: string; schema?: any }) {
-  let error
-  try {
-    let schema = opts.schema
-    if (opts.link) {
-      schema = await getSchema(opts.link)
-    }
-    validate(schema)
-  } catch (err) {
-    error = err
-  }
-  return error
-}
-
-describe("it should be able to validate an automation schema", () => {
-  it("should return automation skeleton schema is valid", async () => {
-    const error = await runTest({ link: automationLink })
-    expect(error).toBeUndefined()
-  })
-
-  it("should fail given invalid automation schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.AUTOMATION,
-        schema: {},
-      },
-    })
-    expect(error).toBeDefined()
-  })
-})
-
-describe("it should be able to validate a component schema", () => {
-  it("should return component skeleton schema is valid", async () => {
-    const error = await runTest({ link: componentLink })
-    expect(error).toBeUndefined()
-  })
-
-  it("should fail given invalid component schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.COMPONENT,
-        schema: {},
-      },
-    })
-    expect(error).toBeDefined()
-  })
-})
-
-describe("it should be able to validate a datasource schema", () => {
-  it("should return datasource skeleton schema is valid", async () => {
-    const error = await runTest({ link: datasourceLink })
-    expect(error).toBeUndefined()
-  })
-
-  it("should fail given invalid datasource schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.DATASOURCE,
-        schema: {},
-      },
-    })
-    expect(error).toBeDefined()
-  })
-})

package/src/plugin/utils.ts

@@ -1,156 +0,0 @@
-import {
-  DatasourceFieldType,
-  QueryType,
-  PluginType,
-  AutomationStepType,
-  AutomationStepIdArray,
-  AutomationIOType,
-  AutomationCustomIOType,
-} from "@budibase/types"
-import joi from "joi"
-
-const DATASOURCE_TYPES = [
-  "Relational",
-  "Non-relational",
-  "Spreadsheet",
-  "Object store",
-  "Graph",
-  "API",
-]
-
-function runJoi(validator: joi.Schema, schema: any) {
-  const { error } = validator.validate(schema)
-  if (error) {
-    throw error
-  }
-}
-
-function validateComponent(schema: any) {
-  const validator = joi.object({
-    type: joi.string().allow(PluginType.COMPONENT).required(),
-    metadata: joi.object().unknown(true).required(),
-    hash: joi.string().optional(),
-    version: joi.string().optional(),
-    schema: joi
-      .object({
-        name: joi.string().required(),
-        settings: joi.array().items(joi.object().unknown(true)).required(),
-      })
-      .unknown(true),
-  })
-  runJoi(validator, schema)
-}
-
-function validateDatasource(schema: any) {
-  const fieldValidator = joi.object({
-    type: joi
-      .string()
-      .allow(...Object.values(DatasourceFieldType))
-      .required(),
-    required: joi.boolean().required(),
-    default: joi.any(),
-    display: joi.string(),
-  })
-
-  const queryValidator = joi
-    .object({
-      type: joi.string().allow(...Object.values(QueryType)),
-      readable: joi.boolean(),
-      fields: joi.object().pattern(joi.string(), fieldValidator),
-    })
-    .required()
-
-  const validator = joi.object({
-    type: joi.string().allow(PluginType.DATASOURCE).required(),
-    metadata: joi.object().unknown(true).required(),
-    hash: joi.string().optional(),
-    version: joi.string().optional(),
-    schema: joi.object({
-      docs: joi.string(),
-      friendlyName: joi.string().required(),
-      type: joi.string().allow(...DATASOURCE_TYPES),
-      description: joi.string().required(),
-      datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
-      query: joi
-        .object()
-        .pattern(joi.string(), queryValidator)
-        .unknown(true)
-        .required(),
-      extra: joi.object().pattern(
-        joi.string(),
-        joi.object({
-          type: joi.string().required(),
-          displayName: joi.string().required(),
-          required: joi.boolean(),
-          data: joi.object(),
-        })
-      ),
-    }),
-  })
-  runJoi(validator, schema)
-}
-
-function validateAutomation(schema: any) {
-  const basePropsValidator = joi.object().pattern(joi.string(), {
-    type: joi
-      .string()
-      .allow(...Object.values(AutomationIOType))
-      .required(),
-    customType: joi.string().allow(...Object.values(AutomationCustomIOType)),
-    title: joi.string(),
-    description: joi.string(),
-    enum: joi.array().items(joi.string()),
-    pretty: joi.array().items(joi.string()),
-  })
-  const stepSchemaValidator = joi
-    .object({
-      properties: basePropsValidator,
-      required: joi.array().items(joi.string()),
-    })
-    .concat(basePropsValidator)
-    .required()
-  const validator = joi.object({
-    type: joi.string().allow(PluginType.AUTOMATION).required(),
-    metadata: joi.object().unknown(true).required(),
-    hash: joi.string().optional(),
-    version: joi.string().optional(),
-    schema: joi.object({
-      name: joi.string().required(),
-      tagline: joi.string().required(),
-      icon: joi.string().required(),
-      description: joi.string().required(),
-      type: joi
-        .string()
-        .allow(AutomationStepType.ACTION, AutomationStepType.LOGIC)
-        .required(),
-      stepId: joi
-        .string()
-        .disallow(...AutomationStepIdArray)
-        .required(),
-      inputs: joi.object().optional(),
-      schema: joi
-        .object({
-          inputs: stepSchemaValidator,
-          outputs: stepSchemaValidator,
-        })
-        .required(),
-    }),
-  })
-  runJoi(validator, schema)
-}
-
-export function validate(schema: any) {
-  switch (schema?.type) {
-    case PluginType.COMPONENT:
-      validateComponent(schema)
-      break
-    case PluginType.DATASOURCE:
-      validateDatasource(schema)
-      break
-    case PluginType.AUTOMATION:
-      validateAutomation(schema)
-      break
-    default:
-      throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`)
-  }
-}

package/src/queue/constants.ts

@@ -1,6 +0,0 @@
-export enum JobQueue {
-  AUTOMATION = "automationQueue",
-  APP_BACKUP = "appBackupQueue",
-  AUDIT_LOG = "auditLogQueue",
-  SYSTEM_EVENT_QUEUE = "systemEventQueue",
-}

package/src/queue/inMemoryQueue.ts

@@ -1,141 +0,0 @@
-import events from "events"
-import { timeout } from "../utils"
-
-/**
- * Bull works with a Job wrapper around all messages that contains a lot more information about
- * the state of the message, this object constructor implements the same schema of Bull jobs
- * for the sake of maintaining API consistency.
- * @param {string} queue The name of the queue which the message will be carried on.
- * @param {object} message The JSON message which will be passed back to the consumer.
- * @returns {Object} A new job which can now be put onto the queue, this is mostly an
- * internal structure so that an in memory queue can be easily swapped for a Bull queue.
- */
-function newJob(queue: string, message: any) {
-  return {
-    timestamp: Date.now(),
-    queue: queue,
-    data: message,
-  }
-}
-
-/**
- * This is designed to replicate Bull (https://github.com/OptimalBits/bull) in memory as a sort of mock.
- * It is relatively simple, using an event emitter internally to register when messages are available
- * to the consumers - in can support many inputs and many consumers.
- */
-class InMemoryQueue {
-  _name: string
-  _opts?: any
-  _messages: any[]
-  _emitter: EventEmitter
-  _runCount: number
-  _addCount: number
-  /**
-   * The constructor the queue, exactly the same as that of Bulls.
-   * @param {string} name The name of the queue which is being configured.
-   * @param {object|null} opts This is not used by the in memory queue as there is no real use
-   * case when in memory, but is the same API as Bull
-   */
-  constructor(name: string, opts = null) {
-    this._name = name
-    this._opts = opts
-    this._messages = []
-    this._emitter = new events.EventEmitter()
-    this._runCount = 0
-    this._addCount = 0
-  }
-
-  /**
-   * Same callback API as Bull, each callback passed to this will consume messages as they are
-   * available. Please note this is a queue service, not a notification service, so each
-   * consumer will receive different messages.
-   * @param {function<object>} func The callback function which will return a "Job", the same
-   * as the Bull API, within this job the property "data" contains the JSON message. Please
-   * note this is incredibly limited compared to Bull as in reality the Job would contain
-   * a lot more information about the queue and current status of Bull cluster.
-   */
-  process(func: any) {
-    this._emitter.on("message", async () => {
-      if (this._messages.length <= 0) {
-        return
-      }
-      let msg = this._messages.shift()
-      let resp = func(msg)
-      if (resp.then != null) {
-        await resp
-      }
-      this._runCount++
-    })
-  }
-
-  // simply puts a message to the queue and emits to the queue for processing
-  /**
-   * Simple function to replicate the add message functionality of Bull, putting
-   * a new message on the queue. This then emits an event which will be used to
-   * return the message to a consumer (if one is attached).
-   * @param {object} msg A message to be transported over the queue, this should be
-   * a JSON message as this is required by Bull.
-   * @param {boolean} repeat serves no purpose for the import queue.
-   */
-  // eslint-disable-next-line no-unused-vars
-  add(msg: any, repeat: boolean) {
-    if (typeof msg !== "object") {
-      throw "Queue only supports carrying JSON."
-    }
-    this._messages.push(newJob(this._name, msg))
-    this._addCount++
-    this._emitter.emit("message")
-  }
-
-  /**
-   * replicating the close function from bull, which waits for jobs to finish.
-   */
-  async close() {
-    return []
-  }
-
-  /**
-   * This removes a cron which has been implemented, this is part of Bull API.
-   * @param {string} cronJobId The cron which is to be removed.
-   */
-  removeRepeatableByKey(cronJobId: string) {
-    // TODO: implement for testing
-    console.log(cronJobId)
-  }
-
-  /**
-   * Implemented for tests
-   */
-  getRepeatableJobs() {
-    return []
-  }
-
-  // eslint-disable-next-line no-unused-vars
-  removeJobs(pattern: string) {
-    // no-op
-  }
-
-  /**
-   * Implemented for tests
-   */
-  async clean() {
-    return []
-  }
-
-  async getJob() {
-    return {}
-  }
-
-  on() {
-    // do nothing
-    return this
-  }
-
-  async waitForCompletion() {
-    do {
-      await timeout(50)
-    } while (this._addCount < this._runCount)
-  }
-}
-
-export default InMemoryQueue

package/src/queue/index.ts

@@ -1,2 +0,0 @@
-export * from "./queue"
-export * from "./constants"

package/src/queue/listeners.ts

@@ -1,195 +0,0 @@
-import { Job, JobId, Queue } from "bull"
-import { JobQueue } from "./constants"
-import * as context from "../context"
-
-export type StalledFn = (job: Job) => Promise<void>
-
-export function addListeners(
-  queue: Queue,
-  jobQueue: JobQueue,
-  removeStalledCb?: StalledFn
-) {
-  logging(queue, jobQueue)
-  if (removeStalledCb) {
-    handleStalled(queue, removeStalledCb)
-  }
-}
-
-function handleStalled(queue: Queue, removeStalledCb?: StalledFn) {
-  queue.on("stalled", async (job: Job) => {
-    if (removeStalledCb) {
-      await removeStalledCb(job)
-    } else if (job.opts.repeat) {
-      const jobId = job.id
-      const repeatJobs = await queue.getRepeatableJobs()
-      for (let repeatJob of repeatJobs) {
-        if (repeatJob.id === jobId) {
-          await queue.removeRepeatableByKey(repeatJob.key)
-        }
-      }
-      console.log(`jobId=${jobId} disabled`)
-    }
-  })
-}
-
-function getLogParams(
-  eventType: QueueEventType,
-  event: BullEvent,
-  opts: {
-    job?: Job
-    jobId?: JobId
-    error?: Error
-  } = {},
-  extra: any = {}
-) {
-  const message = `[BULL] ${eventType}=${event}`
-  const err = opts.error
-
-  const bullLog = {
-    _logKey: "bull",
-    eventType,
-    event,
-    job: opts.job,
-    jobId: opts.jobId || opts.job?.id,
-    ...extra,
-  }
-
-  let automationLog
-  if (opts.job?.data?.automation) {
-    automationLog = {
-      _logKey: "automation",
-      trigger: opts.job
-        ? opts.job.data.automation.definition.trigger.event
-        : undefined,
-    }
-  }
-
-  return [message, err, bullLog, automationLog]
-}
-
-enum BullEvent {
-  ERROR = "error",
-  WAITING = "waiting",
-  ACTIVE = "active",
-  STALLED = "stalled",
-  PROGRESS = "progress",
-  COMPLETED = "completed",
-  FAILED = "failed",
-  PAUSED = "paused",
-  RESUMED = "resumed",
-  CLEANED = "cleaned",
-  DRAINED = "drained",
-  REMOVED = "removed",
-}
-
-enum QueueEventType {
-  AUTOMATION_EVENT = "automation-event",
-  APP_BACKUP_EVENT = "app-backup-event",
-  AUDIT_LOG_EVENT = "audit-log-event",
-  SYSTEM_EVENT = "system-event",
-}
-
-const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
-  [JobQueue.AUTOMATION]: QueueEventType.AUTOMATION_EVENT,
-  [JobQueue.APP_BACKUP]: QueueEventType.APP_BACKUP_EVENT,
-  [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT,
-  [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT,
-}
-
-function logging(queue: Queue, jobQueue: JobQueue) {
-  const eventType = EventTypeMap[jobQueue]
-
-  function doInJobContext(job: Job, task: any) {
-    // if this is an automation job try to get the app id
-    const appId = job.data.event?.appId
-    if (appId) {
-      return context.doInContext(appId, task)
-    } else {
-      task()
-    }
-  }
-
-  queue
-    .on(BullEvent.STALLED, async (job: Job) => {
-      // A job has been marked as stalled. This is useful for debugging job
-      // workers that crash or pause the event loop.
-      await doInJobContext(job, () => {
-        console.error(...getLogParams(eventType, BullEvent.STALLED, { job }))
-      })
-    })
-    .on(BullEvent.ERROR, (error: any) => {
-      // An error occurred.
-      console.error(...getLogParams(eventType, BullEvent.ERROR, { error }))
-    })
-
-  if (process.env.NODE_DEBUG?.includes("bull")) {
-    queue
-      .on(BullEvent.WAITING, (jobId: JobId) => {
-        // A Job is waiting to be processed as soon as a worker is idling.
-        console.info(...getLogParams(eventType, BullEvent.WAITING, { jobId }))
-      })
-      .on(BullEvent.ACTIVE, async (job: Job, jobPromise: any) => {
-        // A job has started. You can use `jobPromise.cancel()`` to abort it.
-        await doInJobContext(job, () => {
-          console.info(...getLogParams(eventType, BullEvent.ACTIVE, { job }))
-        })
-      })
-      .on(BullEvent.PROGRESS, async (job: Job, progress: any) => {
-        // A job's progress was updated
-        await doInJobContext(job, () => {
-          console.info(
-            ...getLogParams(
-              eventType,
-              BullEvent.PROGRESS,
-              { job },
-              { progress }
-            )
-          )
-        })
-      })
-      .on(BullEvent.COMPLETED, async (job: Job, result) => {
-        // A job successfully completed with a `result`.
-        await doInJobContext(job, () => {
-          console.info(
-            ...getLogParams(eventType, BullEvent.COMPLETED, { job }, { result })
-          )
-        })
-      })
-      .on(BullEvent.FAILED, async (job: Job, error: any) => {
-        // A job failed with reason `err`!
-        await doInJobContext(job, () => {
-          console.error(
-            ...getLogParams(eventType, BullEvent.FAILED, { job, error })
-          )
-        })
-      })
-      .on(BullEvent.PAUSED, () => {
-        // The queue has been paused.
-        console.info(...getLogParams(eventType, BullEvent.PAUSED))
-      })
-      .on(BullEvent.RESUMED, () => {
-        // The queue has been resumed.
-        console.info(...getLogParams(eventType, BullEvent.RESUMED))
-      })
-      .on(BullEvent.CLEANED, (jobs: Job[], type: string) => {
-        // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned
-        // jobs, and `type` is the type of jobs cleaned.
-        console.info(
-          ...getLogParams(
-            eventType,
-            BullEvent.CLEANED,
-            {},
-            { length: jobs.length, type }
-          )
-        )
-      })
-      .on(BullEvent.DRAINED, () => {
-        // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed)
-        console.info(...getLogParams(eventType, BullEvent.DRAINED))
-      })
-      .on(BullEvent.REMOVED, (job: Job) => {
-        // A job successfully removed.
-        console.info(...getLogParams(eventType, BullEvent.REMOVED, { job }))
-      })
-  }
-}

package/src/queue/queue.ts

@@ -1,54 +0,0 @@
-import env from "../environment"
-import { getRedisOptions } from "../redis/utils"
-import { JobQueue } from "./constants"
-import InMemoryQueue from "./inMemoryQueue"
-import BullQueue from "bull"
-import { addListeners, StalledFn } from "./listeners"
-import * as timers from "../timers"
-
-const CLEANUP_PERIOD_MS = 60 * 1000
-let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
-let cleanupInterval: NodeJS.Timeout
-
-async function cleanup() {
-  for (let queue of QUEUES) {
-    await queue.clean(CLEANUP_PERIOD_MS, "completed")
-  }
-}
-
-export function createQueue<T>(
-  jobQueue: JobQueue,
-  opts: { removeStalledCb?: StalledFn } = {}
-): BullQueue.Queue<T> {
-  const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
-  const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
-  let queue: any
-  if (!env.isTest()) {
-    queue = new BullQueue(jobQueue, queueConfig)
-  } else {
-    queue = new InMemoryQueue(jobQueue, queueConfig)
-  }
-  addListeners(queue, jobQueue, opts?.removeStalledCb)
-  QUEUES.push(queue)
-  if (!cleanupInterval && !env.isTest()) {
-    cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS)
-    // fire off an initial cleanup
-    cleanup().catch(err => {
-      console.error(`Unable to cleanup automation queue initially - ${err}`)
-    })
-  }
-  return queue
-}
-
-export async function shutdown() {
-  if (cleanupInterval) {
-    timers.clear(cleanupInterval)
-  }
-  if (QUEUES.length) {
-    for (let queue of QUEUES) {
-      await queue.close()
-    }
-    QUEUES = []
-  }
-  console.log("Queues shutdown")
-}

package/src/redis/index.ts

@@ -1,6 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-export { default as Client } from "./redis"
-export * as utils from "./utils"
-export * as clients from "./init"
-export * as locks from "./redlockImpl"