@budibase/backend-core 3.2.4 → 3.2.6

package/src/queue/inMemoryQueue.ts

@@ -1,189 +0,0 @@
-import events from "events"
-import { newid } from "../utils"
-import { Queue, QueueOptions, JobOptions } from "./queue"
-
-interface JobMessage {
-  id: string
-  timestamp: number
-  queue: string
-  data: any
-  opts?: JobOptions
-}
-
-/**
- * Bull works with a Job wrapper around all messages that contains a lot more information about
- * the state of the message, this object constructor implements the same schema of Bull jobs
- * for the sake of maintaining API consistency.
- * @param queue The name of the queue which the message will be carried on.
- * @param message The JSON message which will be passed back to the consumer.
- * @returns A new job which can now be put onto the queue, this is mostly an
- * internal structure so that an in memory queue can be easily swapped for a Bull queue.
- */
-function newJob(queue: string, message: any, opts?: JobOptions): JobMessage {
-  return {
-    id: newid(),
-    timestamp: Date.now(),
-    queue: queue,
-    data: message,
-    opts,
-  }
-}
-
-/**
- * This is designed to replicate Bull (https://github.com/OptimalBits/bull) in memory as a sort of mock.
- * It is relatively simple, using an event emitter internally to register when messages are available
- * to the consumers - in can support many inputs and many consumers.
- */
-class InMemoryQueue implements Partial<Queue> {
-  _name: string
-  _opts?: QueueOptions
-  _messages: JobMessage[]
-  _queuedJobIds: Set<string>
-  _emitter: NodeJS.EventEmitter
-  _runCount: number
-  _addCount: number
-
-  /**
-   * The constructor the queue, exactly the same as that of Bulls.
-   * @param name The name of the queue which is being configured.
-   * @param opts This is not used by the in memory queue as there is no real use
-   * case when in memory, but is the same API as Bull
-   */
-  constructor(name: string, opts?: QueueOptions) {
-    this._name = name
-    this._opts = opts
-    this._messages = []
-    this._emitter = new events.EventEmitter()
-    this._runCount = 0
-    this._addCount = 0
-    this._queuedJobIds = new Set<string>()
-  }
-
-  /**
-   * Same callback API as Bull, each callback passed to this will consume messages as they are
-   * available. Please note this is a queue service, not a notification service, so each
-   * consumer will receive different messages.
-   * as the Bull API, within this job the property "data" contains the JSON message. Please
-   * note this is incredibly limited compared to Bull as in reality the Job would contain
-   * a lot more information about the queue and current status of Bull cluster.
-   */
-  async process(concurrencyOrFunc: number | any, func?: any) {
-    func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
-    this._emitter.on("message", async () => {
-      if (this._messages.length <= 0) {
-        return
-      }
-      let msg = this._messages.shift()
-
-      let resp = func(msg)
-
-      async function retryFunc(fnc: any) {
-        try {
-          await fnc
-        } catch (e: any) {
-          await new Promise<void>(r => setTimeout(() => r(), 50))
-
-          await retryFunc(func(msg))
-        }
-      }
-
-      if (resp.then != null) {
-        try {
-          await retryFunc(resp)
-        } catch (e: any) {
-          console.error(e)
-        }
-      }
-      this._runCount++
-      const jobId = msg?.opts?.jobId?.toString()
-      if (jobId && msg?.opts?.removeOnComplete) {
-        this._queuedJobIds.delete(jobId)
-      }
-    })
-  }
-
-  async isReady() {
-    return this as any
-  }
-
-  // simply puts a message to the queue and emits to the queue for processing
-  /**
-   * Simple function to replicate the add message functionality of Bull, putting
-   * a new message on the queue. This then emits an event which will be used to
-   * return the message to a consumer (if one is attached).
-   * @param msg A message to be transported over the queue, this should be
-   * a JSON message as this is required by Bull.
-   * @param repeat serves no purpose for the import queue.
-   */
-  async add(data: any, opts?: JobOptions) {
-    const jobId = opts?.jobId?.toString()
-    if (jobId && this._queuedJobIds.has(jobId)) {
-      console.log(`Ignoring already queued job ${jobId}`)
-      return
-    }
-
-    if (typeof data !== "object") {
-      throw "Queue only supports carrying JSON."
-    }
-    if (jobId) {
-      this._queuedJobIds.add(jobId)
-    }
-
-    const pushMessage = () => {
-      this._messages.push(newJob(this._name, data, opts))
-      this._addCount++
-      this._emitter.emit("message")
-    }
-
-    const delay = opts?.delay
-    if (delay) {
-      setTimeout(pushMessage, delay)
-    } else {
-      pushMessage()
-    }
-    return { id: jobId } as any
-  }
-
-  /**
-   * replicating the close function from bull, which waits for jobs to finish.
-   */
-  async close() {}
-
-  /**
-   * This removes a cron which has been implemented, this is part of Bull API.
-   * @param cronJobId The cron which is to be removed.
-   */
-  async removeRepeatableByKey(cronJobId: string) {
-    // TODO: implement for testing
-    console.log(cronJobId)
-  }
-
-  /**
-   * Implemented for tests
-   */
-  async getRepeatableJobs() {
-    return []
-  }
-
-  async removeJobs(_pattern: string) {
-    // no-op
-  }
-
-  /**
-   * Implemented for tests
-   */
-  async clean() {
-    return []
-  }
-
-  async getJob() {
-    return null
-  }
-
-  on() {
-    // do nothing
-    return this as any
-  }
-}
-
-export default InMemoryQueue

package/src/queue/index.ts

@@ -1,2 +0,0 @@
-export * from "./queue"
-export * from "./constants"

package/src/queue/listeners.ts

@@ -1,199 +0,0 @@
-import { Job, JobId, Queue } from "bull"
-import { JobQueue } from "./constants"
-import * as context from "../context"
-
-export type StalledFn = (job: Job) => Promise<void>
-
-export function addListeners(
-  queue: Queue,
-  jobQueue: JobQueue,
-  removeStalledCb?: StalledFn
-) {
-  logging(queue, jobQueue)
-  if (removeStalledCb) {
-    handleStalled(queue, removeStalledCb)
-  }
-}
-
-function handleStalled(queue: Queue, removeStalledCb?: StalledFn) {
-  queue.on("stalled", async (job: Job) => {
-    if (removeStalledCb) {
-      await removeStalledCb(job)
-    } else if (job.opts.repeat) {
-      const jobId = job.id
-      const repeatJobs = await queue.getRepeatableJobs()
-      for (let repeatJob of repeatJobs) {
-        if (repeatJob.id === jobId) {
-          await queue.removeRepeatableByKey(repeatJob.key)
-        }
-      }
-      console.log(`jobId=${jobId} disabled`)
-    }
-  })
-}
-
-function getLogParams(
-  eventType: QueueEventType,
-  event: BullEvent,
-  opts: {
-    job?: Job
-    jobId?: JobId
-    error?: Error
-  } = {},
-  extra: any = {}
-) {
-  const message = `[BULL] ${eventType}=${event}`
-  const err = opts.error
-
-  const bullLog = {
-    _logKey: "bull",
-    eventType,
-    event,
-    job: opts.job,
-    jobId: opts.jobId || opts.job?.id,
-    ...extra,
-  }
-
-  let automationLog
-  if (opts.job?.data?.automation) {
-    automationLog = {
-      _logKey: "automation",
-      trigger: opts.job
-        ? opts.job.data.automation.definition.trigger.event
-        : undefined,
-    }
-  }
-
-  return [message, err, bullLog, automationLog]
-}
-
-enum BullEvent {
-  ERROR = "error",
-  WAITING = "waiting",
-  ACTIVE = "active",
-  STALLED = "stalled",
-  PROGRESS = "progress",
-  COMPLETED = "completed",
-  FAILED = "failed",
-  PAUSED = "paused",
-  RESUMED = "resumed",
-  CLEANED = "cleaned",
-  DRAINED = "drained",
-  REMOVED = "removed",
-}
-
-enum QueueEventType {
-  AUTOMATION_EVENT = "automation-event",
-  APP_BACKUP_EVENT = "app-backup-event",
-  AUDIT_LOG_EVENT = "audit-log-event",
-  SYSTEM_EVENT = "system-event",
-  APP_MIGRATION = "app-migration",
-  DOC_WRITETHROUGH = "doc-writethrough",
-}
-
-const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
-  [JobQueue.AUTOMATION]: QueueEventType.AUTOMATION_EVENT,
-  [JobQueue.APP_BACKUP]: QueueEventType.APP_BACKUP_EVENT,
-  [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT,
-  [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT,
-  [JobQueue.APP_MIGRATION]: QueueEventType.APP_MIGRATION,
-  [JobQueue.DOC_WRITETHROUGH_QUEUE]: QueueEventType.DOC_WRITETHROUGH,
-}
-
-function logging(queue: Queue, jobQueue: JobQueue) {
-  const eventType = EventTypeMap[jobQueue]
-
-  function doInJobContext(job: Job, task: any) {
-    // if this is an automation job try to get the app id
-    const appId = job.data.event?.appId
-    if (appId) {
-      return context.doInContext(appId, task)
-    } else {
-      task()
-    }
-  }
-
-  queue
-    .on(BullEvent.STALLED, async (job: Job) => {
-      // A job has been marked as stalled. This is useful for debugging job
-      // workers that crash or pause the event loop.
-      await doInJobContext(job, () => {
-        console.error(...getLogParams(eventType, BullEvent.STALLED, { job }))
-      })
-    })
-    .on(BullEvent.ERROR, (error: any) => {
-      // An error occurred.
-      console.error(...getLogParams(eventType, BullEvent.ERROR, { error }))
-    })
-
-  if (process.env.NODE_DEBUG?.includes("bull")) {
-    queue
-      .on(BullEvent.WAITING, (jobId: JobId) => {
-        // A Job is waiting to be processed as soon as a worker is idling.
-        console.info(...getLogParams(eventType, BullEvent.WAITING, { jobId }))
-      })
-      .on(BullEvent.ACTIVE, async (job: Job) => {
-        // A job has started. You can use `jobPromise.cancel()`` to abort it.
-        await doInJobContext(job, () => {
-          console.info(...getLogParams(eventType, BullEvent.ACTIVE, { job }))
-        })
-      })
-      .on(BullEvent.PROGRESS, async (job: Job, progress: any) => {
-        // A job's progress was updated
-        await doInJobContext(job, () => {
-          console.info(
-            ...getLogParams(
-              eventType,
-              BullEvent.PROGRESS,
-              { job },
-              { progress }
-            )
-          )
-        })
-      })
-      .on(BullEvent.COMPLETED, async (job: Job, result) => {
-        // A job successfully completed with a `result`.
-        await doInJobContext(job, () => {
-          console.info(
-            ...getLogParams(eventType, BullEvent.COMPLETED, { job }, { result })
-          )
-        })
-      })
-      .on(BullEvent.FAILED, async (job: Job, error: any) => {
-        // A job failed with reason `err`!
-        await doInJobContext(job, () => {
-          console.error(
-            ...getLogParams(eventType, BullEvent.FAILED, { job, error })
-          )
-        })
-      })
-      .on(BullEvent.PAUSED, () => {
-        // The queue has been paused.
-        console.info(...getLogParams(eventType, BullEvent.PAUSED))
-      })
-      .on(BullEvent.RESUMED, () => {
-        // The queue has been resumed.
-        console.info(...getLogParams(eventType, BullEvent.RESUMED))
-      })
-      .on(BullEvent.CLEANED, (jobs: Job[], type: string) => {
-        // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned
-        // jobs, and `type` is the type of jobs cleaned.
-        console.info(
-          ...getLogParams(
-            eventType,
-            BullEvent.CLEANED,
-            {},
-            { length: jobs.length, type }
-          )
-        )
-      })
-      .on(BullEvent.DRAINED, () => {
-        // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed)
-        console.info(...getLogParams(eventType, BullEvent.DRAINED))
-      })
-      .on(BullEvent.REMOVED, (job: Job) => {
-        // A job successfully removed.
-        console.info(...getLogParams(eventType, BullEvent.REMOVED, { job }))
-      })
-  }
-}

package/src/queue/queue.ts

@@ -1,84 +0,0 @@
-import env from "../environment"
-import { getRedisOptions } from "../redis/utils"
-import { JobQueue } from "./constants"
-import InMemoryQueue from "./inMemoryQueue"
-import BullQueue, { QueueOptions, JobOptions } from "bull"
-import { addListeners, StalledFn } from "./listeners"
-import { Duration } from "../utils"
-import * as timers from "../timers"
-
-export { QueueOptions, Queue, JobOptions } from "bull"
-
-// the queue lock is held for 5 minutes
-const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
-// queue lock is refreshed every 30 seconds
-const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
-// cleanup the queue every 60 seconds
-const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
-let QUEUES: BullQueue.Queue[] = []
-let cleanupInterval: NodeJS.Timeout
-
-async function cleanup() {
-  for (let queue of QUEUES) {
-    await queue.clean(CLEANUP_PERIOD_MS, "completed")
-    await queue.clean(CLEANUP_PERIOD_MS, "failed")
-  }
-}
-
-export function createQueue<T>(
-  jobQueue: JobQueue,
-  opts: {
-    removeStalledCb?: StalledFn
-    maxStalledCount?: number
-    jobOptions?: JobOptions
-  } = {}
-): BullQueue.Queue<T> {
-  const redisOpts = getRedisOptions()
-  const queueConfig: QueueOptions = {
-    redis: redisOpts,
-    settings: {
-      maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0,
-      lockDuration: QUEUE_LOCK_MS,
-      lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
-    },
-  }
-  if (opts.jobOptions) {
-    queueConfig.defaultJobOptions = opts.jobOptions
-  }
-  let queue: BullQueue.Queue<T>
-  if (!env.isTest()) {
-    queue = new BullQueue(jobQueue, queueConfig)
-  } else if (
-    process.env.BULL_TEST_REDIS_PORT &&
-    !isNaN(+process.env.BULL_TEST_REDIS_PORT)
-  ) {
-    queue = new BullQueue(jobQueue, {
-      redis: { host: "localhost", port: +process.env.BULL_TEST_REDIS_PORT },
-    })
-  } else {
-    queue = new InMemoryQueue(jobQueue, queueConfig) as any
-  }
-  addListeners(queue, jobQueue, opts?.removeStalledCb)
-  QUEUES.push(queue)
-  if (!cleanupInterval && !env.isTest()) {
-    cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS)
-    // fire off an initial cleanup
-    cleanup().catch(err => {
-      console.error(`Unable to cleanup ${jobQueue} initially - ${err}`)
-    })
-  }
-  return queue
-}
-
-export async function shutdown() {
-  if (cleanupInterval) {
-    timers.clear(cleanupInterval)
-  }
-  if (QUEUES.length) {
-    for (let queue of QUEUES) {
-      await queue.close()
-    }
-    QUEUES = []
-  }
-  console.log("Queues shutdown")
-}

package/src/redis/index.ts

@@ -1,6 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-export { default as Client } from "./redis"
-export * as utils from "./utils"
-export * as clients from "./init"
-export * as locks from "./redlockImpl"

package/src/redis/init.ts

@@ -1,118 +0,0 @@
-import Client from "./redis"
-import * as utils from "./utils"
-
-let userClient: Client,
-  sessionClient: Client,
-  appClient: Client,
-  cacheClient: Client,
-  writethroughClient: Client,
-  lockClient: Client,
-  socketClient: Client,
-  inviteClient: Client,
-  passwordResetClient: Client,
-  docWritethroughClient: Client
-
-export async function init() {
-  userClient = await new Client(utils.Databases.USER_CACHE).init()
-  sessionClient = await new Client(utils.Databases.SESSIONS).init()
-  appClient = await new Client(utils.Databases.APP_METADATA).init()
-  cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
-  lockClient = await new Client(utils.Databases.LOCKS).init()
-  writethroughClient = await new Client(utils.Databases.WRITE_THROUGH).init()
-  inviteClient = await new Client(utils.Databases.INVITATIONS).init()
-  passwordResetClient = await new Client(utils.Databases.PW_RESETS).init()
-  socketClient = await new Client(
-    utils.Databases.SOCKET_IO,
-    utils.SelectableDatabase.SOCKET_IO
-  ).init()
-  docWritethroughClient = await new Client(
-    utils.Databases.DOC_WRITE_THROUGH
-  ).init()
-}
-
-export async function shutdown() {
-  if (userClient) await userClient.finish()
-  if (sessionClient) await sessionClient.finish()
-  if (appClient) await appClient.finish()
-  if (cacheClient) await cacheClient.finish()
-  if (writethroughClient) await writethroughClient.finish()
-  if (lockClient) await lockClient.finish()
-  if (inviteClient) await inviteClient.finish()
-  if (passwordResetClient) await passwordResetClient.finish()
-  if (socketClient) await socketClient.finish()
-  if (docWritethroughClient) await docWritethroughClient.finish()
-}
-
-process.on("exit", async () => {
-  await shutdown()
-})
-
-export async function getUserClient() {
-  if (!userClient) {
-    await init()
-  }
-  return userClient
-}
-
-export async function getSessionClient() {
-  if (!sessionClient) {
-    await init()
-  }
-  return sessionClient
-}
-
-export async function getAppClient() {
-  if (!appClient) {
-    await init()
-  }
-  return appClient
-}
-
-export async function getCacheClient() {
-  if (!cacheClient) {
-    await init()
-  }
-  return cacheClient
-}
-
-export async function getWritethroughClient() {
-  if (!writethroughClient) {
-    await init()
-  }
-  return writethroughClient
-}
-
-export async function getLockClient() {
-  if (!lockClient) {
-    await init()
-  }
-  return lockClient
-}
-
-export async function getSocketClient() {
-  if (!socketClient) {
-    await init()
-  }
-  return socketClient
-}
-
-export async function getInviteClient() {
-  if (!inviteClient) {
-    await init()
-  }
-  return inviteClient
-}
-
-export async function getPasswordResetClient() {
-  if (!passwordResetClient) {
-    await init()
-  }
-  return passwordResetClient
-}
-
-export async function getDocWritethroughClient() {
-  if (!writethroughClient) {
-    await init()
-  }
-  return writethroughClient
-}