@budibase/backend-core 2.9.19-alpha.0 → 2.9.20

package/src/redis/init.ts

@@ -1,86 +0,0 @@
-import Client from "./redis"
-import * as utils from "./utils"
-
-let userClient: Client,
-  sessionClient: Client,
-  appClient: Client,
-  cacheClient: Client,
-  writethroughClient: Client,
-  lockClient: Client,
-  socketClient: Client
-
-async function init() {
-  userClient = await new Client(utils.Databases.USER_CACHE).init()
-  sessionClient = await new Client(utils.Databases.SESSIONS).init()
-  appClient = await new Client(utils.Databases.APP_METADATA).init()
-  cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
-  lockClient = await new Client(utils.Databases.LOCKS).init()
-  writethroughClient = await new Client(utils.Databases.WRITE_THROUGH).init()
-  socketClient = await new Client(
-    utils.Databases.SOCKET_IO,
-    utils.SelectableDatabase.SOCKET_IO
-  ).init()
-}
-
-export async function shutdown() {
-  if (userClient) await userClient.finish()
-  if (sessionClient) await sessionClient.finish()
-  if (appClient) await appClient.finish()
-  if (cacheClient) await cacheClient.finish()
-  if (writethroughClient) await writethroughClient.finish()
-  if (lockClient) await lockClient.finish()
-  if (socketClient) await socketClient.finish()
-}
-
-process.on("exit", async () => {
-  await shutdown()
-})
-
-export async function getUserClient() {
-  if (!userClient) {
-    await init()
-  }
-  return userClient
-}
-
-export async function getSessionClient() {
-  if (!sessionClient) {
-    await init()
-  }
-  return sessionClient
-}
-
-export async function getAppClient() {
-  if (!appClient) {
-    await init()
-  }
-  return appClient
-}
-
-export async function getCacheClient() {
-  if (!cacheClient) {
-    await init()
-  }
-  return cacheClient
-}
-
-export async function getWritethroughClient() {
-  if (!writethroughClient) {
-    await init()
-  }
-  return writethroughClient
-}
-
-export async function getLockClient() {
-  if (!lockClient) {
-    await init()
-  }
-  return lockClient
-}
-
-export async function getSocketClient() {
-  if (!socketClient) {
-    await init()
-  }
-  return socketClient
-}

package/src/redis/redis.ts

@@ -1,308 +0,0 @@
-import env from "../environment"
-import Redis from "ioredis"
-// mock-redis doesn't have any typing
-let MockRedis: any | undefined
-if (env.MOCK_REDIS) {
-  try {
-    // ioredis mock is all in memory
-    MockRedis = require("ioredis-mock")
-  } catch (err) {
-    console.log("Mock redis unavailable")
-  }
-}
-import {
-  addDbPrefix,
-  removeDbPrefix,
-  getRedisOptions,
-  SEPARATOR,
-  SelectableDatabase,
-} from "./utils"
-import * as timers from "../timers"
-
-const RETRY_PERIOD_MS = 2000
-const STARTUP_TIMEOUT_MS = 5000
-const CLUSTERED = env.REDIS_CLUSTERED
-const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
-
-// for testing just generate the client once
-let CLOSED = false
-let CLIENTS: { [key: number]: any } = {}
-0
-let CONNECTED = false
-
-// mock redis always connected
-if (env.MOCK_REDIS) {
-  CONNECTED = true
-}
-
-function pickClient(selectDb: number): any {
-  return CLIENTS[selectDb]
-}
-
-function connectionError(
-  selectDb: number,
-  timeout: NodeJS.Timeout,
-  err: Error | string
-) {
-  // manually shut down, ignore errors
-  if (CLOSED) {
-    return
-  }
-  pickClient(selectDb).disconnect()
-  CLOSED = true
-  // always clear this on error
-  clearTimeout(timeout)
-  CONNECTED = false
-  console.error("Redis connection failed - " + err)
-  setTimeout(() => {
-    init()
-  }, RETRY_PERIOD_MS)
-}
-
-/**
- * Inits the system, will error if unable to connect to redis cluster (may take up to 10 seconds) otherwise
- * will return the ioredis client which will be ready to use.
- */
-function init(selectDb = DEFAULT_SELECT_DB) {
-  const RedisCore = env.MOCK_REDIS && MockRedis ? MockRedis : Redis
-  let timeout: NodeJS.Timeout
-  CLOSED = false
-  let client = pickClient(selectDb)
-  // already connected, ignore
-  if (client && CONNECTED) {
-    return
-  }
-  // testing uses a single in memory client
-  if (env.MOCK_REDIS) {
-    CLIENTS[selectDb] = new RedisCore(getRedisOptions())
-  }
-  // start the timer - only allowed 5 seconds to connect
-  timeout = setTimeout(() => {
-    if (!CONNECTED) {
-      connectionError(
-        selectDb,
-        timeout,
-        "Did not successfully connect in timeout"
-      )
-    }
-  }, STARTUP_TIMEOUT_MS)
-
-  // disconnect any lingering client
-  if (client) {
-    client.disconnect()
-  }
-  const { redisProtocolUrl, opts, host, port } = getRedisOptions()
-
-  if (CLUSTERED) {
-    client = new RedisCore.Cluster([{ host, port }], opts)
-  } else if (redisProtocolUrl) {
-    client = new RedisCore(redisProtocolUrl)
-  } else {
-    client = new RedisCore(opts)
-  }
-  // attach handlers
-  client.on("end", (err: Error) => {
-    if (env.isTest()) {
-      // don't try to re-connect in test env
-      // allow the process to exit
-      return
-    }
-    connectionError(selectDb, timeout, err)
-  })
-  client.on("error", (err: Error) => {
-    connectionError(selectDb, timeout, err)
-  })
-  client.on("connect", () => {
-    clearTimeout(timeout)
-    CONNECTED = true
-  })
-  CLIENTS[selectDb] = client
-}
-
-function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
-  return new Promise(resolve => {
-    if (pickClient(selectDb) == null) {
-      init()
-    } else if (CONNECTED) {
-      resolve("")
-      return
-    }
-    // check if the connection is ready
-    const interval = timers.set(() => {
-      if (CONNECTED) {
-        timers.clear(interval)
-        resolve("")
-      }
-    }, 500)
-  })
-}
-
-/**
- * Utility function, takes a redis stream and converts it to a promisified response -
- * this can only be done with redis streams because they will have an end.
- * @param stream A redis stream, specifically as this type of stream will have an end.
- * @param client The client to use for further lookups.
- * @return {Promise<object>} The final output of the stream
- */
-function promisifyStream(stream: any, client: RedisWrapper) {
-  return new Promise((resolve, reject) => {
-    const outputKeys = new Set()
-    stream.on("data", (keys: string[]) => {
-      keys.forEach(key => {
-        outputKeys.add(key)
-      })
-    })
-    stream.on("error", (err: Error) => {
-      reject(err)
-    })
-    stream.on("end", async () => {
-      const keysArray: string[] = Array.from(outputKeys) as string[]
-      try {
-        let getPromises = []
-        for (let key of keysArray) {
-          getPromises.push(client.get(key))
-        }
-        const jsonArray = await Promise.all(getPromises)
-        resolve(
-          keysArray.map(key => ({
-            key: removeDbPrefix(key),
-            value: JSON.parse(jsonArray.shift()),
-          }))
-        )
-      } catch (err) {
-        reject(err)
-      }
-    })
-  })
-}
-
-class RedisWrapper {
-  _db: string
-  _select: number
-
-  constructor(db: string, selectDb: number | null = null) {
-    this._db = db
-    this._select = selectDb || DEFAULT_SELECT_DB
-  }
-
-  getClient() {
-    return pickClient(this._select)
-  }
-
-  async init() {
-    CLOSED = false
-    init(this._select)
-    await waitForConnection(this._select)
-    if (this._select && !env.isTest()) {
-      this.getClient().select(this._select)
-    }
-    return this
-  }
-
-  async finish() {
-    CLOSED = true
-    this.getClient().disconnect()
-  }
-
-  async scan(key = ""): Promise<any> {
-    const db = this._db
-    key = `${db}${SEPARATOR}${key}`
-    let stream
-    if (CLUSTERED) {
-      let node = this.getClient().nodes("master")
-      stream = node[0].scanStream({ match: key + "*", count: 100 })
-    } else {
-      stream = this.getClient().scanStream({ match: key + "*", count: 100 })
-    }
-    return promisifyStream(stream, this.getClient())
-  }
-
-  async keys(pattern: string) {
-    const db = this._db
-    return this.getClient().keys(addDbPrefix(db, pattern))
-  }
-
-  async exists(key: string) {
-    const db = this._db
-    return await this.getClient().exists(addDbPrefix(db, key))
-  }
-
-  async get(key: string) {
-    const db = this._db
-    let response = await this.getClient().get(addDbPrefix(db, key))
-    // overwrite the prefixed key
-    if (response != null && response.key) {
-      response.key = key
-    }
-    // if its not an object just return the response
-    try {
-      return JSON.parse(response)
-    } catch (err) {
-      return response
-    }
-  }
-
-  async bulkGet(keys: string[]) {
-    const db = this._db
-    if (keys.length === 0) {
-      return {}
-    }
-    const prefixedKeys = keys.map(key => addDbPrefix(db, key))
-    let response = await this.getClient().mget(prefixedKeys)
-    if (Array.isArray(response)) {
-      let final: any = {}
-      let count = 0
-      for (let result of response) {
-        if (result) {
-          let parsed
-          try {
-            parsed = JSON.parse(result)
-          } catch (err) {
-            parsed = result
-          }
-          final[keys[count]] = parsed
-        }
-        count++
-      }
-      return final
-    } else {
-      throw new Error(`Invalid response: ${response}`)
-    }
-  }
-
-  async store(key: string, value: any, expirySeconds: number | null = null) {
-    const db = this._db
-    if (typeof value === "object") {
-      value = JSON.stringify(value)
-    }
-    const prefixedKey = addDbPrefix(db, key)
-    await this.getClient().set(prefixedKey, value)
-    if (expirySeconds) {
-      await this.getClient().expire(prefixedKey, expirySeconds)
-    }
-  }
-
-  async getTTL(key: string) {
-    const db = this._db
-    const prefixedKey = addDbPrefix(db, key)
-    return this.getClient().ttl(prefixedKey)
-  }
-
-  async setExpiry(key: string, expirySeconds: number | null) {
-    const db = this._db
-    const prefixedKey = addDbPrefix(db, key)
-    await this.getClient().expire(prefixedKey, expirySeconds)
-  }
-
-  async delete(key: string) {
-    const db = this._db
-    await this.getClient().del(addDbPrefix(db, key))
-  }
-
-  async clear() {
-    let items = await this.scan()
-    await Promise.all(items.map((obj: any) => this.delete(obj.key)))
-  }
-}
-
-export default RedisWrapper

package/src/redis/redlockImpl.ts

@@ -1,139 +0,0 @@
-import Redlock from "redlock"
-import { getLockClient } from "./init"
-import { LockOptions, LockType } from "@budibase/types"
-import * as context from "../context"
-import env from "../environment"
-
-async function getClient(
-  type: LockType,
-  opts?: Redlock.Options
-): Promise<Redlock> {
-  if (type === LockType.CUSTOM) {
-    return newRedlock(opts)
-  }
-  if (env.isTest() && type !== LockType.TRY_ONCE) {
-    return newRedlock(OPTIONS.TEST)
-  }
-  switch (type) {
-    case LockType.TRY_ONCE: {
-      return newRedlock(OPTIONS.TRY_ONCE)
-    }
-    case LockType.TRY_TWICE: {
-      return newRedlock(OPTIONS.TRY_TWICE)
-    }
-    case LockType.DEFAULT: {
-      return newRedlock(OPTIONS.DEFAULT)
-    }
-    case LockType.DELAY_500: {
-      return newRedlock(OPTIONS.DELAY_500)
-    }
-    default: {
-      throw new Error(`Could not get redlock client: ${type}`)
-    }
-  }
-}
-
-const OPTIONS = {
-  TRY_ONCE: {
-    // immediately throws an error if the lock is already held
-    retryCount: 0,
-  },
-  TRY_TWICE: {
-    retryCount: 1,
-  },
-  TEST: {
-    // higher retry count in unit tests
-    // due to high contention.
-    retryCount: 100,
-  },
-  DEFAULT: {
-    // the expected clock drift; for more details
-    // see http://redis.io/topics/distlock
-    driftFactor: 0.01, // multiplied by lock ttl to determine drift time
-
-    // the max number of times Redlock will attempt
-    // to lock a resource before erroring
-    retryCount: 10,
-
-    // the time in ms between attempts
-    retryDelay: 200, // time in ms
-
-    // the max time in ms randomly added to retries
-    // to improve performance under high contention
-    // see https://www.awsarchitectureblog.com/2015/03/backoff.html
-    retryJitter: 100, // time in ms
-  },
-  DELAY_500: {
-    retryDelay: 500,
-  },
-}
-
-export async function newRedlock(opts: Redlock.Options = {}) {
-  let options = { ...OPTIONS.DEFAULT, ...opts }
-  const redisWrapper = await getLockClient()
-  const client = redisWrapper.getClient()
-  return new Redlock([client], options)
-}
-
-type SuccessfulRedlockExecution<T> = {
-  executed: true
-  result: T
-}
-type UnsuccessfulRedlockExecution = {
-  executed: false
-}
-
-type RedlockExecution<T> =
-  | SuccessfulRedlockExecution<T>
-  | UnsuccessfulRedlockExecution
-
-function getLockName(opts: LockOptions) {
-  // determine lock name
-  // by default use the tenantId for uniqueness, unless using a system lock
-  const prefix = opts.systemLock ? "system" : context.getTenantId()
-  let name: string = `lock:${prefix}_${opts.name}`
-  // add additional unique name if required
-  if (opts.resource) {
-    name = name + `_${opts.resource}`
-  }
-  return name
-}
-
-export async function doWithLock<T>(
-  opts: LockOptions,
-  task: () => Promise<T>
-): Promise<RedlockExecution<T>> {
-  const redlock = await getClient(opts.type, opts.customOptions)
-  let lock
-  try {
-    const name = getLockName(opts)
-
-    // create the lock
-    lock = await redlock.lock(name, opts.ttl)
-
-    // perform locked task
-    // need to await to ensure completion before unlocking
-    const result = await task()
-    return { executed: true, result }
-  } catch (e: any) {
-    console.warn("lock error")
-    // lock limit exceeded
-    if (e.name === "LockError") {
-      if (opts.type === LockType.TRY_ONCE) {
-        // don't throw for try-once locks, they will always error
-        // due to retry count (0) exceeded
-        return { executed: false }
-      } else {
-        console.error(e)
-        throw e
-      }
-    } else {
-      console.error(e)
-      throw e
-    }
-  } finally {
-    if (lock) {
-      await lock.unlock()
-    }
-  }
-}

package/src/redis/utils.ts

@@ -1,117 +0,0 @@
-import env from "../environment"
-
-const SLOT_REFRESH_MS = 2000
-const CONNECT_TIMEOUT_MS = 10000
-export const SEPARATOR = "-"
-
-/**
- * These Redis databases help us to segment up a Redis keyspace by prepending the
- * specified database name onto the cache key. This means that a single real Redis database
- * can be split up a bit; allowing us to use scans on small databases to find some particular
- * keys within.
- * If writing a very large volume of keys is expected (say 10K+) then it is better to keep these out
- * of the default keyspace and use a separate one - the SelectableDatabase can be used for this.
- */
-export enum Databases {
-  PW_RESETS = "pwReset",
-  VERIFICATIONS = "verification",
-  INVITATIONS = "invitation",
-  DEV_LOCKS = "devLocks",
-  DEBOUNCE = "debounce",
-  SESSIONS = "session",
-  USER_CACHE = "users",
-  FLAGS = "flags",
-  APP_METADATA = "appMetadata",
-  QUERY_VARS = "queryVars",
-  LICENSES = "license",
-  GENERIC_CACHE = "data_cache",
-  WRITE_THROUGH = "writeThrough",
-  LOCKS = "locks",
-  SOCKET_IO = "socket_io",
-}
-
-/**
- * These define the numeric Redis databases that can be access with the SELECT command -
- * (https://redis.io/commands/select/). By default a Redis server/cluster will have 16 selectable
- * databases, increasing this count increases the amount of CPU/memory required to run the server.
- * Ideally new Redis keyspaces should be used sparingly, only when absolutely necessary for performance
- * to be maintained. Generally a keyspace can grow to be very large is scans are not needed or desired,
- * but if you need to walk through all values in a database periodically then a separate selectable
- * keyspace should be used.
- */
-export enum SelectableDatabase {
-  DEFAULT = 0,
-  SOCKET_IO = 1,
-  UNUSED_1 = 2,
-  UNUSED_2 = 3,
-  UNUSED_3 = 4,
-  UNUSED_4 = 5,
-  UNUSED_5 = 6,
-  UNUSED_6 = 7,
-  UNUSED_7 = 8,
-  UNUSED_8 = 9,
-  UNUSED_9 = 10,
-  UNUSED_10 = 11,
-  UNUSED_11 = 12,
-  UNUSED_12 = 13,
-  UNUSED_13 = 14,
-  UNUSED_14 = 15,
-}
-
-export function getRedisOptions() {
-  let password = env.REDIS_PASSWORD
-  let url: string[] | string = env.REDIS_URL.split("//")
-  // get rid of the protocol
-  url = url.length > 1 ? url[1] : url[0]
-  // check for a password etc
-  url = url.split("@")
-  if (url.length > 1) {
-    // get the password
-    password = url[0].split(":")[1]
-    url = url[1]
-  } else {
-    url = url[0]
-  }
-  const [host, port] = url.split(":")
-
-  let redisProtocolUrl
-
-  // fully qualified redis URL
-  if (/rediss?:\/\//.test(env.REDIS_URL)) {
-    redisProtocolUrl = env.REDIS_URL
-  }
-
-  const opts: any = {
-    connectTimeout: CONNECT_TIMEOUT_MS,
-  }
-  if (env.REDIS_CLUSTERED) {
-    opts.redisOptions = {}
-    opts.redisOptions.tls = {}
-    opts.redisOptions.password = password
-    opts.slotsRefreshTimeout = SLOT_REFRESH_MS
-    opts.dnsLookup = (address: string, callback: any) => callback(null, address)
-  } else {
-    opts.host = host
-    opts.port = port
-    opts.password = password
-  }
-  return { opts, host, port: parseInt(port), redisProtocolUrl }
-}
-
-export function addDbPrefix(db: string, key: string) {
-  if (key.includes(db)) {
-    return key
-  }
-  return `${db}${SEPARATOR}${key}`
-}
-
-export function removeDbPrefix(key: string) {
-  let parts = key.split(SEPARATOR)
-  if (parts.length >= 2) {
-    parts.shift()
-    return parts.join(SEPARATOR)
-  } else {
-    // return the only part
-    return parts[0]
-  }
-}