@budibase/backend-core 3.2.4 → 3.2.6

package/src/redis/redis.ts

@@ -1,358 +0,0 @@
-import env from "../environment"
-import Redis, { Cluster } from "ioredis"
-// mock-redis doesn't have any typing
-let MockRedis: any | undefined
-if (env.MOCK_REDIS) {
-  try {
-    // ioredis mock is all in memory
-    MockRedis = require("ioredis-mock")
-  } catch (err) {
-    console.log("Mock redis unavailable")
-  }
-}
-import {
-  addDbPrefix,
-  removeDbPrefix,
-  getRedisOptions,
-  SEPARATOR,
-  SelectableDatabase,
-  getRedisConnectionDetails,
-} from "./utils"
-import { logAlert } from "../logging"
-import * as timers from "../timers"
-
-const RETRY_PERIOD_MS = 2000
-const STARTUP_TIMEOUT_MS = 5000
-const CLUSTERED = env.REDIS_CLUSTERED
-const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
-
-// for testing just generate the client once
-let CLOSED = false
-const CLIENTS: Record<number, Redis> = {}
-let CONNECTED = false
-
-// mock redis always connected
-if (env.MOCK_REDIS) {
-  CONNECTED = true
-}
-
-function pickClient(selectDb: number) {
-  return CLIENTS[selectDb]
-}
-
-function connectionError(timeout: NodeJS.Timeout, err: Error | string) {
-  // manually shut down, ignore errors
-  if (CLOSED) {
-    return
-  }
-  CLOSED = true
-  // always clear this on error
-  clearTimeout(timeout)
-  CONNECTED = false
-  logAlert("Redis connection failed", err)
-  setTimeout(() => {
-    init()
-  }, RETRY_PERIOD_MS)
-}
-
-/**
- * Inits the system, will error if unable to connect to redis cluster (may take up to 10 seconds) otherwise
- * will return the ioredis client which will be ready to use.
- */
-function init(selectDb = DEFAULT_SELECT_DB) {
-  const RedisCore = env.MOCK_REDIS && MockRedis ? MockRedis : Redis
-  let timeout: NodeJS.Timeout
-  CLOSED = false
-  let client = pickClient(selectDb)
-  // already connected, ignore
-  if (client && CONNECTED) {
-    return
-  }
-  // testing uses a single in memory client
-  if (env.MOCK_REDIS) {
-    CLIENTS[selectDb] = new RedisCore(getRedisOptions())
-  }
-  // start the timer - only allowed 5 seconds to connect
-  timeout = setTimeout(() => {
-    if (!CONNECTED) {
-      connectionError(timeout, "Did not successfully connect in timeout")
-    }
-  }, STARTUP_TIMEOUT_MS)
-
-  // disconnect any lingering client
-  if (client) {
-    client.disconnect()
-  }
-  const { host, port } = getRedisConnectionDetails()
-  const opts = getRedisOptions()
-
-  if (CLUSTERED) {
-    client = new RedisCore.Cluster([{ host, port }], opts)
-  } else {
-    client = new RedisCore(opts)
-  }
-  // attach handlers
-  client.on("end", (err: Error) => {
-    if (env.isTest()) {
-      // don't try to re-connect in test env
-      // allow the process to exit
-      return
-    }
-    connectionError(timeout, err)
-  })
-  client.on("error", (err: Error) => {
-    connectionError(timeout, err)
-  })
-  client.on("connect", () => {
-    console.log(`Connected to Redis DB: ${selectDb}`)
-    clearTimeout(timeout)
-    CONNECTED = true
-  })
-  CLIENTS[selectDb] = client
-}
-
-export function closeAll() {
-  Object.values(CLIENTS).forEach(client => client.disconnect())
-}
-
-function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
-  return new Promise(resolve => {
-    if (pickClient(selectDb) == null) {
-      init()
-    } else if (CONNECTED) {
-      resolve("")
-      return
-    }
-    // check if the connection is ready
-    const interval = timers.set(() => {
-      if (CONNECTED) {
-        timers.clear(interval)
-        resolve("")
-      }
-    }, 500)
-  })
-}
-
-/**
- * Utility function, takes a redis stream and converts it to a promisified response -
- * this can only be done with redis streams because they will have an end.
- * @param stream A redis stream, specifically as this type of stream will have an end.
- * @param client The client to use for further lookups.
- * @return The final output of the stream
- */
-function promisifyStream(stream: any, client: RedisWrapper) {
-  return new Promise((resolve, reject) => {
-    const outputKeys = new Set()
-    stream.on("data", (keys: string[]) => {
-      keys.forEach(key => {
-        outputKeys.add(key)
-      })
-    })
-    stream.on("error", (err: Error) => {
-      reject(err)
-    })
-    stream.on("end", async () => {
-      const keysArray: string[] = Array.from(outputKeys) as string[]
-      try {
-        let getPromises = []
-        for (let key of keysArray) {
-          getPromises.push(client.get(key))
-        }
-        const jsonArray = await Promise.all(getPromises)
-        resolve(
-          keysArray.map(key => ({
-            key: removeDbPrefix(key),
-            value: JSON.parse(jsonArray.shift()),
-          }))
-        )
-      } catch (err) {
-        reject(err)
-      }
-    })
-  })
-}
-
-class RedisWrapper {
-  _db: string
-  _select: number
-
-  constructor(db: string, selectDb: number | null = null) {
-    this._db = db
-    this._select = selectDb || DEFAULT_SELECT_DB
-  }
-
-  getClient() {
-    return pickClient(this._select)
-  }
-
-  async init() {
-    CLOSED = false
-    init(this._select)
-    await waitForConnection(this._select)
-    if (this._select && !env.isTest()) {
-      this.getClient().select(this._select)
-    }
-    return this
-  }
-
-  async finish() {
-    CLOSED = true
-    this.getClient().disconnect()
-  }
-
-  async scan(key = ""): Promise<any> {
-    const db = this._db
-    key = `${db}${SEPARATOR}${key}`
-    let stream
-    if (CLUSTERED) {
-      let node = (this.getClient() as never as Cluster).nodes("master")
-      stream = node[0].scanStream({ match: key + "*", count: 100 })
-    } else {
-      stream = (this.getClient() as Redis).scanStream({
-        match: key + "*",
-        count: 100,
-      })
-    }
-    return promisifyStream(stream, this.getClient() as any)
-  }
-
-  async keys(pattern: string) {
-    const db = this._db
-    return this.getClient().keys(addDbPrefix(db, pattern))
-  }
-
-  async exists(key: string) {
-    const db = this._db
-    return await this.getClient().exists(addDbPrefix(db, key))
-  }
-
-  async get(key: string) {
-    const db = this._db
-    const response = await this.getClient().get(addDbPrefix(db, key))
-    // overwrite the prefixed key
-    // @ts-ignore
-    if (response != null && response.key) {
-      // @ts-ignore
-      response.key = key
-    }
-    // if its not an object just return the response
-    try {
-      return JSON.parse(response!)
-    } catch (err) {
-      return response
-    }
-  }
-
-  async bulkGet<T>(keys: string[]) {
-    const db = this._db
-    if (keys.length === 0) {
-      return {}
-    }
-    const prefixedKeys = keys.map(key => addDbPrefix(db, key))
-    let response = await this.getClient().mget(prefixedKeys)
-    if (Array.isArray(response)) {
-      let final: Record<string, T> = {}
-      let count = 0
-      for (let result of response) {
-        if (result) {
-          let parsed
-          try {
-            parsed = JSON.parse(result)
-          } catch (err) {
-            parsed = result
-          }
-          final[keys[count]] = parsed
-        }
-        count++
-      }
-      return final
-    } else {
-      throw new Error(`Invalid response: ${response}`)
-    }
-  }
-
-  async store(key: string, value: any, expirySeconds: number | null = null) {
-    const db = this._db
-    if (typeof value === "object") {
-      value = JSON.stringify(value)
-    }
-    const prefixedKey = addDbPrefix(db, key)
-    await this.getClient().set(prefixedKey, value)
-    if (expirySeconds) {
-      await this.getClient().expire(prefixedKey, expirySeconds)
-    }
-  }
-
-  async bulkStore(
-    data: Record<string, any>,
-    expirySeconds: number | null = null
-  ) {
-    const client = this.getClient()
-
-    const dataToStore = Object.entries(data).reduce((acc, [key, value]) => {
-      acc[addDbPrefix(this._db, key)] =
-        typeof value === "object" ? JSON.stringify(value) : value
-      return acc
-    }, {} as Record<string, any>)
-
-    const pipeline = client.pipeline()
-    pipeline.mset(dataToStore)
-
-    if (expirySeconds !== null) {
-      for (const key of Object.keys(dataToStore)) {
-        pipeline.expire(key, expirySeconds)
-      }
-    }
-
-    await pipeline.exec()
-  }
-
-  async getTTL(key: string) {
-    const db = this._db
-    const prefixedKey = addDbPrefix(db, key)
-    return this.getClient().ttl(prefixedKey)
-  }
-
-  async setExpiry(key: string, expirySeconds: number) {
-    const db = this._db
-    const prefixedKey = addDbPrefix(db, key)
-    await this.getClient().expire(prefixedKey, expirySeconds)
-  }
-
-  async delete(key: string) {
-    const db = this._db
-    await this.getClient().del(addDbPrefix(db, key))
-  }
-
-  async bulkDelete(keys: string[]) {
-    const db = this._db
-    await this.getClient().del(keys.map(key => addDbPrefix(db, key)))
-  }
-
-  async clear() {
-    let items = await this.scan()
-    await Promise.all(items.map((obj: any) => this.delete(obj.key)))
-  }
-
-  async increment(key: string) {
-    const result = await this.getClient().incr(addDbPrefix(this._db, key))
-    if (isNaN(result)) {
-      throw new Error(`Redis ${key} does not contain a number`)
-    }
-    return result
-  }
-
-  async deleteIfValue(key: string, value: any) {
-    const client = this.getClient()
-
-    const luaScript = `
-      if redis.call('GET', KEYS[1]) == ARGV[1] then
-        redis.call('DEL', KEYS[1])
-      end
-      `
-
-    await client.eval(luaScript, 1, addDbPrefix(this._db, key), value)
-  }
-}
-
-export default RedisWrapper

package/src/redis/redlockImpl.ts

@@ -1,155 +0,0 @@
-import Redlock from "redlock"
-import { getLockClient } from "./init"
-import { LockOptions, LockType } from "@budibase/types"
-import * as context from "../context"
-import { utils } from "@budibase/shared-core"
-import { Duration } from "../utils"
-
-async function getClient(
-  type: LockType,
-  opts?: Redlock.Options
-): Promise<Redlock> {
-  if (type === LockType.CUSTOM) {
-    return newRedlock(opts)
-  }
-
-  switch (type) {
-    case LockType.TRY_ONCE: {
-      return newRedlock(OPTIONS.TRY_ONCE)
-    }
-    case LockType.TRY_TWICE: {
-      return newRedlock(OPTIONS.TRY_TWICE)
-    }
-    case LockType.DEFAULT: {
-      return newRedlock(OPTIONS.DEFAULT)
-    }
-    case LockType.DELAY_500: {
-      return newRedlock(OPTIONS.DELAY_500)
-    }
-    case LockType.AUTO_EXTEND: {
-      return newRedlock(OPTIONS.AUTO_EXTEND)
-    }
-    default: {
-      throw utils.unreachable(type)
-    }
-  }
-}
-
-const OPTIONS: Record<keyof typeof LockType, Redlock.Options> = {
-  TRY_ONCE: {
-    // immediately throws an error if the lock is already held
-    retryCount: 0,
-  },
-  TRY_TWICE: {
-    retryCount: 1,
-  },
-  DEFAULT: {
-    // the expected clock drift; for more details
-    // see http://redis.io/topics/distlock
-    driftFactor: 0.01, // multiplied by lock ttl to determine drift time
-
-    // the max number of times Redlock will attempt
-    // to lock a resource before erroring
-    retryCount: 10,
-
-    // the time in ms between attempts
-    retryDelay: 200, // time in ms
-
-    // the max time in ms randomly added to retries
-    // to improve performance under high contention
-    // see https://www.awsarchitectureblog.com/2015/03/backoff.html
-    retryJitter: 100, // time in ms
-  },
-  DELAY_500: {
-    retryDelay: 500,
-  },
-  CUSTOM: {},
-  AUTO_EXTEND: {
-    retryCount: -1,
-  },
-}
-
-export async function newRedlock(opts: Redlock.Options = {}) {
-  const options = { ...OPTIONS.DEFAULT, ...opts }
-  const redisWrapper = await getLockClient()
-  const client = redisWrapper.getClient() as any
-  return new Redlock([client], options)
-}
-
-type SuccessfulRedlockExecution<T> = {
-  executed: true
-  result: T
-}
-type UnsuccessfulRedlockExecution = {
-  executed: false
-}
-
-type RedlockExecution<T> =
-  | SuccessfulRedlockExecution<T>
-  | UnsuccessfulRedlockExecution
-
-function getLockName(opts: LockOptions) {
-  // determine lock name
-  // by default use the tenantId for uniqueness, unless using a system lock
-  const prefix = opts.systemLock ? "system" : context.getTenantId()
-  let name: string = `lock:${prefix}_${opts.name}`
-  // add additional unique name if required
-  if (opts.resource) {
-    name = name + `_${opts.resource}`
-  }
-  return name
-}
-
-export const AUTO_EXTEND_POLLING_MS = Duration.fromSeconds(10).toMs()
-
-export async function doWithLock<T>(
-  opts: LockOptions,
-  task: () => Promise<T>
-): Promise<RedlockExecution<T>> {
-  const redlock = await getClient(opts.type, opts.customOptions)
-  let lock: Redlock.Lock | undefined
-  let timeout
-  try {
-    const name = getLockName(opts)
-
-    const ttl =
-      opts.type === LockType.AUTO_EXTEND ? AUTO_EXTEND_POLLING_MS : opts.ttl
-
-    // create the lock
-    lock = await redlock.lock(name, ttl)
-
-    if (opts.type === LockType.AUTO_EXTEND) {
-      // We keep extending the lock while the task is running
-      const extendInIntervals = (): void => {
-        timeout = setTimeout(async () => {
-          lock = await lock!.extend(ttl, () => opts.onExtend && opts.onExtend())
-
-          extendInIntervals()
-        }, ttl / 2)
-      }
-
-      extendInIntervals()
-    }
-
-    // perform locked task
-    // need to await to ensure completion before unlocking
-    const result = await task()
-    return { executed: true, result }
-  } catch (e: any) {
-    // lock limit exceeded
-    if (e.name === "LockError") {
-      if (opts.type === LockType.TRY_ONCE) {
-        // don't throw for try-once locks, they will always error
-        // due to retry count (0) exceeded
-        return { executed: false }
-      } else {
-        throw e
-      }
-    } else {
-      throw e
-    }
-  } finally {
-    clearTimeout(timeout)
-    await lock?.unlock()
-  }
-}

package/src/redis/tests/redis.spec.ts

@@ -1,207 +0,0 @@
-import { GenericContainer, StartedTestContainer } from "testcontainers"
-import { generator, structures } from "../../../tests"
-import RedisWrapper, { closeAll } from "../redis"
-import env from "../../environment"
-import { randomUUID } from "crypto"
-
-jest.setTimeout(30000)
-
-describe("redis", () => {
-  let redis: RedisWrapper
-  let container: StartedTestContainer
-
-  beforeAll(async () => {
-    const container = await new GenericContainer("redis")
-      .withExposedPorts(6379)
-      .start()
-
-    env._set(
-      "REDIS_URL",
-      `${container.getHost()}:${container.getMappedPort(6379)}`
-    )
-    env._set("MOCK_REDIS", 0)
-    env._set("REDIS_PASSWORD", 0)
-  })
-
-  afterAll(() => {
-    container?.stop()
-    closeAll()
-  })
-
-  beforeEach(async () => {
-    redis = new RedisWrapper(structures.db.id())
-    await redis.init()
-  })
-
-  describe("store", () => {
-    it("a basic value can be persisted", async () => {
-      const key = structures.uuid()
-      const value = generator.word()
-
-      await redis.store(key, value)
-
-      expect(await redis.get(key)).toEqual(value)
-    })
-
-    it("objects can be persisted", async () => {
-      const key = structures.uuid()
-      const value = { [generator.word()]: generator.word() }
-
-      await redis.store(key, value)
-
-      expect(await redis.get(key)).toEqual(value)
-    })
-  })
-
-  describe("bulkStore", () => {
-    function createRandomObject(
-      keyLength: number,
-      valueGenerator: () => any = () => randomUUID()
-    ) {
-      return generator
-        .unique(() => randomUUID(), keyLength)
-        .reduce((acc, key) => {
-          acc[key] = valueGenerator()
-          return acc
-        }, {} as Record<string, string>)
-    }
-
-    it("a basic object can be persisted", async () => {
-      const data = createRandomObject(10)
-
-      await redis.bulkStore(data)
-
-      for (const [key, value] of Object.entries(data)) {
-        expect(await redis.get(key)).toEqual(value)
-      }
-
-      expect(await redis.keys("*")).toHaveLength(10)
-    })
-
-    it("a complex object can be persisted", async () => {
-      const data = {
-        ...createRandomObject(10, () => createRandomObject(5)),
-        ...createRandomObject(5),
-      }
-
-      await redis.bulkStore(data)
-
-      for (const [key, value] of Object.entries(data)) {
-        expect(await redis.get(key)).toEqual(value)
-      }
-
-      expect(await redis.keys("*")).toHaveLength(15)
-    })
-
-    it("no TTL is set by default", async () => {
-      const data = createRandomObject(10)
-
-      await redis.bulkStore(data)
-
-      for (const [key, value] of Object.entries(data)) {
-        expect(await redis.get(key)).toEqual(value)
-        expect(await redis.getTTL(key)).toEqual(-1)
-      }
-    })
-
-    it("a bulk store can be persisted with TTL", async () => {
-      const ttl = 500
-      const data = createRandomObject(8)
-
-      await redis.bulkStore(data, ttl)
-
-      for (const [key, value] of Object.entries(data)) {
-        expect(await redis.get(key)).toEqual(value)
-        expect(await redis.getTTL(key)).toEqual(ttl)
-      }
-
-      expect(await redis.keys("*")).toHaveLength(8)
-    })
-
-    it("setting a TTL of -1 will not persist the key", async () => {
-      const ttl = -1
-      const data = createRandomObject(5)
-
-      await redis.bulkStore(data, ttl)
-
-      for (const key of Object.keys(data)) {
-        expect(await redis.get(key)).toBe(null)
-      }
-
-      expect(await redis.keys("*")).toHaveLength(0)
-    })
-  })
-
-  describe("increment", () => {
-    it("can increment on a new key", async () => {
-      const key = structures.uuid()
-      const result = await redis.increment(key)
-      expect(result).toBe(1)
-    })
-
-    it("can increment multiple times", async () => {
-      const key = structures.uuid()
-      const results = [
-        await redis.increment(key),
-        await redis.increment(key),
-        await redis.increment(key),
-        await redis.increment(key),
-        await redis.increment(key),
-      ]
-      expect(results).toEqual([1, 2, 3, 4, 5])
-    })
-
-    it("can increment multiple times in parallel", async () => {
-      const key = structures.uuid()
-      const results = await Promise.all(
-        Array.from({ length: 100 }).map(() => redis.increment(key))
-      )
-      expect(results).toHaveLength(100)
-      expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1))
-    })
-
-    it("can increment existing set keys", async () => {
-      const key = structures.uuid()
-      await redis.store(key, 70)
-      await redis.increment(key)
-
-      const result = await redis.increment(key)
-      expect(result).toBe(72)
-    })
-
-    it.each([
-      generator.word(),
-      generator.bool(),
-      { [generator.word()]: generator.word() },
-    ])("cannot increment if the store value is not a number", async value => {
-      const key = structures.uuid()
-      await redis.store(key, value)
-
-      await expect(redis.increment(key)).rejects.toThrow(
-        "ERR value is not an integer or out of range"
-      )
-    })
-  })
-
-  describe("deleteIfValue", () => {
-    it("can delete if the value matches", async () => {
-      const key = structures.uuid()
-      const value = generator.word()
-      await redis.store(key, value)
-
-      await redis.deleteIfValue(key, value)
-
-      expect(await redis.get(key)).toBeNull()
-    })
-
-    it("will not delete if the value does not matches", async () => {
-      const key = structures.uuid()
-      const value = generator.word()
-      await redis.store(key, value)
-
-      await redis.deleteIfValue(key, generator.word())
-
-      expect(await redis.get(key)).toEqual(value)
-    })
-  })
-})