effect-distributed-lock 0.0.3 → 0.0.5

package/redis-semaphore/src/semaphore/refresh/lua.ts

@@ -0,0 +1,25 @@
+import { createEval } from '../../utils/index'
+
+export const refreshLua = createEval<
+  [string, number, string, number, number],
+  0 | 1
+>(
+  `
+  local key = KEYS[1]
+  local limit = tonumber(ARGV[1])
+  local identifier = ARGV[2]
+  local lockTimeout = tonumber(ARGV[3])
+  local now = tonumber(ARGV[4])
+  local expiredTimestamp = now - lockTimeout
+
+  redis.call('zremrangebyscore', key, '-inf', expiredTimestamp)
+
+  if redis.call('zscore', key, identifier) then
+    redis.call('zadd', key, now, identifier)
+    redis.call('pexpire', key, lockTimeout)
+    return 1
+  else
+    return 0
+  end`,
+  1
+)

package/redis-semaphore/src/semaphore/release.ts

@@ -0,0 +1,14 @@
+import createDebug from 'debug'
+import { RedisClient } from '../types'
+
+const debug = createDebug('redis-semaphore:semaphore:release')
+
+export async function releaseSemaphore(
+  client: RedisClient,
+  key: string,
+  identifier: string
+): Promise<void> {
+  debug(key, identifier)
+  const result = await client.zrem(key, identifier)
+  debug('result', typeof result, result)
+}

package/redis-semaphore/src/types.ts

@@ -0,0 +1,63 @@
+import LostLockError from './errors/LostLockError'
+import { Lock } from './Lock'
+
+import type * as ioredis from 'ioredis'
+
+/**
+ * ioredis-like Redis client
+ */
+export type RedisClient = Pick<
+  ioredis.Redis,
+  'eval' | 'evalsha' | 'get' | 'set' | 'zrem'
+> &
+  Partial<Pick<ioredis.Redis, 'options'>>
+
+export interface LockLostCallback {
+  (this: Lock, err: LostLockError): void
+}
+
+export interface TimeoutOptions {
+  lockTimeout?: number
+  acquireTimeout?: number
+  acquireAttemptsLimit?: number
+  retryInterval?: number
+  refreshInterval?: number
+}
+
+export interface LockOptions extends TimeoutOptions {
+  /**
+   * @deprecated Use `identifier` + `acquiredExternally: true` instead. Will be removed in next major release.
+   */
+  externallyAcquiredIdentifier?: string
+
+  /**
+   * @deprecated Provide custom `identifier` instead. Will be removed in next major release.
+   */
+  identifierSuffix?: string
+
+  /**
+   * Identifier of lock. By default is `crypto.randomUUID()`.
+   *
+   * Must be unique between parallel executors otherwise locks with same identifier *can* be treated as the same lock holder.
+   *
+   * Override only if you know what you are doing, see `acquireExternally` option.
+   */
+  identifier?: string
+
+  /**
+   * If `identifier` provided and `acquiredExternally` is `true` then `_refresh` will be used instead of `_acquire` in `.tryAcquire()`/`.acquire()`.
+   *
+   * Useful for lock sharing between processes: acquire in scheduler, refresh and release in handler.
+   */
+  acquiredExternally?: true
+
+  onLockLost?: LockLostCallback
+}
+
+export interface AcquireOptions {
+  identifier: string
+  lockTimeout: number
+  acquireTimeout: number
+  acquireAttemptsLimit: number
+  retryInterval: number
+}

package/redis-semaphore/src/utils/createEval.ts

@@ -0,0 +1,45 @@
+import { createHash } from 'crypto'
+import createDebug from 'debug'
+import { RedisClient } from '../types'
+import { getConnectionName } from './index'
+
+const debug = createDebug('redis-semaphore:eval')
+
+function createSHA1(script: string): string {
+  return createHash('sha1').update(script, 'utf8').digest('hex')
+}
+
+function isNoScriptError(err: Error): boolean {
+  return err.toString().indexOf('NOSCRIPT') !== -1
+}
+
+export default function createEval<Args extends Array<number | string>, Result>(
+  script: string,
+  keysCount: number
+): (client: RedisClient, args: Args) => Promise<Result> {
+  const sha1 = createSHA1(script)
+  debug('creating script:', script, 'sha1:', sha1)
+  return async function optimizedEval(
+    client: RedisClient,
+    args: Args
+  ): Promise<Result> {
+    const connectionName = getConnectionName(client)
+    const evalSHAArgs = [sha1, keysCount, ...args]
+    debug(connectionName, sha1, 'attempt, args:', evalSHAArgs)
+    try {
+      return (await client.evalsha(sha1, keysCount, ...args)) as Promise<Result>
+    } catch (err) {
+      if (err instanceof Error && isNoScriptError(err)) {
+        const evalArgs = [script, keysCount, ...args]
+        debug(connectionName, sha1, 'fallback to eval, args:', evalArgs)
+        return (await client.eval(
+          script,
+          keysCount,
+          ...args
+        )) as Promise<Result>
+      } else {
+        throw err
+      }
+    }
+  }
+}

package/redis-semaphore/src/utils/index.ts

@@ -0,0 +1,13 @@
+import { RedisClient } from '../types'
+import createEval from './createEval'
+
+export { createEval }
+
+export async function delay(ms: number): Promise<void> {
+  return await new Promise(resolve => setTimeout(resolve, ms))
+}
+
+export function getConnectionName(client: RedisClient): string {
+  const connectionName = client.options?.connectionName
+  return connectionName ? `<${connectionName}>` : '<unknown client>'
+}

package/redis-semaphore/src/utils/redlock.ts

@@ -0,0 +1,7 @@
+export function getQuorum(clientCount: number): number {
+  return Math.round((clientCount + 1) / 2)
+}
+
+export function smartSum(count: number, zeroOrOne: number): number {
+  return count + zeroOrOne
+}

package/redis-semaphore/test/init.test.ts

@@ -0,0 +1,9 @@
+import { init, removeAllListeners } from './unhandledRejection'
+
+before(() => {
+  init()
+})
+
+after(() => {
+  removeAllListeners()
+})

package/redis-semaphore/test/redisClient.ts

@@ -0,0 +1,82 @@
+import Redis from 'ioredis'
+import RedisMock from 'ioredis-mock'
+import { once } from 'stream'
+
+function createClient(num: number) {
+  const serverURL =
+    process.env[`REDIS_URI${num}`] || `redis://127.0.0.1:${6000 + num}`
+  const client = new Redis(serverURL, {
+    connectionName: `client${num}`,
+    lazyConnect: true,
+    enableOfflineQueue: false,
+    autoResendUnfulfilledCommands: false, // dont queue commands while server is offline (dont break test logic)
+    maxRetriesPerRequest: 0, // dont retry, fail faster (default is 20)
+
+    // https://github.com/luin/ioredis#auto-reconnect
+    // retryStrategy is a function that will be called when the connection is lost.
+    // The argument times means this is the nth reconnection being made and the return value represents how long (in ms) to wait to reconnect.
+    retryStrategy() {
+      return 100 // for tests we disable increasing timeout
+    }
+  })
+  client.on('error', err => {
+    console.log('Redis client error:', err.message)
+  })
+  return client
+}
+
+function createClientMock(num: number) {
+  return new RedisMock(`redis://mock:${4200 + num}`, {
+    connectionName: `client-mock${num}`,
+    lazyConnect: true,
+    enableOfflineQueue: false,
+    autoResendUnfulfilledCommands: false, // dont queue commands while server is offline (dont break test logic)
+    maxRetriesPerRequest: 0, // dont retry, fail faster (default is 20)
+
+    // https://github.com/luin/ioredis#auto-reconnect
+    // retryStrategy is a function that will be called when the connection is lost.
+    // The argument times means this is the nth reconnection being made and the return value represents how long (in ms) to wait to reconnect.
+    retryStrategy() {
+      return 100 // for tests we disable increasing timeout
+    }
+  })
+}
+
+export const client1 = createClient(1)
+export const client2 = createClient(2)
+export const client3 = createClient(3)
+
+export const allClients = [client1, client2, client3]
+
+export const clientMock1 = createClientMock(1)
+export const clientMock2 = createClientMock(2)
+export const clientMock3 = createClientMock(3)
+
+export const allClientMocks = [clientMock1, clientMock2, clientMock3]
+
+before(async () => {
+  await Promise.all(allClients.map(c => c.connect()))
+  await Promise.all(allClientMocks.map(c => c.connect()))
+})
+
+beforeEach(async () => {
+  await Promise.all(
+    allClients.map(c => {
+      if (c.status !== 'ready') {
+        console.warn(
+          `client ${c.options.connectionName} status = ${c.status}. Wait for ready.`
+        )
+        return once(c, 'ready')
+      }
+      return null
+    })
+  )
+  await Promise.all(allClients.map(c => c.flushdb()))
+  await Promise.all(allClientMocks.map(c => c.flushdb()))
+})
+
+after(async () => {
+  await Promise.all(allClients.map(c => c.quit()))
+  await Promise.all(allClientMocks.map(c => c.quit()))
+  // allClients.forEach(c => c.disconnect())
+})

package/redis-semaphore/test/setup.ts

@@ -0,0 +1,6 @@
+import chai from 'chai'
+import chaiAsPromised from 'chai-as-promised'
+import sinonChai from 'sinon-chai'
+
+chai.use(chaiAsPromised)
+chai.use(sinonChai)

package/redis-semaphore/test/shell.test.ts

@@ -0,0 +1,15 @@
+import { downRedisServer, upRedisServer } from './shell'
+
+describe('TEST UTILS', () => {
+  describe('shell', () => {
+    it('should up redis server', async function () {
+      this.timeout(30000)
+      await upRedisServer(1)
+    })
+    it('should down and up redis servers', async function () {
+      this.timeout(30000)
+      await downRedisServer(1)
+      await upRedisServer(1)
+    })
+  })
+})

package/redis-semaphore/test/shell.ts

@@ -0,0 +1,48 @@
+import { exec } from 'child_process'
+
+import { delay } from '../src/utils/index'
+
+const LOGGING = !!process.env.LOGSHELL
+
+function sh(cmd: string) {
+  return new Promise<void>((resolve, reject) => {
+    const cp = exec(cmd, (err, stdout, stderr) => {
+      if (stdout && LOGGING) {
+        console.log(`[${cp.pid}] stdout:`)
+        console.log(stdout)
+      }
+      if (stderr && LOGGING) {
+        console.log(`[${cp.pid}] stderr:`)
+        console.log(stderr)
+      }
+      if (err) {
+        reject(err)
+      } else {
+        resolve()
+      }
+    })
+    console.log(`[${cp.pid}] ${cmd}`)
+  })
+}
+
+export async function upRedisServer(num: number) {
+  const port = 6000 + num
+  await sh(
+    `docker compose up -d redis${num} && yarn wait-for --redis redis://127.0.0.1:${port}`
+  )
+}
+
+export async function downRedisServer(num: number) {
+  const port = 6000 + num
+  await sh(`docker compose stop redis${num}`)
+  let tries = 0
+  while (true) {
+    try {
+      console.log(`wait server${num} shut down... ${++tries}`)
+      await sh(`yarn wait-for --redis redis://127.0.0.1:${port} -c 1`)
+      await delay(100)
+    } catch {
+      break
+    }
+  }
+}

package/redis-semaphore/test/src/Lock.test.ts

@@ -0,0 +1,37 @@
+import { LockOptions } from '../../src'
+import { Lock } from '../../src/Lock'
+import { delay } from '../../src/utils'
+
+describe('Lock', () => {
+  describe('refresh and release race condition', () => {
+    class TestLock extends Lock {
+      protected _kind = 'test-lock'
+      protected _key: string
+      constructor(key: string, options: LockOptions) {
+        super(options)
+        this._key = key
+      }
+      protected async _refresh(): Promise<boolean> {
+        await delay(200)
+        return false
+      }
+      protected async _acquire(): Promise<boolean> {
+        return true
+      }
+      protected async _release(): Promise<void> {}
+    }
+    it('should not throw LostLock error when refresh started but not finished before release happened', async function () {
+      const lock = new TestLock('key', {
+        lockTimeout: 1000,
+        acquireTimeout: 1000,
+        refreshInterval: 50
+      })
+      try {
+        await lock.acquire()
+        await delay(100)
+      } finally {
+        await lock.release()
+      }
+    })
+  })
+})