master_lock 0.1.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66672f888e12006963d8dabbd231b62d140d9a1a
-  data.tar.gz: db57a3095ea591040a5e92a44c0479da528c1248
+  metadata.gz: 321dbc834c9f475dac38543343c696ae1eafda08
+  data.tar.gz: 5f764d38a83de5434bf318944de4dc4b478202e1
 SHA512:
-  metadata.gz: abf1344d1976edbb88d59c542c0de4c936467589a14c32b015f47616e535dbd9015e8de0966afbfb16a5a7822f71d9baccac1f5e12d009b33f4ad8538df153e9
-  data.tar.gz: 277bddc942ef6085421e1d6efca315e9dd70b13271cc5a71de360a1d092c3f5593777245b4a846ac4881ebc9ff678133276c92c70dbb196446f5ae5c2572bfcc
+  metadata.gz: 67f9681ccef00d1fa3e4956524fc419dbb097b770ebb985cbdf16a4f584ecdaaee7220ff4ab68d9699723ebef24131002ee5cc0ec4b42338e79114618c4c145a
+  data.tar.gz: e6b55e755805fc89ad8c26a942e3d6be4770631285d1ed207ad250c2dc5ad95c7d3b77315ec48035e1cd2a317e8dae4235f87f340af349eebe1ea938a4e9bfe9

data/.travis.yml

@@ -3,5 +3,5 @@ language: ruby
 rvm:
   - 2.3.1
 services:
-  - redis_server
+  - redis-server
 before_install: gem install bundler -v 1.13.1

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in master_lock.gemspec
 gemspec
+
+group :test do
+  gem 'coveralls', '~> 0.8', require: false
+end

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016 Jim Posen
+Copyright (c) 2016 Coinbase, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,8 +1,10 @@
 # MasterLock
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/master_lock`. To experiment with that code, run `bin/console` for an interactive prompt.
+[![Build Status](https://travis-ci.org/coinbase/master_lock.svg?branch=master)](https://travis-ci.org/coinbase/master_lock)
+[![Coverage Status](https://coveralls.io/repos/github/coinbase/master_lock/badge.svg?branch=master)](https://coveralls.io/github/coinbase/master_lock?branch=master)
+[![Gem Version](https://badge.fury.io/rb/master_lock.svg)](https://badge.fury.io/rb/master_lock)
 
-TODO: Delete this and the text above, and describe your gem
+MasterLock is a Ruby library for interprocess locking using Redis. Critical sections of code can be wrapped in a MasterLock block that ensures only one thread will run the code at a time. The locks are resilient to process failures by expiring after the thread obtaining them dies.
 
 ## Installation
 
@@ -32,10 +34,8 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/master_lock.
-
+Bug reports and pull requests are welcome on GitHub at https://github.com/coinbase/master_lock.
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/lib/master_lock.rb

@@ -1,4 +1,146 @@
-require "master_lock/version"
+require 'master_lock/version'
 
+require 'socket'
+
+# MasterLock is a system for interprocess locking. Resources can be locked by a
+# string identifier such that only one thread may have the lock at a time. Lock
+# state and owners are stored on a Redis server shared by all processes. Locks
+# are held until either the block of synchronized code completes or the thread
+# that obtained the lock is killed. To prevent the locks from being held
+# indefinitely in the event that the process dies without releasing them, the
+# locks have an expiration time in Redis. While the thread owning the lock is
+# alive, a separate thread will extend the lifetime of the locks so that they
+# do not expire even when the code in the critical section takes a long time to
+# execute.
 module MasterLock
+  class UnconfiguredError < StandardError; end
+  class NotStartedError < StandardError; end
+  class LockNotAcquiredError < StandardError; end
+
+  DEFAULT_ACQUIRE_TIMEOUT = 5
+  DEFAULT_EXTEND_INTERVAL = 15
+  DEFAULT_KEY_PREFIX = "masterlock".freeze
+  DEFAULT_SLEEP_TIME = 5
+  DEFAULT_TTL = 60
+
+  Config = Struct.new(
+    :acquire_timeout,
+    :extend_interval,
+    :hostname,
+    :key_prefix,
+    :process_id,
+    :redis,
+    :sleep_time,
+    :ttl
+  )
+
+  class << self
+    # Obtain a mutex around a critical section of code. Only one thread on any
+    # machine can execute the given block at a time. Returns the result of the
+    # block.
+    #
+    # @param key [String] the unique identifier for the locked resource
+    # @option options [Fixnum] :ttl (60) the length of time in seconds before
+    #   the lock expires
+    # @option options [Fixnum] :acquire_timeout (5) the length of time to wait
+    #   to acquire the lock before timing out
+    # @option options [Fixnum] :extend_interval (15) the amount of time in
+    #   seconds that may pass before extending the lock
+    # @option options [Boolean] :if if this option is falsey, the block will be
+    #   executed without obtaining the lock
+    # @option options [Boolean] :unless if this option is truthy, the block will
+    #   be executed without obtaining the lock
+    # @raise [UnconfiguredError] if a required configuration variable is unset
+    # @raise [NotStartedError] if called before {#start}
+    # @raise [LockNotAcquiredError] if the lock cannot be acquired before the
+    #   timeout
+    def synchronize(key, options = {})
+      check_configured
+      raise NotStartedError unless @registry
+
+      ttl = options[:ttl] || config.ttl
+      acquire_timeout = options[:acquire_timeout] || config.acquire_timeout
+      extend_interval = options[:extend_interval] || config.extend_interval
+
+      raise ArgumentError, "extend_interval cannot be negative" if extend_interval < 0
+      raise ArgumentError, "ttl must be greater extend_interval" if ttl <= extend_interval
+
+      if (options.include?(:if) && !options[:if]) ||
+          (options.include?(:unless) && options[:unless])
+        return yield
+      end
+
+      lock = RedisLock.new(
+        redis: config.redis,
+        key: redis_key(key),
+        ttl: ttl,
+        owner: generate_owner
+      )
+      if !lock.acquire(timeout: acquire_timeout)
+        raise LockNotAcquiredError, key
+      end
+
+      registration =
+        @registry.register(lock, extend_interval)
+      begin
+        yield
+      ensure
+        @registry.unregister(registration)
+        lock.release # TODO: Check result of this
+      end
+    end
+
+    # Starts the background thread to manage and extend currently held locks.
+    # The thread remains alive for the lifetime of the process. This must be
+    # called before any locks may be acquired.
+    def start
+      @registry = Registry.new
+      Thread.new do
+        loop do
+          @registry.extend_locks
+          sleep(config.sleep_time)
+        end
+      end
+    end
+
+    # @return [Config] MasterLock configuration settings
+    def config
+      if !defined?(@config)
+        @config = Config.new
+        @config.acquire_timeout = DEFAULT_ACQUIRE_TIMEOUT
+        @config.extend_interval = DEFAULT_EXTEND_INTERVAL
+        @config.hostname = Socket.gethostname
+        @config.key_prefix = DEFAULT_KEY_PREFIX
+        @config.process_id = Process.pid
+        @config.sleep_time = DEFAULT_SLEEP_TIME
+        @config.ttl = DEFAULT_TTL
+      end
+      @config
+    end
+
+    # Configure MasterLock using block syntax. Simply yields {#config} to the
+    # block.
+    #
+    # @yield [Config] the configuration
+    def configure
+      yield config
+    end
+
+    private
+
+    def check_configured
+      raise UnconfiguredError, "redis must be configured" unless config.redis
+    end
+
+    def generate_owner
+      "#{config.hostname}:#{config.process_id}:#{Thread.current.object_id}"
+    end
+
+    def redis_key(key)
+      "#{config.key_prefix}:#{key}"
+    end
+  end
 end
+
+require 'master_lock/redis_lock'
+require 'master_lock/registry'

data/lib/master_lock/redis_lock.rb

@@ -0,0 +1,97 @@
+require 'master_lock/redis_scripts'
+
+module MasterLock
+  # RedisLock implements a mutex in Redis according to the strategy documented
+  # at http://redis.io/commands/SET#patterns. The lock has a string identifier
+  # and when acquired will be registered to an owner, also identified by a
+  # string. Locks have an expiration time, after which they will be released
+  # automatically so that unexpected failures do not result in locks getting
+  # stuck.
+  class RedisLock
+    DEFAULT_SLEEP_INTERVAL = 0.1
+
+    # @return [Redis] the Redis connection used to manage lock
+    attr_reader :redis
+
+    # @return [String] the unique identifier for the locked resource
+    attr_reader :key
+
+    # @return [String] the identity of the owner acquiring the lock
+    attr_reader :owner
+
+    # @return [Fixnum] the lifetime of the lock in seconds
+    attr_reader :ttl
+
+    def initialize(
+      redis:,
+      key:,
+      owner:,
+      ttl:,
+      sleep_interval: DEFAULT_SLEEP_INTERVAL
+    )
+      @redis = redis
+      @key = key
+      @owner = owner
+      @ttl = ttl
+      @sleep_interval = sleep_interval
+    end
+
+    # Attempt to acquire the lock. If the lock is already held, this will
+    # attempt multiple times to acquire the lock until the timeout period is up.
+    #
+    # @param [Fixnum] how long to wait to acquire the lock before failing
+    # @return [Boolean] whether the lock was acquired successfully
+    def acquire(timeout:)
+      timeout_time = Time.now + timeout
+      loop do
+        locked = redis.set(key, owner, nx: true, px: ttl_ms)
+        return true if locked
+        return false if Time.now >= timeout_time
+        sleep(@sleep_interval)
+      end
+    end
+
+    # Extend the expiration time of the lock if still held by this owner. If the
+    # lock is no longer held by the owner, this method will fail and return
+    # false. The lock lifetime is extended by the configured ttl.
+    #
+    # @return [Boolean] whether the lock was extended successfully
+    def extend
+      result = eval_script(
+        RedisScripts::EXTEND_SCRIPT,
+        RedisScripts::EXTEND_SCRIPT_HASH,
+        keys: [key],
+        argv: [owner, ttl_ms]
+      )
+      result != 0
+    end
+
+    # Release the lock if still held by this owner. If the lock is no longer
+    # held by the owner, this method will fail and return false.
+    #
+    # @return [Boolean] whether the lock was released successfully
+    def release
+      result = eval_script(
+        RedisScripts::RELEASE_SCRIPT,
+        RedisScripts::RELEASE_SCRIPT_HASH,
+        keys: [key],
+        argv: [owner]
+      )
+      result != 0
+    end
+
+    private
+
+    def ttl_ms
+      (ttl * 1000).to_i
+    end
+
+    def eval_script(script, script_hash, keys:, argv:)
+      begin
+        redis.evalsha(script_hash, keys: keys, argv: argv)
+      rescue Redis::CommandError
+        redis.eval(script, keys: keys, argv: argv)
+      end
+    end
+  end
+end

data/lib/master_lock/redis_scripts.rb

@@ -0,0 +1,25 @@
+require 'digest/sha1'
+
+module MasterLock
+  module RedisScripts
+    RELEASE_SCRIPT = <<EOS
+if redis.call("GET", KEYS[1]) == ARGV[1]
+then
+    return redis.call("DEL", KEYS[1])
+else
+    return 0
+end
+EOS
+    RELEASE_SCRIPT_HASH = Digest::SHA1.hexdigest(RELEASE_SCRIPT)
+
+    EXTEND_SCRIPT = <<EOS
+if redis.call("GET", KEYS[1]) == ARGV[1]
+then
+    return redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[2]))
+else
+    return 0
+end
+EOS
+    EXTEND_SCRIPT_HASH = Digest::SHA1.hexdigest(EXTEND_SCRIPT)
+  end
+end

data/lib/master_lock/registry.rb

@@ -0,0 +1,86 @@
+module MasterLock
+  # When MasterLock acquires a lock, it registers it with a global registry.
+  # MasterLock will periodically renew all locks that are registered as long as
+  # the thread that acquired the lock is still alive and has not explicitly
+  # released the lock yet. If there is a failure to renew the lock, MasterLock
+  # identifies the lock as having already been released.
+  class Registry
+    Registration = Struct.new(
+      :lock,
+      :mutex,
+      :thread,
+      :acquired_at,
+      :released,
+      :extend_interval
+    )
+
+    # @return [Array<Registration>] currently registered locks
+    attr_reader :locks
+
+    def initialize
+      @locks = []
+      @locks_mutex = Mutex.new
+    end
+
+    # Register a lock to be renewed every extend_interval seconds.
+    #
+    # @param lock [#extend] a currently held lock that can be extended
+    # @param extend_interval [Fixnum] the interval in seconds after before the
+    #   lock is extended
+    # @return [Registration] the receipt of registration
+    def register(lock, extend_interval)
+      registration = Registration.new
+      registration.lock = lock
+      registration.mutex = Mutex.new
+      registration.thread = Thread.current
+      registration.acquired_at = Time.now
+      registration.extend_interval = extend_interval
+      registration.released = false
+      @locks_mutex.synchronize do
+        locks << registration
+      end
+      registration
+    end
+
+    # Unregister a lock that has been registered.
+    #
+    # @param registration [Registration] the registration returned by the call
+    #   to {#register}
+    def unregister(registration)
+      registration.mutex.synchronize do
+        registration.released = true
+      end
+    end
+
+    # Extend all currently registered locks that have been held longer than the
+    # extend_interval since they were last acquired/extended. If any locks have
+    # expired (should not happen), it will release them.
+    def extend_locks
+      # Make a local copy of the locks array to avoid accessing it outside of the mutex.
+      locks_copy = @locks_mutex.synchronize { locks.dup }
+      locks_copy.each { |registration| extend_lock(registration) }
+      @locks_mutex.synchronize do
+        locks.delete_if(&:released)
+      end
+    end
+
+    private
+
+    def extend_lock(registration)
+      registration.mutex.synchronize do
+        time = Time.now
+        if !registration.thread.alive?
+          registration.released = true
+        elsif !registration.released &&
+            registration.acquired_at + registration.extend_interval < time
+          if registration.lock.extend
+            registration.acquired_at = time
+          else
+            registration.released = true
+            # TODO: Notify of failure somehow
+          end
+        end
+      end
+    end
+  end
+end

data/lib/master_lock/version.rb

@@ -1,3 +1,3 @@
 module MasterLock
-  VERSION = "0.1.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: master_lock
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Jim Posen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-01 00:00:00.000000000 Z
+date: 2016-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -83,9 +83,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/console
-- bin/setup
 - lib/master_lock.rb
+- lib/master_lock/redis_lock.rb
+- lib/master_lock/redis_scripts.rb
+- lib/master_lock/registry.rb
 - lib/master_lock/version.rb
 - master_lock.gemspec
 homepage: https://github.com/coinbase/master_lock
@@ -108,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Inter-process locking library using Redis.

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "master_lock"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here