lock_and_cache 1.1.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 240e0a726e297cc0a7ceb14a90f64ded7a6a2717
-  data.tar.gz: 2ebadb4c71d8e9044ccceb7e5b9025d3d1c67631
+  metadata.gz: 3d82a253311d779887e8cc365b71c27004d894ee
+  data.tar.gz: 54764d5ef99783e9476406fd6c89c53ad9ae9e0f
 SHA512:
-  metadata.gz: 440b0ab4a18aadec5170d99600467353fb25a3d6f448bbc5e8628845941fa366c810637f526e49e4918be0f58729a85f5094372aec63addfc5ae9162ab13a142
-  data.tar.gz: 5ee8fd515dcfde1ad4f20c2cc4c17526a548dda99c1fa8b6160b912a5a9486d6dfdafe0b1389777f8362a362e8397090e22611b9234aab65cae4f874e640e56a
+  metadata.gz: edc54e74ebf38fb009f8211999881b7bc6e8b0addf193c0db828caf6044e63a178c89dc6802a2c1ff459bd35ff3e44531520d9ce18e37b66fe31d572e8f08aed
+  data.tar.gz: 8dc9e372a03506867174c05b52ba9e15067937388549bacc7afc7f70fccfd26b883c1d36159329e605da7dcaea022a6b0521a4d154273772e5474dea170ba184

data/.yardopts

@@ -1,2 +1,3 @@
 --no-private
 --readme README.md
+--markup-provider redcarpet

data/CHANGELOG

@@ -1,3 +1,19 @@
+2.0.0 / 2015-09-11
+
+* Breaking changes
+
+  * Stricter key digest - differentiates symbols and strings
+  * No more lock_expires or lock_spin options
+
+* Bug fixes
+
+  * Allow method names with non-word chars like #foo?
+
+* Enhancements
+
+  * heartbeats so that SIGKILL will effectively clear the lock
+  * #lock_and_cache_clear now clears lock too
+
 1.1.0 / 2015-08-07
 
 * Breaking changes

data/README.md

@@ -4,22 +4,45 @@
 [![Code Climate](https://codeclimate.com/github/seamusabshere/lock_and_cache/badges/gpa.svg)](https://codeclimate.com/github/seamusabshere/lock_and_cache)
 [![Dependency Status](https://gemnasium.com/seamusabshere/lock_and_cache.svg)](https://gemnasium.com/seamusabshere/lock_and_cache)
 [![Gem Version](https://badge.fury.io/rb/lock_and_cache.svg)](http://badge.fury.io/rb/lock_and_cache)
-[![security](https://hakiri.io/github/seamusabshere/lock_and_cache/master.svg)](https://hakiri.io/github/seamusabshere/lock_and_cache/master)
+[![Security](https://hakiri.io/github/seamusabshere/lock_and_cache/master.svg)](https://hakiri.io/github/seamusabshere/lock_and_cache/master)
 [![Inline docs](http://inch-ci.org/github/seamusabshere/lock_and_cache.svg?branch=master)](http://inch-ci.org/github/seamusabshere/lock_and_cache)
 
 Lock and cache using redis!
 
-## Redlock locking
+## Sponsor
+
+<p><a href="http://faraday.io"><img src="https://s3.amazonaws.com/photos.angel.co/startups/i/175701-a63ebd1b56a401e905963c64958204d4-medium_jpg.jpg" alt="Faraday logo"/></a></p>
+
+We use [`lock_and_cache`](https://rubygems.org/gems/lock_and_cache) for [big data-driven marketing at Faraday](http://faraday.io).
+
+## Theory
+
+`lock_and_cache`...
+
+1. returns cached value if found
+2. acquires a lock
+3. returns cached value if found (just in case it was calculated while we were waiting for a lock)
+4. calculates and caches the value
+5. releases the lock
+6. returns the value
+
+As you can see, most caching libraries only take care of (1) and (4).
+
+## Practice
+
+### Locking (antirez's Redlock)
 
 Based on [antirez's Redlock algorithm](http://redis.io/topics/distlock).
 
+Above and beyond Redlock, a 2-second heartbeat is used that will clear the lock if a process is killed. This is implemented using lock extensions.
+
 ```ruby
 LockAndCache.storage = Redis.new
 ```
 
 It will use this redis for both locking and storing cached values.
 
-## Convenient caching
+### Caching (block inside of a method)
 
 (be sure to set up storage as above)
 
@@ -52,8 +75,6 @@ end
 ## Tunables
 
 * `LockAndCache.storage=[redis]`
-* `LockAndCache.lock_expires=[seconds]` default is 3 days
-* `LockAndCache.lock_spin=[seconds]` (how long to wait before retrying lock) default is 0.1 seconds
 * `ENV['LOCK_AND_CACHE_DEBUG']='true'` if you want some debugging output on `$stderr`
 
 ## Few dependencies
@@ -61,13 +82,6 @@ end
 * [activesupport](https://rubygems.org/gems/activesupport) (come on, it's the bomb)
 * [redis](https://github.com/redis/redis-rb)
 * [redlock](https://github.com/leandromoreira/redlock-rb)
-* [hash_digest](https://github.com/seamusabshere/hash_digest) (which requires [murmurhash3](https://github.com/funny-falcon/murmurhash3-ruby))
-
-## Real-world usage
-
-<p><a href="http://faraday.io"><img src="https://s3.amazonaws.com/photos.angel.co/startups/i/175701-a63ebd1b56a401e905963c64958204d4-medium_jpg.jpg" alt="Faraday logo"/></a></p>
-
-We use [`lock_and_cache`](https://rubygems.org/gems/lock_and_cache) for [big data-driven marketing at Faraday](http://angel.co/faraday).
 
 ## Contributing
 

data/lib/lock_and_cache.rb

@@ -1,23 +1,31 @@
 require 'lock_and_cache/version'
 require 'timeout'
+require 'digest/sha1'
+require 'base64'
 require 'redis'
 require 'redlock'
-require 'hash_digest'
 require 'active_support'
 require 'active_support/core_ext'
 
+# Lock and cache methods using redis!
+#
+# I bet you're caching, but are you locking?
 module LockAndCache
-  DEFAULT_LOCK_EXPIRES = 60 * 60 * 24 * 1 * 1000 # 1 day in milliseconds
-  DEFAULT_LOCK_SPIN = 0.1
   DEFAULT_MAX_LOCK_WAIT = 60 * 60 * 24 # 1 day in seconds
 
+  # @private
+  LOCK_HEARTBEAT_EXPIRES = 2
+
+  # @private
+  LOCK_HEARTBEAT_PERIOD = 1
+
   class TimeoutWaitingForLock < StandardError; end
 
   # @param redis_connection [Redis] A redis connection to be used for lock and cached value storage
   def LockAndCache.storage=(redis_connection)
     raise "only redis for now" unless redis_connection.class.to_s == 'Redis'
     @storage = redis_connection
-    @lock_manager = Redlock::Client.new [redis_connection]
+    @lock_manager = Redlock::Client.new [redis_connection], retry_count: 1
   end
 
   # @return [Redis] The redis connection used for lock and cached value storage
@@ -32,31 +40,6 @@ module LockAndCache
     storage.flushdb
   end
 
-  # @param seconds [Numeric] Lock expiry in seconds.
-  #
-  # @note Can be overridden by putting `expires:` in your call to `#lock_and_cache`
-  def LockAndCache.lock_expires=(seconds)
-    @lock_expires = seconds.to_f * 1000
-  end
-
-  # @return [Numeric] Lock expiry in milliseconds.
-  # @private
-  def LockAndCache.lock_expires
-    @lock_expires || DEFAULT_LOCK_EXPIRES
-  end
-
-  # @param seconds [Numeric] How long to wait before trying a lock again, in seconds
-  #
-  # @note Can be overridden by putting `lock_spin:` in your call to `#lock_and_cache`
-  def LockAndCache.lock_spin=(seconds)
-    @lock_spin = seconds.to_f
-  end
-
-  # @private
-  def LockAndCache.lock_spin
-    @lock_spin || DEFAULT_LOCK_SPIN
-  end
-
   # @param seconds [Numeric] Maximum wait time to get a lock
   #
   # @note Can be overridden by putting `max_lock_wait:` in your call to `#lock_and_cache`
@@ -81,18 +64,28 @@ module LockAndCache
 
     def initialize(obj, method_id, parts)
       @obj = obj
-      @method_id = method_id
+      @method_id = method_id.to_sym
       @_parts = parts
     end
 
+    # A (non-cryptographic) digest of the key parts for use as the cache key
     def digest
-      @digest ||= ::HashDigest.digest3([obj_class_name, method_id] + parts)
+      @digest ||= ::Digest::SHA1.hexdigest ::Marshal.dump(key)
     end
 
-    def debug
-      @debug ||= [obj_class_name, method_id] + parts
+    # A (non-cryptographic) digest of the key parts for use as the lock key
+    def lock_digest
+      @lock_digest ||= 'lock/' + digest
     end
 
+    # A human-readable representation of the key parts
+    def key
+      @key ||= [obj_class_name, method_id, parts]
+    end
+
+    alias debug key
+
+    # An array of the parts we use for the key
     def parts
       @parts ||= @_parts.map do |v|
         case v
@@ -104,21 +97,26 @@ module LockAndCache
       end
     end
 
+    # An object (or its class's) name
     def obj_class_name
       @obj_class_name ||= (obj.class == ::Class) ? obj.name : obj.class.name
     end
 
   end
 
-  # Clear a cache given exactly the method and exactly the same arguments
-  #
-  # @note Does not unlock.
+  def lock_and_cache_locked?(method_id, *key_parts)
+    debug = (ENV['LOCK_AND_CACHE_DEBUG'] == 'true')
+    key = LockAndCache::Key.new self, method_id, key_parts
+    LockAndCache.storage.exists key.lock_digest
+  end
+
+  # Clear a lock and cache given exactly the method and exactly the same arguments
   def lock_and_cache_clear(method_id, *key_parts)
     debug = (ENV['LOCK_AND_CACHE_DEBUG'] == 'true')
     key = LockAndCache::Key.new self, method_id, key_parts
-    Thread.exclusive { $stderr.puts "[lock_and_cache] clear #{key.debug}" } if debug
-    digest = key.digest
-    LockAndCache.storage.del digest
+    Thread.exclusive { $stderr.puts "[lock_and_cache] clear #{key.debug} #{Base64.encode64(key.digest).strip} #{Digest::SHA1.hexdigest key.digest}" } if debug
+    LockAndCache.storage.del key.digest
+    LockAndCache.storage.del key.lock_digest
   end
 
   # Lock and cache a method given key parts.
@@ -129,43 +127,58 @@ module LockAndCache
   def lock_and_cache(*key_parts)
     raise "need a block" unless block_given?
     debug = (ENV['LOCK_AND_CACHE_DEBUG'] == 'true')
-    caller[0] =~ /in `(\w+)'/
-    method_id = $1 or raise "couldn't get method_id from #{kaller[0]}"
+    caller[0] =~ /in `([^']+)'/
+    method_id = $1 or raise "couldn't get method_id from #{caller[0]}"
     options = key_parts.last.is_a?(Hash) ? key_parts.pop.stringify_keys : {}
     expires = options['expires']
-    lock_expires = options.fetch 'lock_expires', LockAndCache.lock_expires
-    lock_spin = options.fetch 'lock_spin', LockAndCache.lock_spin
     max_lock_wait = options.fetch 'max_lock_wait', LockAndCache.max_lock_wait
     key = LockAndCache::Key.new self, method_id, key_parts
     digest = key.digest
-    storage = LockAndCache.storage
-    Thread.exclusive { $stderr.puts "[lock_and_cache] A1 #{key.debug}" } if debug
+    storage = LockAndCache.storage or raise("must set LockAndCache.storage=[Redis]")
+    Thread.exclusive { $stderr.puts "[lock_and_cache] A1 #{key.debug} #{Base64.encode64(digest).strip} #{Digest::SHA1.hexdigest digest}" } if debug
     if storage.exists digest
       return ::Marshal.load(storage.get(digest))
     end
-    Thread.exclusive { $stderr.puts "[lock_and_cache] B1 #{key.debug}" } if debug
+    Thread.exclusive { $stderr.puts "[lock_and_cache] B1 #{key.debug} #{Base64.encode64(digest).strip} #{Digest::SHA1.hexdigest digest}" } if debug
     retval = nil
     lock_manager = LockAndCache.lock_manager
-    lock_digest = 'lock/' + digest
+    lock_digest = key.lock_digest
     lock_info = nil
     begin
       Timeout.timeout(max_lock_wait, TimeoutWaitingForLock) do
-        until lock_info = lock_manager.lock(lock_digest, lock_expires)
-          Thread.exclusive { $stderr.puts "[lock_and_cache] C1 #{key.debug}" } if debug
-          sleep lock_spin
+        until lock_info = lock_manager.lock(lock_digest, LockAndCache::LOCK_HEARTBEAT_EXPIRES*1000)
+          Thread.exclusive { $stderr.puts "[lock_and_cache] C1 #{key.debug} #{Base64.encode64(digest).strip} #{Digest::SHA1.hexdigest digest}" } if debug
+          sleep rand
         end
       end
-      Thread.exclusive { $stderr.puts "[lock_and_cache] D1 #{key.debug}" } if debug
+      Thread.exclusive { $stderr.puts "[lock_and_cache] D1 #{key.debug} #{Base64.encode64(digest).strip} #{Digest::SHA1.hexdigest digest}" } if debug
       if storage.exists digest
-        Thread.exclusive { $stderr.puts "[lock_and_cache] E1 #{key.debug}" } if debug
+        Thread.exclusive { $stderr.puts "[lock_and_cache] E1 #{key.debug} #{Base64.encode64(digest).strip} #{Digest::SHA1.hexdigest digest}" } if debug
         retval = ::Marshal.load storage.get(digest)
       else
-        Thread.exclusive { $stderr.puts "[lock_and_cache] F1 #{key.debug}" } if debug
-        retval = yield
-        if expires
-          storage.setex digest, expires, ::Marshal.dump(retval)
-        else
-          storage.set digest, ::Marshal.dump(retval)
+        Thread.exclusive { $stderr.puts "[lock_and_cache] F1 #{key.debug} #{Base64.encode64(digest).strip} #{Digest::SHA1.hexdigest digest}" } if debug
+        done = false
+        begin
+          lock_extender = Thread.new do
+            loop do
+              Thread.exclusive { $stderr.puts "[lock_and_cache] heartbeat1 #{key.debug} #{Base64.encode64(digest).strip} #{Digest::SHA1.hexdigest digest}" } if debug
+              break if done
+              sleep LockAndCache::LOCK_HEARTBEAT_PERIOD
+              break if done
+              Thread.exclusive { $stderr.puts "[lock_and_cache] heartbeat2 #{key.debug} #{Base64.encode64(digest).strip} #{Digest::SHA1.hexdigest digest}" } if debug
+              lock_manager.lock lock_digest, LockAndCache::LOCK_HEARTBEAT_EXPIRES*1000, extend: lock_info
+            end
+          end
+          retval = yield
+          if expires
+            storage.setex digest, expires, ::Marshal.dump(retval)
+          else
+            storage.set digest, ::Marshal.dump(retval)
+          end
+        ensure
+          done = true
+          lock_extender.exit if lock_extender.alive?
+          lock_extender.join if lock_extender.status.nil?
         end
       end
     ensure

data/lib/lock_and_cache/version.rb

@@ -1,3 +1,3 @@
 module LockAndCache
-  VERSION = '1.1.0'
+  VERSION = '2.0.0'
 end

data/lock_and_cache.gemspec

@@ -19,9 +19,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'activesupport'
-  spec.add_runtime_dependency 'hash_digest'
   spec.add_runtime_dependency 'redis'
-  spec.add_runtime_dependency 'redlock'
+  # temporary until https://github.com/leandromoreira/redlock-rb/pull/20 is merged
+  spec.add_runtime_dependency 'seamusabshere-redlock'
 
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'bundler', '~> 1.6'
@@ -29,4 +29,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'thread'
   spec.add_development_dependency 'yard'
+  spec.add_development_dependency 'redcarpet'
 end

data/spec/lock_and_cache_spec.rb

@@ -65,6 +65,24 @@ class Bar
   end
 end
 
+class Sleeper
+  include LockAndCache
+
+  def initialize
+    @id = SecureRandom.hex
+  end
+
+  def poke
+    lock_and_cache do
+      sleep
+    end
+  end
+
+  def lock_and_cache_key
+    @id
+  end
+end
+
 describe LockAndCache do
   before do
     LockAndCache.flush
@@ -168,6 +186,59 @@ describe LockAndCache do
       end
     end
 
+    it 'unlocks if a process dies' do
+      child = nil
+      begin
+        sleeper = Sleeper.new
+        child = fork do
+          sleeper.poke
+        end
+        sleep 0.1
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true)  # the other process has it
+        Process.kill 'KILL', child
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true)  # the other (dead) process still has it
+        sleep 2
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(false) # but now it should be cleared because no heartbeat
+      ensure
+        Process.kill('KILL', child) rescue Errno::ESRCH
+      end
+    end
+
+    it "pays attention to heartbeats" do
+      child = nil
+      begin
+        sleeper = Sleeper.new
+        child = fork do
+          sleeper.poke
+        end
+        sleep 0.1
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true) # the other process has it
+        sleep 2
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true) # the other process still has it
+        sleep 2
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true) # the other process still has it
+        sleep 2
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true) # the other process still has it
+      ensure
+        Process.kill('TERM', child) rescue Errno::ESRCH
+      end
+    end
+
   end
 
+  describe 'keying' do
+    it "doesn't conflate symbol and string args" do
+      symbol = LockAndCache::Key.new(Foo.new(:me), :click, a: 1)
+      string = LockAndCache::Key.new(Foo.new(:me), :click, 'a' => 1)
+      expect(symbol.digest).not_to eq(string.digest)
+    end
+
+    it "cares about order" do
+      symbol = LockAndCache::Key.new(Foo.new(:me), :click, {a: 1, b: 2})
+      string = LockAndCache::Key.new(Foo.new(:me), :click, {b: 2, a: 1})
+      expect(symbol.digest).not_to eq(string.digest)
+    end
+  end
+
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lock_and_cache
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Seamus Abshere
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-08 00:00:00.000000000 Z
+date: 2015-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: hash_digest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -53,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redlock
+  name: seamusabshere-redlock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -150,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Lock and cache methods, in case things should only be calculated once
   across processes.
 email: