suo 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef6a0837cc8be9a03a3767df546c630adf1e74cf
-  data.tar.gz: d2fefd52a16706479a3f62b4a89557ece8e74672
+  metadata.gz: c3c678d658e34c698d95ca20d8d7ddfea5c11c6c
+  data.tar.gz: cb7bf87ef58efff50356c9310adc1fb10fad6ad1
 SHA512:
-  metadata.gz: 4dc9fa8f2377e19efce5d6d4d009a145214360fd51efac442aaabcd9ba9077e49bdbad7828304c9aab9a273ad76b017c6fadce7bccf83c1d0897d31c63a0333d
-  data.tar.gz: a0de3ab597fc8d61fe6f47f2b239aeaa11baf9d6cde481481840f96a8f673150a9d10db13e53796cd552cafceb01fbedffaa4160228686b6396335444dbf64e5
+  metadata.gz: c33bc02743629ab053afa32ff38e0c837cc1b2deacdbb50342e14de50c83321f7d232df4e2dd2338ee8ec16e54b0aa7cc81b97adc67bd86d75c597c06822be49
+  data.tar.gz: ef2790ad5ad8ac1567346fd04b340288ec8ea6924a267d8b9ea4daea02808b6c665b267bbaee61c90c544bfeb4de3a6953100d12e36d70365ee505765b73c833

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.2.3
+
+- Clarify documentation further with respect to semaphores.
+
+## 0.2.2
+
+- Fix bug with refresh - typo would've prevented real use.
+- Clean up code.
+- Improve documentation a bit.
+- 100% test coverage.
+
 ## 0.2.1
 
 - Fix bug when dealing with real-world Redis error conditions.
@@ -18,7 +29,7 @@
 
 ## 0.1.1
 
-- Use [MessagePack](https://github.com/msgpack/msgpack-ruby) for semaphore serialization.
+- Use [MessagePack](https://github.com/msgpack/msgpack-ruby) for lock serialization.
 
 ## 0.1.0
 

data/README.md

@@ -1,8 +1,8 @@
-# Suo [![Build Status](https://travis-ci.org/nickelser/suo.svg?branch=master)](https://travis-ci.org/nickelser/suo) [![Gem Version](https://badge.fury.io/rb/suo.svg)](http://badge.fury.io/rb/suo)
+# Suo [![Build Status](https://travis-ci.org/nickelser/suo.svg?branch=master)](https://travis-ci.org/nickelser/suo) [![Code Climate](https://codeclimate.com/github/nickelser/suo/badges/gpa.svg)](https://codeclimate.com/github/nickelser/suo) [![Test Coverage](https://codeclimate.com/github/nickelser/suo/badges/coverage.svg)](https://codeclimate.com/github/nickelser/suo) [![Gem Version](https://badge.fury.io/rb/suo.svg)](http://badge.fury.io/rb/suo)
 
-:lock: Distributed semaphores using Memcached or Redis in Ruby.
+:lock: Distributed locks using Memcached or Redis in Ruby.
 
-Suo provides a very performant distributed lock solution using Compare-And-Set (`CAS`) commands in Memcached, and `WATCH/MULTI` in Redis.
+Suo provides a very performant distributed lock solution using Compare-And-Set (`CAS`) commands in Memcached, and `WATCH/MULTI` in Redis. It allows locking both single exclusion (a mutex - sharing one resource), and multiple resources.
 
 ## Installation
 
@@ -31,6 +31,7 @@ suo.lock("some_key") do
   @puppies.pet!
 end
 
+# The second argument to lock is the number of arguments (defaulting to one - a mutex)
 Thread.new { suo.lock("other_key", 2) { puts "One"; sleep 2 } }
 Thread.new { suo.lock("other_key", 2) { puts "Two"; sleep 2 } }
 Thread.new { suo.lock("other_key", 2) { puts "Three" } }
@@ -41,16 +42,39 @@ Thread.new { suo.lock("other_key", 2) { puts "Three" } }
 suo = Suo::Client::Memcached.new(client: some_dalli_client, acquisition_timeout: 1) # in seconds
 
 # manually locking/unlocking
-suo.lock("a_key")
+# the return value from lock without a block is a unique token valid only for the current lock
+# which must be unlocked manually
+lock = suo.lock("a_key")
 foo.baz!
-suo.unlock("a_key")
+suo.unlock("a_key", lock)
 
-# custom stale lock cleanup (cleaning of dead clients)
+# custom stale lock expiration (cleaning of dead locks)
 suo = Suo::Client::Redis.new(client: some_redis_client, stale_lock_expiration: 60*5)
 ```
 
+### Stale locks
+
+"Stale locks" - those acquired more than `stale_lock_expiration` (defaulting to 3600 or one hour) ago - are automatically cleared during any operation on the key (`lock`, `unlock`, `refresh`). The `locked?` method will not return true if only stale locks exist, but will not modify the key itself.
+
+To re-acquire a lock in the middle of a block, you can use the refresh method on client.
+
+```ruby
+suo = Suo::Client::Redis.new
+
+# lock is the same token as seen in the manual example, above
+suo.lock("foo") do |lock|
+  5.times do
+    baz.bar!
+    suo.refresh("foo", lock)
+  end
+end
+```
+
+## Semaphore
+
+With multiple resources, Suo acts like a semaphore, but is not strictly a semaphore according to the traditional definition, as the lock acquires ownership.
+
 ## TODO
- - better stale key handling (refresh blocks)
  - more race condition tests
 
 ## History

data/lib/suo/client/base.rb

@@ -24,7 +24,7 @@ module Suo
 
         if block_given? && token
           begin
-            yield
+            yield(token)
           ensure
             unlock(key, token)
           end
@@ -39,9 +39,9 @@ module Suo
 
       def locks(key)
         val, _ = get(key)
-        locks = deserialize_locks(val)
+        cleared_locks = deserialize_and_clear_locks(val)
 
-        locks
+        cleared_locks
       end
 
       def refresh(key, acquisition_token)
@@ -53,11 +53,11 @@ module Suo
             next
           end
 
-          locks = deserialize_and_clear_locks(val)
+          cleared_locks = deserialize_and_clear_locks(val)
 
-          refresh_lock(locks, acquisition_token)
+          refresh_lock(cleared_locks, acquisition_token)
 
-          break if set(key, serialize_locks(locks), cas)
+          break if set(key, serialize_locks(cleared_locks), cas)
         end
       end
 
@@ -69,12 +69,12 @@ module Suo
 
           break if val.nil?
 
-          locks = deserialize_and_clear_locks(val)
+          cleared_locks = deserialize_and_clear_locks(val)
 
-          acquisition_lock = remove_lock(locks, acquisition_token)
+          acquisition_lock = remove_lock(cleared_locks, acquisition_token)
 
           break unless acquisition_lock
-          break if set(key, serialize_locks(locks), cas)
+          break if set(key, serialize_locks(cleared_locks), cas)
         end
       rescue LockClientError => _ # rubocop:disable Lint/HandleExceptions
         # ignore - assume success due to optimistic locking
@@ -87,7 +87,6 @@ module Suo
       private
 
       def acquire_lock(key, resources = 1)
-        acquisition_token = nil
         token = SecureRandom.base64(16)
 
         retry_with_timeout(key) do
@@ -98,32 +97,29 @@ module Suo
             next
           end
 
-          locks = deserialize_and_clear_locks(val)
+          cleared_locks = deserialize_and_clear_locks(val)
 
-          if locks.size < resources
-            add_lock(locks, token)
+          if cleared_locks.size < resources
+            add_lock(cleared_locks, token)
 
-            newval = serialize_locks(locks)
+            newval = serialize_locks(cleared_locks)
 
-            if set(key, newval, cas)
-              acquisition_token = token
-              break
-            end
+            return token if set(key, newval, cas)
           end
         end
 
-        acquisition_token
+        nil
       end
 
       def get(key) # rubocop:disable Lint/UnusedMethodArgument
         fail NotImplementedError
       end
 
-      def set(key, newval, oldval) # rubocop:disable Lint/UnusedMethodArgument
+      def set(key, newval, cas) # rubocop:disable Lint/UnusedMethodArgument
         fail NotImplementedError
       end
 
-      def initial_set(key) # rubocop:disable Lint/UnusedMethodArgument
+      def initial_set(key, val = "") # rubocop:disable Lint/UnusedMethodArgument
         fail NotImplementedError
       end
 
@@ -135,8 +131,8 @@ module Suo
         start = Time.now.to_f
 
         @retry_count.times do
-          now = Time.now.to_f
-          break if now - start > @options[:acquisition_timeout]
+          elapsed = Time.now.to_f - start
+          break if elapsed >= @options[:acquisition_timeout]
 
           synchronize(key) do
             yield
@@ -162,7 +158,7 @@ module Suo
         unpacked.map do |time, token|
           [Time.at(time), token]
         end
-      rescue EOFError => _
+      rescue EOFError, MessagePack::MalformedFormatError => _
         []
       end
 
@@ -171,8 +167,8 @@ module Suo
         locks.reject { |time, _| time < expired }
       end
 
-      def add_lock(locks, token)
-        locks << [Time.now.to_f, token]
+      def add_lock(locks, token, time = Time.now.to_f)
+        locks << [time, token]
       end
 
       def remove_lock(locks, acquisition_token)
@@ -182,7 +178,7 @@ module Suo
 
       def refresh_lock(locks, acquisition_token)
         remove_lock(locks, acquisition_token)
-        add_lock(locks, token)
+        add_lock(locks, acquisition_token)
       end
     end
   end

data/lib/suo/client/memcached.rb

@@ -20,8 +20,8 @@ module Suo
         @client.set_cas(key, newval, cas)
       end
 
-      def initial_set(key)
-        @client.set(key, "")
+      def initial_set(key, val = "")
+        @client.set(key, val)
       end
     end
   end

data/lib/suo/client/redis.rb

@@ -32,8 +32,8 @@ module Suo
         @client.unwatch
       end
 
-      def initial_set(key)
-        @client.set(key, "")
+      def initial_set(key, val = "")
+        @client.set(key, val)
       end
     end
   end

data/lib/suo/version.rb

@@ -1,3 +1,3 @@
 module Suo
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 end

data/suo.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Nick Elser"]
   spec.email         = ["nick.elser@gmail.com"]
 
-  spec.summary       = %q(Distributed semaphores using Memcached or Redis.)
-  spec.description   = %q(Distributed semaphores using Memcached or Redis.)
+  spec.summary       = %q(Distributed locks using Memcached or Redis.)
+  spec.description   = %q(Distributed locks using Memcached or Redis.)
   spec.homepage      = "https://github.com/nickelser/suo"
   spec.license       = "MIT"
 
@@ -30,4 +30,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rubocop", "~> 0.30.0"
   spec.add_development_dependency "minitest", "~> 5.5.0"
+  spec.add_development_dependency "codeclimate-test-reporter", "~> 0.4.7"
 end

data/test/client_test.rb

@@ -3,6 +3,10 @@ require "test_helper"
 TEST_KEY = "suo_test_key".freeze
 
 module ClientTests
+  def client(options = {})
+    @client.class.new(options.merge(client: @client.client))
+  end
+
   def test_throws_failed_error_on_bad_client
     assert_raises(Suo::LockClientError) do
       client = @client.class.new(client: {})
@@ -27,6 +31,23 @@ module ClientTests
     assert_equal false, locked
   end
 
+  def test_empty_lock_on_invalid_data
+    @client.send(:initial_set, TEST_KEY, "bad value")
+    locked = @client.locked?(TEST_KEY)
+    assert_equal false, locked
+  end
+
+  def test_clear
+    lock1 = @client.lock(TEST_KEY, 1)
+    refute_nil lock1
+
+    @client.clear(TEST_KEY)
+
+    locked = @client.locked?(TEST_KEY, 1)
+
+    assert_equal false, locked
+  end
+
   def test_multiple_resource_locking
     lock1 = @client.lock(TEST_KEY, 2)
     refute_nil lock1
@@ -77,7 +98,7 @@ module ClientTests
     sleep 0.1
     threads << Thread.new { @client.lock(TEST_KEY, 2) { output << "Three" } }
 
-    threads.map(&:join)
+    threads.each(&:join)
 
     ret = []
 
@@ -88,13 +109,14 @@ module ClientTests
 
     assert_equal 0, output.size
     assert_equal %w(One Two), ret
+    assert_equal false, @client.locked?(TEST_KEY)
   end
 
   def test_block_multiple_resource_locking
     success_counter = Queue.new
     failure_counter = Queue.new
 
-    client = @client.class.new(acquisition_timeout: 0.9, client: @client.client)
+    client = client(acquisition_timeout: 0.9)
 
     100.times.map do |i|
       Thread.new do
@@ -105,17 +127,18 @@ module ClientTests
 
         failure_counter << i unless success
       end
-    end.map(&:join)
+    end.each(&:join)
 
     assert_equal 50, success_counter.size
     assert_equal 50, failure_counter.size
+    assert_equal false, client.locked?(TEST_KEY)
   end
 
   def test_block_multiple_resource_locking_longer_timeout
     success_counter = Queue.new
     failure_counter = Queue.new
 
-    client = @client.class.new(acquisition_timeout: 3, client: @client.client)
+    client = client(acquisition_timeout: 3)
 
     100.times.map do |i|
       Thread.new do
@@ -126,10 +149,187 @@ module ClientTests
 
         failure_counter << i unless success
       end
-    end.map(&:join)
+    end.each(&:join)
 
     assert_equal 100, success_counter.size
     assert_equal 0, failure_counter.size
+    assert_equal false, client.locked?(TEST_KEY)
+  end
+
+  def test_unstale_lock_acquisition
+    success_counter = Queue.new
+    failure_counter = Queue.new
+
+    client = client(stale_lock_expiration: 0.5)
+
+    t1 = Thread.new { client.lock(TEST_KEY) { sleep 0.6; success_counter << 1 } }
+    sleep 0.3
+    t2 = Thread.new do
+      locked = client.lock(TEST_KEY) { success_counter << 1 }
+      failure_counter << 1 unless locked
+    end
+
+    [t1, t2].each(&:join)
+
+    assert_equal 1, success_counter.size
+    assert_equal 1, failure_counter.size
+    assert_equal false, client.locked?(TEST_KEY)
+  end
+
+  def test_stale_lock_acquisition
+    success_counter = Queue.new
+    failure_counter = Queue.new
+
+    client = client(stale_lock_expiration: 0.5)
+
+    t1 = Thread.new { client.lock(TEST_KEY) { sleep 0.6; success_counter << 1 } }
+    sleep 0.55
+    t2 = Thread.new do
+      locked = client.lock(TEST_KEY) { success_counter << 1 }
+      failure_counter << 1 unless locked
+    end
+
+    [t1, t2].each(&:join)
+
+    assert_equal 2, success_counter.size
+    assert_equal 0, failure_counter.size
+    assert_equal false, client.locked?(TEST_KEY)
+  end
+
+  def test_refresh
+    client = client(stale_lock_expiration: 0.5)
+
+    lock1 = client.lock(TEST_KEY)
+
+    assert_equal true, client.locked?(TEST_KEY)
+
+    client.refresh(TEST_KEY, lock1)
+
+    assert_equal true, client.locked?(TEST_KEY)
+
+    sleep 0.55
+
+    assert_equal false, client.locked?(TEST_KEY)
+
+    lock2 = client.lock(TEST_KEY)
+
+    client.refresh(TEST_KEY, lock1)
+
+    assert_equal true, client.locked?(TEST_KEY)
+
+    client.unlock(TEST_KEY, lock1)
+
+    # edge case with refresh lock in the middle
+    assert_equal true, client.locked?(TEST_KEY)
+
+    client.clear(TEST_KEY)
+
+    assert_equal false, client.locked?(TEST_KEY)
+
+    client.refresh(TEST_KEY, lock2)
+
+    assert_equal true, client.locked?(TEST_KEY)
+
+    client.unlock(TEST_KEY, lock2)
+
+    # now finally unlocked
+    assert_equal false, client.locked?(TEST_KEY)
+  end
+
+  def test_block_refresh
+    success_counter = Queue.new
+    failure_counter = Queue.new
+
+    client = client(stale_lock_expiration: 0.5)
+
+    t1 = Thread.new do
+      client.lock(TEST_KEY) do |token|
+        sleep 0.6
+        client.refresh(TEST_KEY, token)
+        sleep 1
+        success_counter << 1
+      end
+    end
+
+    t2 = Thread.new do
+      sleep 0.8
+      locked = client.lock(TEST_KEY) { success_counter << 1 }
+      failure_counter << 1 unless locked
+    end
+
+    [t1, t2].each(&:join)
+
+    assert_equal 1, success_counter.size
+    assert_equal 1, failure_counter.size
+    assert_equal false, client.locked?(TEST_KEY)
+  end
+
+  def test_refresh_multi
+    success_counter = Queue.new
+    failure_counter = Queue.new
+
+    client = client(stale_lock_expiration: 0.5)
+
+    t1 = Thread.new do
+      client.lock(TEST_KEY, 2) do |token|
+        sleep 0.4
+        client.refresh(TEST_KEY, token)
+        success_counter << 1
+        sleep 0.5
+      end
+    end
+
+    t2 = Thread.new do
+      sleep 0.55
+      locked = client.lock(TEST_KEY, 2) do
+        success_counter << 1
+        sleep 0.5
+      end
+
+      failure_counter << 1 unless locked
+    end
+
+    t3 = Thread.new do
+      sleep 0.75
+      locked = client.lock(TEST_KEY, 2) { success_counter << 1 }
+      failure_counter << 1 unless locked
+    end
+
+    [t1, t2, t3].each(&:join)
+
+    assert_equal 2, success_counter.size
+    assert_equal 1, failure_counter.size
+    assert_equal false, client.locked?(TEST_KEY)
+  end
+
+  def test_increment_reused_client
+    i = 0
+
+    threads = 2.times.map do
+      Thread.new do
+        @client.lock(TEST_KEY) { i += 1 }
+      end
+    end
+
+    threads.each(&:join)
+
+    assert_equal 2, i
+    assert_equal false, client.locked?(TEST_KEY)
+  end
+
+  def test_increment_new_client
+    i = 0
+
+    threads = 2.times.map do
+      Thread.new do
+        client.lock(TEST_KEY) { i += 1 }
+      end
+    end
+
+    threads.each(&:join)
+
+    assert_equal 2, i
+    assert_equal false, client.locked?(TEST_KEY)
   end
 end
 
@@ -142,6 +342,18 @@ class TestBaseClient < Minitest::Test
     assert_raises(NotImplementedError) do
       @client.send(:get, TEST_KEY)
     end
+
+    assert_raises(NotImplementedError) do
+      @client.send(:set, TEST_KEY, "", "")
+    end
+
+    assert_raises(NotImplementedError) do
+      @client.send(:initial_set, TEST_KEY)
+    end
+
+    assert_raises(NotImplementedError) do
+      @client.send(:clear, TEST_KEY)
+    end
   end
 end
 
@@ -151,6 +363,7 @@ class TestMemcachedClient < Minitest::Test
   def setup
     @dalli = Dalli::Client.new("127.0.0.1:11211")
     @client = Suo::Client::Memcached.new
+    teardown
   end
 
   def teardown
@@ -164,6 +377,7 @@ class TestRedisClient < Minitest::Test
   def setup
     @redis = Redis.new
     @client = Suo::Client::Redis.new
+    teardown
   end
 
   def teardown

data/test/test_helper.rb

@@ -1,5 +1,10 @@
 $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
 
+if ENV["CODECLIMATE_REPO_TOKEN"]
+  require "codeclimate-test-reporter"
+  CodeClimate::TestReporter.start
+end
+
 require "suo"
 require "thread"
 require "minitest/autorun"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suo
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Nick Elser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-13 00:00:00.000000000 Z
+date: 2015-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dalli
@@ -108,7 +108,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 5.5.0
-description: Distributed semaphores using Memcached or Redis.
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.7
+description: Distributed locks using Memcached or Redis.
 email:
 - nick.elser@gmail.com
 executables:
@@ -159,7 +173,7 @@ rubyforge_project:
 rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
-summary: Distributed semaphores using Memcached or Redis.
+summary: Distributed locks using Memcached or Redis.
 test_files:
 - test/client_test.rb
 - test/test_helper.rb