lock_and_cache_msgpack 4.0.7.pre1

data/lib/lock_and_cache_msgpack/version.rb

@@ -0,0 +1,3 @@
+module LockAndCacheMsgpack
+  VERSION = '4.0.7.pre1'
+end

data/lib/messagepack_ext.rb

@@ -0,0 +1,16 @@
+class Time
+
+  def to_msgpack(packer=nil)
+    packer ||= MessagePack::Packer.new
+    packer.pack self
+  end
+
+end
+
+MessagePack::DefaultFactory.register_type(0x00, Symbol)
+
+MessagePack::DefaultFactory.register_type(0x01, Time,
+  packer: ->(t){ t.iso8601.to_msgpack },
+  unpacker: ->(d){
+    Time.parse(MessagePack.unpack(d.force_encoding("ASCII-8BIT")))
+  })

data/lock_and_cache_msgpack.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'lock_and_cache_msgpack/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "lock_and_cache_msgpack"
+  spec.version       = LockAndCacheMsgpack::VERSION
+  spec.authors       = ["Seamus Abshere", "Matt E. Patterson"]
+  spec.email         = ["seamus@abshere.net", "mpatterson@skillsengine.com"]
+  spec.summary       = %q{Lock and cache methods, with MessagePack.}
+  spec.description   = %q{Lock and cache methods, in case things should only be calculated once across processes.}
+  spec.homepage      = "https://github.com/c4eo/lock_and_cache_msgpack"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'activesupport'
+  spec.add_runtime_dependency 'redis'
+  # temporary until https://github.com/leandromoreira/redlock-rb/pull/20 is merged
+  spec.add_runtime_dependency 'redlock', '>=0.1.3'
+  spec.add_runtime_dependency 'msgpack', '~> 1.1.0'
+
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'thread'
+  spec.add_development_dependency 'yard'
+  spec.add_development_dependency 'redcarpet'
+end

data/spec/lock_and_cache/key_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+class KeyTestId
+  def id
+    'id'
+  end
+end
+class KeyTestLockAndCacheMsgpackKey
+  def lock_and_cache_key
+    'lock_and_cache_key'
+  end
+end
+class KeyTest1
+  def lock_and_cache_key
+    KeyTestLockAndCacheMsgpackKey.new
+  end
+end
+describe LockAndCacheMsgpack::Key do
+  describe 'parts' do
+    it "has a known issue differentiating between {a: 1} and [[:a, 1]]" do
+      expect(described_class.new(a: 1).send(:parts)).to eq(described_class.new([[:a, 1]]).send(:parts))
+    end
+    
+    now = Time.now
+    today = Date.today
+    {
+      [1]                                                => [1],
+      ['you']                                            => ['you'],
+      [['you']]                                          => [['you']],
+      [['you'], "person"]                                => [['you'], "person"],
+      [['you'], {:silly=>:person}]                       => [['you'], [[:silly, :person]] ],
+      [now]                                              => [now.to_s],
+      [[now]]                                            => [[now.to_s]],
+      [today]                                            => [today.to_s],
+      [[today]]                                          => [[today.to_s]],
+      { hi: 'you' }                                      => [[:hi, 'you']],
+      { hi: 123 }                                        => [[:hi, 123]],
+      { hi: 123.0 }                                      => [[:hi, 123.0]],
+      { hi: now }                                        => [[:hi, now.to_s]],
+      { hi: today }                                      => [[:hi, today.to_s]],
+      [KeyTestId.new]                                    => ['id'],
+      [[KeyTestId.new]]                                  => [['id']],
+      { a: KeyTestId.new }                               => [[:a, "id"]],
+      [{ a: KeyTestId.new }]                             => [[[:a, "id"]]],
+      [[{ a: KeyTestId.new }]]                           => [[ [[:a, "id"]] ]],
+      [[{ a: [ KeyTestId.new ] }]]                       => [[[[:a, ["id"]]]]],
+      [[{ a: { b: KeyTestId.new } }]]                    => [[ [[ :a, [[:b, "id"]] ]] ]],
+      [[{ a: { b: [ KeyTestId.new ] } }]]                => [[ [[ :a, [[:b, ["id"]]] ]] ]],
+      [KeyTestLockAndCacheMsgpackKey.new]                       => ['lock_and_cache_key'],
+      [[KeyTestLockAndCacheMsgpackKey.new]]                     => [['lock_and_cache_key']],
+      { a: KeyTestLockAndCacheMsgpackKey.new }                  => [[:a, "lock_and_cache_key"]],
+      [{ a: KeyTestLockAndCacheMsgpackKey.new }]                => [[[:a, "lock_and_cache_key"]]],
+      [[{ a: KeyTestLockAndCacheMsgpackKey.new }]]              => [[ [[:a, "lock_and_cache_key"]] ]],
+      [[{ a: [ KeyTestLockAndCacheMsgpackKey.new ] }]]          => [[[[:a, ["lock_and_cache_key"]]]]],
+      [[{ a: { b: KeyTestLockAndCacheMsgpackKey.new } }]]       => [[ [[ :a, [[:b, "lock_and_cache_key"]] ]] ]],
+      [[{ a: { b: [ KeyTestLockAndCacheMsgpackKey.new ] } }]]   => [[ [[ :a, [[:b, ["lock_and_cache_key"]]] ]] ]],
+
+      [[{ a: { b: [ KeyTest1.new ] } }]]   => [[ [[ :a, [[:b, ["lock_and_cache_key"]]] ]] ]],
+    }.each do |i, o|
+      it "turns #{i} into #{o}" do
+        expect(described_class.new(i).send(:parts)).to eq(o)
+      end
+    end
+  end
+end

data/spec/lock_and_cache_spec.rb

@@ -0,0 +1,476 @@
+require 'spec_helper'
+
+class Foo
+  include LockAndCacheMsgpack
+
+  def initialize(id)
+    @id = id
+    @count = 0
+    @count_exp = 0
+    @click_single_hash_arg_as_options = 0
+    @click_last_hash_as_options = 0
+  end
+
+  def click
+    lock_and_cache do
+      @count += 1
+    end
+  end
+
+  def cached_rand
+    lock_and_cache do
+      rand
+    end
+  end
+
+  def click_null
+    lock_and_cache do
+      nil
+    end
+  end
+
+  def click_exp
+    lock_and_cache(expires: 1) do
+      @count_exp += 1
+    end
+  end
+
+  # foo will be treated as option, so this is cacheable
+  def click_single_hash_arg_as_options
+    lock_and_cache(foo: rand, expires: 1) do
+      @click_single_hash_arg_as_options += 1
+    end
+  end
+
+  # foo will be treated as part of cache key, so this is uncacheable
+  def click_last_hash_as_options
+    lock_and_cache({foo: rand}, expires: 1) do
+      @click_last_hash_as_options += 1
+    end
+  end
+
+  def lock_and_cache_key
+    @id
+  end
+end
+
+class FooId
+  include LockAndCacheMsgpack
+  def click
+    lock_and_cache do
+      nil
+    end
+  end
+  def id
+    @id ||= rand
+  end
+end
+
+class FooClass
+  class << self
+    include LockAndCacheMsgpack
+    def click
+      lock_and_cache do
+        nil
+      end
+    end
+    def id
+      raise "called id"
+    end
+  end
+end
+
+require 'set'
+$clicking = Set.new
+class Bar
+  include LockAndCacheMsgpack
+
+  def initialize(id)
+    @id = id
+    @count = 0
+    @mutex = Mutex.new
+  end
+
+  def unsafe_click
+    @mutex.synchronize do
+      # puts "clicking bar #{@id} - #{$clicking.to_a} - #{$clicking.include?(@id)} - #{@id == $clicking.to_a[0]}"
+      raise "somebody already clicking Bar #{@id}" if $clicking.include?(@id)
+      $clicking << @id
+    end
+    sleep 1
+    @count += 1
+    $clicking.delete @id
+    @count
+  end
+
+  def click
+    lock_and_cache do
+      unsafe_click
+    end
+  end
+
+  def slow_click
+    lock_and_cache do
+      sleep 1
+    end
+  end
+
+  def lock_and_cache_key
+    @id
+  end
+end
+
+class Sleeper
+  include LockAndCacheMsgpack
+
+  def initialize
+    @id = SecureRandom.hex
+  end
+
+  def poke
+    lock_and_cache heartbeat_expires: 2 do
+      sleep
+    end
+  end
+
+  def lock_and_cache_key
+    @id
+  end
+end
+
+describe LockAndCacheMsgpack do
+  before do
+    LockAndCacheMsgpack.flush
+  end
+
+  it 'has a version number' do
+    expect(LockAndCacheMsgpack::VERSION).not_to be nil
+  end
+
+  describe "caching" do
+    let(:foo) { Foo.new(rand.to_s) }
+    it "works" do
+      expect(foo.click).to eq(1)
+      expect(foo.click).to eq(1)
+    end
+
+    it "can be cleared" do
+      expect(foo.click).to eq(1)
+      foo.lock_and_cache_clear :click
+      expect(foo.click).to eq(2)
+    end
+
+    it "can be expired" do
+      expect(foo.click_exp).to eq(1)
+      expect(foo.click_exp).to eq(1)
+      sleep 1.5
+      expect(foo.click_exp).to eq(2)
+    end
+
+    it "can cache null" do
+      expect(foo.click_null).to eq(nil)
+      expect(foo.click_null).to eq(nil)
+    end
+
+    it "treats single hash arg as options" do
+      expect(foo.click_single_hash_arg_as_options).to eq(1)
+      expect(foo.click_single_hash_arg_as_options).to eq(1)
+      sleep 1.1
+      expect(foo.click_single_hash_arg_as_options).to eq(2)
+    end
+
+    it "treats last hash as options" do
+      expect(foo.click_last_hash_as_options).to eq(1)
+      expect(foo.click_last_hash_as_options).to eq(2) # it's uncacheable to prove we're not using as part of options
+      expect(foo.click_last_hash_as_options).to eq(3)
+    end
+
+    it "calls #lock_and_cache_key" do
+      expect(foo).to receive(:lock_and_cache_key)
+      foo.click
+    end
+
+    it "calls #lock_and_cache_key to differentiate" do
+      a = Foo.new 1
+      b = Foo.new 2
+      expect(a.cached_rand).not_to eq(b.cached_rand)
+    end
+  end
+
+  describe 'self-identification in context mode' do
+    it "calls #id for non-class" do
+      foo_id = FooId.new
+      expect(foo_id).to receive(:id)
+      foo_id.click
+    end
+    it "calls class name for non-class" do
+      foo_id = FooId.new
+      expect(FooId).to receive(:name)
+      foo_id.click
+    end
+    it "uses class name for class" do
+      expect(FooClass).to receive(:name)
+      expect(FooClass).not_to receive(:id)
+      FooClass.click
+    end
+  end
+
+  describe "locking" do
+    let(:bar) { Bar.new(rand.to_s) }
+
+    it "it blows up normally (simple thread)" do
+      a = Thread.new do
+        bar.unsafe_click
+      end
+      b = Thread.new do
+        bar.unsafe_click
+      end
+      expect do
+        a.join
+        b.join
+      end.to raise_error(/somebody/)
+    end
+
+    it "it blows up (pre-existing thread pool, more reliable)" do
+      pool = Thread.pool 2
+      Thread::Pool.abort_on_exception = true
+      expect do
+        pool.process do
+          bar.unsafe_click
+        end
+        pool.process do
+          bar.unsafe_click
+        end
+        pool.shutdown
+      end.to raise_error(/somebody/)
+    end
+
+    it "doesn't blow up if you lock it (simple thread)" do
+      a = Thread.new do
+        bar.click
+      end
+      b = Thread.new do
+        bar.click
+      end
+      a.join
+      b.join
+    end
+
+    it "doesn't blow up if you lock it (pre-existing thread pool, more reliable)" do
+      pool = Thread.pool 2
+      Thread::Pool.abort_on_exception = true
+      pool.process do
+        bar.click
+      end
+      pool.process do
+        bar.click
+      end
+      pool.shutdown
+    end
+
+    it "can set a wait time" do
+      pool = Thread.pool 2
+      Thread::Pool.abort_on_exception = true
+      begin
+        old_max = LockAndCacheMsgpack.max_lock_wait
+        LockAndCacheMsgpack.max_lock_wait = 0.5
+        expect do
+          pool.process do
+            bar.slow_click
+          end
+          pool.process do
+            bar.slow_click
+          end
+          pool.shutdown
+        end.to raise_error(LockAndCacheMsgpack::TimeoutWaitingForLock)
+      ensure
+        LockAndCacheMsgpack.max_lock_wait = old_max
+      end
+    end
+
+    it 'unlocks if a process dies' do
+      child = nil
+      begin
+        sleeper = Sleeper.new
+        child = fork do
+          sleeper.poke
+        end
+        sleep 0.1
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true)  # the other process has it
+        Process.kill 'KILL', child
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true)  # the other (dead) process still has it
+        sleep 2
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(false) # but now it should be cleared because no heartbeat
+      ensure
+        Process.kill('KILL', child) rescue Errno::ESRCH
+      end
+    end
+
+    it "pays attention to heartbeats" do
+      child = nil
+      begin
+        sleeper = Sleeper.new
+        child = fork do
+          sleeper.poke
+        end
+        sleep 0.1
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true) # the other process has it
+        sleep 2
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true) # the other process still has it
+        sleep 2
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true) # the other process still has it
+        sleep 2
+        expect(sleeper.lock_and_cache_locked?(:poke)).to eq(true) # the other process still has it
+      ensure
+        Process.kill('TERM', child) rescue Errno::ESRCH
+      end
+    end
+
+  end
+
+  describe 'standalone' do
+    it 'works like you expect' do
+      count = 0
+      expect(LockAndCacheMsgpack.lock_and_cache('hello') { count += 1 }).to eq(1)
+      expect(count).to eq(1)
+      expect(LockAndCacheMsgpack.lock_and_cache('hello') { count += 1 }).to eq(1)
+      expect(count).to eq(1)
+    end
+
+    it "can be queried for cached?" do
+      expect(LockAndCacheMsgpack.cached?('hello')).to be_falsy
+      LockAndCacheMsgpack.lock_and_cache('hello') { nil }
+      expect(LockAndCacheMsgpack.cached?('hello')).to be_truthy
+    end
+
+    it 'allows expiry' do
+      count = 0
+      expect(LockAndCacheMsgpack.lock_and_cache('hello', expires: 1) { count += 1 }).to eq(1)
+      expect(count).to eq(1)
+      expect(LockAndCacheMsgpack.lock_and_cache('hello') { count += 1 }).to eq(1)
+      expect(count).to eq(1)
+      sleep 1.1
+      expect(LockAndCacheMsgpack.lock_and_cache('hello') { count += 1 }).to eq(2)
+      expect(count).to eq(2)
+    end
+
+    it "allows float expiry" do
+      expect{LockAndCacheMsgpack.lock_and_cache('hello', expires: 1.5) {}}.not_to raise_error
+    end
+
+    it 'can be nested' do
+      expect(LockAndCacheMsgpack.lock_and_cache('hello') do
+        LockAndCacheMsgpack.lock_and_cache('world') do
+          LockAndCacheMsgpack.lock_and_cache('privyet') do
+            123
+          end
+        end
+      end).to eq(123)
+    end
+
+    it "requires a key" do
+      expect do
+        LockAndCacheMsgpack.lock_and_cache do
+          raise "this won't happen"
+        end
+      end.to raise_error(/need/)
+    end
+
+    it 'allows checking locks' do
+      expect(LockAndCacheMsgpack.locked?(:sleeper)).to be_falsey
+      t = Thread.new do
+        LockAndCacheMsgpack.lock_and_cache(:sleeper) { sleep 1 }
+      end
+      sleep 0.2
+      expect(LockAndCacheMsgpack.locked?(:sleeper)).to be_truthy
+      t.join
+    end
+
+    it 'allows clearing' do
+      count = 0
+      expect(LockAndCacheMsgpack.lock_and_cache('hello') { count += 1 }).to eq(1)
+      expect(count).to eq(1)
+      LockAndCacheMsgpack.clear('hello')
+      expect(LockAndCacheMsgpack.lock_and_cache('hello') { count += 1 }).to eq(2)
+      expect(count).to eq(2)
+    end
+
+    it 'allows clearing (complex keys)' do
+      count = 0
+      expect(LockAndCacheMsgpack.lock_and_cache('hello', {world: 1}, expires: 100) { count += 1 }).to eq(1)
+      expect(count).to eq(1)
+      LockAndCacheMsgpack.clear('hello', world: 1)
+      expect(LockAndCacheMsgpack.lock_and_cache('hello', {world: 1}, expires: 100) { count += 1 }).to eq(2)
+      expect(count).to eq(2)
+    end
+
+    it 'allows multi-part keys' do
+      count = 0
+      expect(LockAndCacheMsgpack.lock_and_cache(['hello', 1, { target: 'world' }]) { count += 1 }).to eq(1)
+      expect(count).to eq(1)
+      expect(LockAndCacheMsgpack.lock_and_cache(['hello', 1, { target: 'world' }]) { count += 1 }).to eq(1)
+      expect(count).to eq(1)
+    end
+    
+    it 'treats a single hash arg as a cache key (not as options)' do
+      count = 0
+      LockAndCacheMsgpack.lock_and_cache(hello: 'world', expires: 100) { count += 1 }
+      expect(count).to eq(1)
+      LockAndCacheMsgpack.lock_and_cache(hello: 'world', expires: 100) { count += 1 }
+      expect(count).to eq(1)
+      LockAndCacheMsgpack.lock_and_cache(hello: 'world', expires: 200) { count += 1 } # expires is being treated as part of cache key
+      expect(count).to eq(2)
+    end
+
+    it "correctly identifies options hash" do
+      count = 0
+      LockAndCacheMsgpack.lock_and_cache({ hello: 'world' }, expires: 1, ignored: rand) { count += 1 }
+      expect(count).to eq(1)
+      LockAndCacheMsgpack.lock_and_cache({ hello: 'world' }, expires: 1, ignored: rand) { count += 1 } # expires is not being treated as part of cache key
+      expect(count).to eq(1)
+      sleep 1.1
+      LockAndCacheMsgpack.lock_and_cache({ hello: 'world' }) { count += 1 }
+      expect(count).to eq(2)
+    end
+  end
+
+  describe "shorter expiry for null results" do
+    it "optionally caches null for less time" do
+      count = 0
+      LockAndCacheMsgpack.lock_and_cache('hello', nil_expires: 1, expires: 2) { count += 1; nil }
+      expect(count).to eq(1)
+      LockAndCacheMsgpack.lock_and_cache('hello', nil_expires: 1, expires: 2) { count += 1; nil }
+      expect(count).to eq(1)
+      sleep 1.1 # this is enough to expire
+      LockAndCacheMsgpack.lock_and_cache('hello', nil_expires: 1, expires: 2) { count += 1; nil }
+      expect(count).to eq(2)
+    end
+
+    it "normally caches null for the same amount of time" do
+      count = 0
+      expect(LockAndCacheMsgpack.lock_and_cache('hello', expires: 1) { count += 1; nil }).to be_nil
+      expect(count).to eq(1)
+      expect(LockAndCacheMsgpack.lock_and_cache('hello', expires: 1) { count += 1; nil }).to be_nil
+      expect(count).to eq(1)
+      sleep 1.1
+      expect(LockAndCacheMsgpack.lock_and_cache('hello', expires: 1) { count += 1; nil }).to be_nil
+      expect(count).to eq(2)
+    end
+
+    it "caches non-null for normal time" do
+      count = 0
+      LockAndCacheMsgpack.lock_and_cache('hello', nil_expires: 1, expires: 2) { count += 1; true }
+      expect(count).to eq(1)
+      LockAndCacheMsgpack.lock_and_cache('hello', nil_expires: 1, expires: 2) { count += 1; true }
+      expect(count).to eq(1)
+      sleep 1.1
+      LockAndCacheMsgpack.lock_and_cache('hello', nil_expires: 1, expires: 2) { count += 1; true }
+      expect(count).to eq(1)
+      sleep 1
+      LockAndCacheMsgpack.lock_and_cache('hello', nil_expires: 1, expires: 2) { count += 1; true }
+      expect(count).to eq(2)
+    end
+  end
+
+
+end