moneta 1.1.0 → 1.4.0

data/spec/moneta/adapters/sequel/helper.rb

@@ -0,0 +1,75 @@
+RSpec.shared_context :sequel do
+  def mysql_uri database=nil
+    database ||= mysql_database1
+    if defined?(JRUBY_VERSION)
+      "jdbc:mysql://localhost/#{database}?user=#{mysql_username}&useSSL=false"
+    else
+      "mysql2://#{mysql_username}:@localhost/#{database}"
+    end
+  end
+
+  def sqlite_uri file_name
+    "#{defined?(JRUBY_VERSION) && 'jdbc:'}sqlite://" + File.join(tempdir, file_name)
+  end
+
+  def postgres_options database=nil
+    database ||= postgres_database1
+    if defined?(JRUBY_VERSION)
+      {db: "jdbc:postgresql://localhost/#{database}?user=#{postgres_username}"}
+    else
+      {
+        db: "postgres://localhost/#{database}",
+        user: postgres_username
+      }
+    end
+  end
+
+  def postgres_hstore_options database=nil
+    postgres_options(database).merge \
+      table: 'hstore_table1',
+      hstore: 'row'
+  end
+
+  def h2_uri
+    "jdbc:h2:" + tempdir
+  end
+end 
+
+RSpec.shared_examples :adapter_sequel do |specs, optimize: true|
+  shared_examples :each_key_server do
+    context "with each_key server" do
+      let(:opts) do
+        base_opts.merge(
+          servers: {each_key: {}},
+          each_key_server: :each_key
+        )
+      end
+
+      moneta_specs specs
+    end
+
+    context "without each_key server" do
+      let(:opts) { base_opts }
+      moneta_specs specs
+    end
+  end
+
+  if optimize
+    context 'with backend optimizations' do
+      let(:base_opts) { {table: "adapter_sequel"} }
+
+      include_examples :each_key_server
+    end
+  end
+
+  context 'without backend optimizations' do
+    let(:base_opts) do
+      {
+        table: "adapter_sequel",
+        optimize: false
+      }
+    end
+
+    include_examples :each_key_server
+  end
+end

data/spec/moneta/adapters/sequel/standard_sequel_spec.rb

@@ -1,14 +1,8 @@
+require_relative './helper.rb'
+
 describe 'standard_sequel', adapter: :Sequel do
-  moneta_store :Sequel do
-    {
-      db: if defined?(JRUBY_VERSION)
-            "jdbc:mysql://localhost/#{mysql_database1}?user=#{mysql_username}"
-          else
-            "mysql2://#{mysql_username}:@localhost/#{mysql_database1}"
-          end,
-      table: "simple_sequel"
-    }
-  end
+  include_context :sequel
 
-  moneta_specs STANDARD_SPECS
+  moneta_store(:Sequel) { postgres_options.merge(table: "standard_sequel") }
+  moneta_specs STANDARD_SPECS.with_each_key
 end

data/spec/moneta/adapters/sequel/standard_sequel_with_expires_spec.rb

@@ -1,19 +1,18 @@
+require_relative './helper.rb'
+
 describe 'standard_sequel_with_expires', adapter: :Sequel do
+  include_context :sequel
+
   let(:t_res) { 0.125 }
   let(:min_ttl) { t_res }
   use_timecop
 
   moneta_store :Sequel do
-    {
-      db: if defined?(JRUBY_VERSION)
-            "jdbc:mysql://localhost/#{mysql_database1}?user=#{mysql_username}"
-          else
-            "mysql2://#{mysql_username}:@localhost/#{mysql_database1}"
-          end,
-      table: "simple_sequel_with_expires",
+    postgres_options.merge(
+      table: "standard_sequel_with_expires",
       expires: true
-    }
+    )
   end
 
-  moneta_specs STANDARD_SPECS.with_expires
+  moneta_specs STANDARD_SPECS.with_expires.with_each_key
 end

data/spec/moneta/adapters/sqlite/standard_sqlite_spec.rb

@@ -3,5 +3,5 @@ describe 'standard_sqlite', unsupported: defined?(JRUBY_VERSION), adapter: :Sqli
     {file: File.join(tempdir, "standard_sqlite")}
   end
 
-  moneta_specs STANDARD_SPECS.without_concurrent
+  moneta_specs STANDARD_SPECS.without_concurrent.with_each_key
 end

data/spec/moneta/adapters/sqlite/standard_sqlite_with_expires_spec.rb

@@ -11,5 +11,5 @@ describe 'standard_sqlite_with_expires', unsupported: defined?(JRUBY_VERSION), a
     }
   end
 
-  moneta_specs STANDARD_SPECS.with_expires.without_concurrent
+  moneta_specs STANDARD_SPECS.with_expires.without_concurrent.with_each_key
 end

data/spec/moneta/adapters/tdb/standard_tdb_spec.rb

@@ -3,5 +3,5 @@ describe 'standard_tdb', unsupported: defined?(JRUBY_VERSION), adapter: :TDB do
     {file: File.join(tempdir, "simple_tdb")}
   end
 
-  moneta_specs STANDARD_SPECS.without_multiprocess
+  moneta_specs STANDARD_SPECS.without_multiprocess.with_each_key
 end

data/spec/moneta/adapters/tdb/standard_tdb_with_expires_spec.rb

@@ -7,5 +7,5 @@ describe 'standard_tdb_with_expires', unsupported: defined?(JRUBY_VERSION), adap
     {file: File.join(tempdir, "simple_tdb_with_expires"), expires: true}
   end
 
-  moneta_specs STANDARD_SPECS.without_multiprocess.with_expires
+  moneta_specs STANDARD_SPECS.without_multiprocess.with_expires.with_each_key
 end

data/spec/moneta/adapters/tokyocabinet/standard_tokyocabinet_spec.rb

@@ -3,5 +3,5 @@ describe 'standard_tokyocabinet', unsupported: defined?(JRUBY_VERSION), adapter:
     {file: File.join(tempdir, "simple_tokyocabinet")}
   end
 
-  moneta_specs STANDARD_SPECS.without_multiprocess
+  moneta_specs STANDARD_SPECS.without_multiprocess.with_each_key
 end

data/spec/moneta/adapters/tokyocabinet/standard_tokyocabinet_with_expires_spec.rb

@@ -7,5 +7,5 @@ describe 'standard_tokyocabinet_with_expires', unsupported: defined?(JRUBY_VERSI
     {file: File.join(tempdir, "simple_tokyocabinet_with_expires"), expires: true}
   end
 
-  moneta_specs STANDARD_SPECS.without_multiprocess.with_expires
+  moneta_specs STANDARD_SPECS.without_multiprocess.with_expires.with_each_key
 end

data/spec/moneta/adapters/tokyotyrant/adapter_tokyotyrant_spec.rb

@@ -1,6 +1,10 @@
-describe 'adapter_tokyotyrant', isolate: true, adapter: :TokyoTyrant do
+require_relative './helper.rb'
+
+describe 'adapter_tokyotyrant', adapter: :TokyoTyrant do
+  include_context :start_tokyotyrant, 10654
+
   moneta_build do
-    Moneta::Adapters::TokyoTyrant.new
+    Moneta::Adapters::TokyoTyrant.new(port: 10654)
   end
 
   moneta_specs ADAPTER_SPECS

data/spec/moneta/adapters/tokyotyrant/helper.rb

@@ -0,0 +1,12 @@
+RSpec.shared_context :start_tokyotyrant do |port|
+  before :context do
+    @tokyotyrant = spawn("ttserver -port #{port} -le -log #{tempdir}/tokyotyrant#{port}.log #{tempdir}/tokyotyrant#{port}.tch")
+    sleep 0.5
+  end
+
+  after :context do
+    Process.kill("TERM", @tokyotyrant)
+    Process.wait(@tokyotyrant)
+    @tokyotyrant = nil
+  end
+end

data/spec/moneta/adapters/tokyotyrant/standard_tokyotyrant_spec.rb

@@ -1,4 +1,7 @@
-describe 'standard_tokyotyrant', isolate: true, adapter: :TokyoTyrant do
-  moneta_store :TokyoTyrant
+require_relative './helper.rb'
+
+describe 'standard_tokyotyrant', adapter: :TokyoTyrant do
+  include_context :start_tokyotyrant, 10655
+  moneta_store :TokyoTyrant, port: 10655
   moneta_specs STANDARD_SPECS
 end

data/spec/moneta/adapters/tokyotyrant/standard_tokyotyrant_with_expires_spec.rb

@@ -1,8 +1,12 @@
+require_relative './helper.rb'
+
 describe 'standard_tokyotyrant_with_expires', adapter: :TokyoTyrant do
   let(:t_res) { 0.125 }
   let(:min_ttl) { t_res }
   use_timecop
 
-  moneta_store :TokyoTyrant, {expires: true}
+  include_context :start_tokyotyrant, 10656
+
+  moneta_store :TokyoTyrant, expires: true, port: 10656
   moneta_specs STANDARD_SPECS.with_expires
 end

data/spec/moneta/adapters/yaml/standard_yaml_spec.rb

@@ -5,5 +5,5 @@ describe 'standard_yaml', adapter: :YAML do
 
   moneta_loader{ |value| value }
 
-  moneta_specs STANDARD_SPECS.without_marshallable_value.without_concurrent
+  moneta_specs STANDARD_SPECS.without_marshallable_value.without_concurrent.with_each_key
 end

data/spec/moneta/adapters/yaml/standard_yaml_with_expires_spec.rb

@@ -9,5 +9,5 @@ describe 'standard_yaml_with_expires', adapter: :YAML do
 
   moneta_loader{ |value| value }
 
-  moneta_specs STANDARD_SPECS.without_marshallable_value.with_expires.without_concurrent
+  moneta_specs STANDARD_SPECS.without_marshallable_value.with_expires.without_concurrent.with_each_key
 end

data/spec/moneta/builder_spec.rb

@@ -25,4 +25,26 @@ describe Moneta::Builder do
       end.build
     end.to raise_error /Please check/
   end
+
+  it 'dups options before passing them to each middleware' do
+    my_adapter = Class.new do
+      def initialize(options)
+        throw "a is missing" unless options.delete(:a)
+      end
+    end
+
+    my_middleware = Class.new do
+      def initialize(backend, options)
+        throw "a is missing" unless options.delete(:a)
+      end
+    end
+
+    options = { a: 1 }
+    Moneta::Builder.new do
+      use my_middleware, options
+      adapter my_adapter, options
+    end.build
+
+    expect(options).to include(a: 1)
+  end
 end

data/spec/moneta/proxies/enumerable/enumerable_spec.rb

@@ -0,0 +1,26 @@
+describe 'enumerable', proxy: :Enumerable do
+  moneta_build do
+    Moneta.build do
+      use :Enumerable
+      adapter :Memory
+    end
+  end
+
+  moneta_specs STANDARD_SPECS.without_transform.returnsame.without_persist.with_each_key
+
+  it 'includes the enumerable interface' do
+    expect(store).to be_a Enumerable
+    expect(Enumerable.instance_methods).to all satisfy { |m| store.respond_to? m }
+  end
+
+  it 'allows enumeration over key-value pairs' do
+    moneta_property_of(keys: 100, values: 100) do |keys:, values:|
+      pairs = keys.zip(values)
+      store.merge!(pairs)
+      expect(store.to_a).to contain_exactly(*pairs)
+      expect(store.each.to_a).to contain_exactly(*pairs)
+      expect(store.each_pair.to_a).to contain_exactly(*pairs)
+      store.clear
+    end
+  end
+end

data/spec/moneta/proxies/expires/expires_file_spec.rb

@@ -13,7 +13,7 @@ describe 'expires_file', proxy: :Expires do
     end
   end
 
-  moneta_specs STANDARD_SPECS.with_expires.stringvalues_only
+  moneta_specs STANDARD_SPECS.with_expires.stringvalues_only.with_each_key
 
   it 'deletes expired value in underlying file storage' do
     store.store('foo', 'bar', expires: 2)

data/spec/moneta/proxies/fallback/fallback_spec.rb

@@ -0,0 +1,42 @@
+describe 'fallback', proxy: :Fallback do
+  context 'when the adapter is working' do
+    moneta_build do
+      Moneta.build do
+        use :Fallback
+        adapter :Memory
+      end
+    end
+
+    moneta_specs STANDARD_SPECS.without_transform.returnsame.without_persist.with_each_key
+  end
+
+  context 'when the adapter is broken' do
+    moneta_build do
+      Moneta.build do
+        use :Fallback #, rescue: [IOError, NoMethodError]
+        adapter(Class.new do
+          include Moneta::Defaults
+
+          def load(key, options = {})
+            raise IOError, "deliberate error for load"
+          end
+
+          def store(key, value, options = {})
+            raise IOError, "deliberate error for store"
+          end
+
+          def delete(key, options = {})
+            raise IOError, "deliberate error for delete"
+          end
+
+          def clear(options = {})
+            raise IOError, "deliberate error for clear"
+          end
+        end.new)
+      end
+    end
+
+    # Null adapter behaviour
+    moneta_specs MonetaSpecs.new(specs: [:null, :not_increment, :not_create, :not_persist])
+  end
+end

data/spec/moneta/proxies/pool/pool_spec.rb

@@ -1,12 +1,325 @@
+require 'timeout'
+
 describe "pool", proxy: :Pool do
-  moneta_build do
-    tempdir = self.tempdir
-    Moneta.build do
-      use :Pool do
-        adapter :File, dir: File.join(tempdir, "pool")
+  describe "Moneta::Pool" do
+    # To test the pool, we create the store once and pass the same object around
+    # whenever a new store is requested.
+    before :context do
+      tempdir = @pool_tempdir = Dir.mktmpdir
+      @pool_store = Moneta.build do
+        use :Pool, max: 2, timeout: 5 do
+          adapter :File, dir: tempdir
+        end
       end
     end
+
+    # Tell the manager to close all open stores
+    after :context do
+      @pool_store.stop
+      FileUtils.remove_dir(@pool_tempdir)
+    end
+
+    moneta_build { @pool_store }
+    moneta_specs ADAPTER_SPECS.with_each_key
+
+    it "raises an error on check-out the builder fails" do
+      store = Moneta.build do
+        use :Pool do
+          adapter(Class.new do
+            def initialize(options = {})
+              raise "boom"
+            end
+          end)
+        end
+      end
+
+      expect { store['x'] }.to raise_error "boom"
+    end
   end
 
-  moneta_specs ADAPTER_SPECS.with_each_key
+  describe "Moneta::Pool::PoolManager" do
+    let :builder do
+      double('builder').tap do |builder|
+        i = -1
+        allow(builder).to receive(:build) do
+          [stores[i += 1]]
+        end
+      end
+    end
+
+    let(:stores) { (0...num).map { |i| double("store#{i}") } }
+
+    after { subject.kill! }
+
+    shared_examples :no_ttl do
+      context "with one store" do
+        let(:num) { 1 }
+
+        it "never expires the store" do
+          store = stores.first
+          expect(builder).to receive(:build).once
+          expect(subject.check_out).to be store
+          expect(subject.stats).to include(stores: 1, available: 0)
+          expect(subject.check_in(store)).to eq nil
+          expect(subject.stats).to include(stores: 1, available: 1)
+          sleep 1
+          expect(subject.stats).to include(stores: 1, available: 1)
+          expect(subject.check_out).to be store
+          expect(subject.check_in(store)).to eq nil
+        end
+      end
+    end
+
+    shared_examples :no_max do
+      context "with 1,000 stores" do
+        let(:num) { 1_000 }
+
+        it "never blocks" do
+          # Check out 1000 stores in 1000 threads
+          threads = (0...num).map do
+            Thread.new { subject.check_out }
+          end
+          expect(threads.map(&:value)).to contain_exactly(*stores)
+          expect(subject.stats).to include(stores: num, available: 0, waiting: 0)
+
+          # Check in the first 50
+          expect(stores.take(50).map { |store| subject.check_in(store) }).to all(be nil)
+          expect(subject.stats).to include(stores: num, available: 50, waiting: 0)
+
+          # Now check those 50 out again
+          threads = (0...50).map do
+            Thread.new { subject.check_out }
+          end
+          expect(threads.map(&:value)).to contain_exactly(*stores.take(50))
+
+          # Finally check in all stores
+          expect(stores.map { |store| subject.check_in(store) }).to all(be nil)
+        end
+      end
+    end
+
+    shared_examples :min do |min|
+      describe "initial number of stores" do
+        let(:num) { min }
+
+        it "starts with #{min} available stores" do
+          expect(subject.stats).to include(stores: min, available: min)
+          expect((0...min).map { subject.check_out }).to contain_exactly(*stores)
+        end
+      end
+    end
+
+    shared_examples :max do |max, timeout: 5|
+      describe "maximum number of stores" do
+        let(:num) { max }
+
+        after do
+          expect(stores.map { |store| subject.check_in(store) }).to all be_nil
+        end
+
+        it "blocks after #{max} stores have been created" do
+          # All stores are checked out
+          expect(max.times.map { subject.check_out }).to contain_exactly(*stores)
+
+          # Launch threads that make checkout requests (in order).  These will
+          # all block.
+          threads = max.times.map { Thread.new { subject.check_out } }
+          Timeout.timeout(timeout) { Thread.pass until subject.stats[:waiting] == max }
+          expect(threads).to all be_alive
+
+          # Check all stores except the first one back in.  This will cause all
+          # but one of the threads to return
+          expect(stores.drop(1).map { |store| subject.check_in(store) }).to all be_nil
+          Timeout.timeout(timeout) { Thread.pass until threads.select(&:alive?).length == 1 }
+          expect(subject.stats).to include(waiting: 1)
+
+          # Ensure that the stores that were checked back in are the ones that
+          # were given to the waiting threads.
+          alive, dead = threads.partition(&:alive?)
+          expect(dead.map(&:value)).to contain_exactly(*stores.drop(1))
+
+          # Check the last store back in and make sure it goes to the last
+          # waiting thread.
+          last_thread = alive.first
+          expect(subject.check_in(stores.first)).to eq nil
+          Timeout.timeout(timeout) { Thread.pass while last_thread.alive? }
+          expect(subject.stats).to include(waiting: 0)
+          expect(last_thread.value).to be stores.first
+        end
+      end
+    end
+
+    shared_examples :ttl do |ttl, min: 0, max: nil|
+      describe "closing stores after TTL expiry" do
+        let(:num) { max || min + 10 }
+
+        it "closes available stores after ttl" do
+          stores.each do |store|
+            allow(store).to receive(:close)
+          end
+
+          Timeout.timeout(5) { Thread.pass until subject.stats[:stores] == min }
+          expect(stores.length.times.map { subject.check_out }).to contain_exactly(*stores)
+          expect(subject.stats).to include(stores: num, available: 0)
+          expect(stores.map { |store| subject.check_in(store) }).to all be_nil
+          expect(subject.stats).to include(stores: num, available: num)
+          sleep ttl
+          expect(subject.stats).to include(stores: min, available: min)
+        end
+      end
+    end
+
+    shared_examples :timeout do |timeout, max:|
+      describe "raising an error after timeout" do
+        let(:num) { max }
+
+        it "raises a timeout error after waiting too long" do
+          expect((0...num).map { subject.check_out }).to contain_exactly(*stores)
+          # One extra checkout request in a separate thread
+          t = Thread.new do
+            Thread.current.report_on_exception = false if Thread.current.respond_to? :report_on_exception
+            subject.check_out
+          end
+          Timeout.timeout(timeout) { Thread.pass until subject.stats[:waiting] == 1 }
+          expect(subject.stats[:longest_wait]).to be_a Time
+          expect(t).to be_alive
+          sleep timeout
+          Timeout.timeout(timeout) { Thread.pass while t.alive? }
+          expect { t.value }.to raise_error Moneta::Pool::TimeoutError
+          expect(subject.stats).to include(waiting: 0, longest_wait: nil)
+          expect(stores.map { |store| subject.check_in store }).to all be_nil
+        end
+      end
+    end
+
+    context "with default arguments" do
+      subject { Moneta::Pool::PoolManager.new(builder) }
+      after { subject.kill! }
+
+      include_examples :no_ttl
+      include_examples :no_max
+      include_examples :min, 0
+    end
+
+    context "with max: 10, timeout: 4" do
+      subject { Moneta::Pool::PoolManager.new(builder, max: 10, timeout: 4) }
+      after { subject.kill! }
+
+      include_examples :no_ttl
+      include_examples :max, 10, timeout: 4
+      include_examples :min, 0
+      include_examples :timeout, 4, max: 10
+    end
+
+    context "with min: 10" do
+      subject { Moneta::Pool::PoolManager.new(builder, min: 10) }
+      after { subject.kill! }
+
+      include_examples :no_max
+      include_examples :min, 10
+    end
+
+    context "with ttl: 1" do
+      subject { Moneta::Pool::PoolManager.new(builder, ttl: 1) }
+      after { subject.kill! }
+
+      include_examples :ttl, 1, min: 0
+    end
+
+    context "with min: 10, max: 20, ttl: 1, timeout: 4" do
+      subject { Moneta::Pool::PoolManager.new(builder, min: 10, max: 20, ttl: 1, timeout: 4) }
+      after { subject.kill! }
+
+      include_examples :min, 10
+      include_examples :max, 20, timeout: 4
+      include_examples :ttl, 1, min: 10, max: 20
+      include_examples :timeout, 4, max: 20
+    end
+
+    context "with min: 10, max: 10, ttl: 2, timeout: 4" do
+      subject { Moneta::Pool::PoolManager.new(builder, min: 10, max: 10, ttl: 2, timeout: 4) }
+      after { subject.kill! }
+
+      include_examples :min, 10
+      include_examples :max, 10, timeout: 4
+      include_examples :ttl, 2, min: 10, max: 10
+      include_examples :timeout, 4, max: 10
+    end
+
+    describe '#check_out' do
+      subject { Moneta::Pool::PoolManager.new(builder, max: 1, timeout: 5) }
+      after { subject.kill! }
+
+      let(:num) { 1 }
+
+      it 'yields the store to requesters first come, first served' do
+        store = stores.first
+        procs = (0...10)
+          .map { |i| double("proc#{i}") }
+          .each { |p| expect(p).to receive(:call).with(store).ordered }
+
+        # Each thread stops immediately after starting.  We can then ensure
+        # that each checkout is called in order.
+        threads = procs.map do |p|
+          Thread.new do
+            Thread.stop
+            p.call(subject.check_out)
+          end
+        end
+
+        Timeout.timeout(5) { Thread.pass until threads.all?(&:stop?) }
+
+        # The first thread should return immediately after waking
+        threads.first.wakeup
+        Timeout.timeout(5) { threads.first.join }
+
+        # Wait for the remaining threads to block, one at a time.
+        threads.drop(1).each_with_index do |thread, i|
+          thread.wakeup
+          Timeout.timeout(5) { Thread.pass until subject.stats[:waiting] == i + 1 }
+        end
+
+        # The remaining threads should be waiting for the store to be checked back in
+        expect(threads.drop(1)).to all be_alive
+        expect(subject.stats).to include(waiting: 9)
+
+        Timeout.timeout(5) do
+          threads.each do |t|
+            t.join
+            subject.check_in(store)
+          end
+        end
+      end
+
+      it "raises a ShutdownError if the pool is stopped while waiting for a store" do
+        # Exaust the pool
+        store = stores.first
+        allow(store).to receive(:close).once
+        expect(subject.check_out).to eq store
+
+        # Simulate a new thread requesting a check-out
+        t1 = Thread.new do
+          Thread.current.report_on_exception = false if Thread.current.respond_to? :report_on_exception
+          subject.check_out
+        end
+        Timeout.timeout(5) { Thread.pass until subject.stats[:waiting] > 0 }
+        expect(t1).to be_alive
+
+        # Meanwhile in another thread, the pool is stopped.
+        t2 = Thread.new { subject.stop }
+
+        # The requesting thread should error out immediately
+        expect { t1.value }.to raise_error Moneta::Pool::ShutdownError
+
+        # In this thread we return the store to the pool, allowing graceful shutdown to complete.
+        subject.check_in(store)
+        expect(t2.value).to be_nil
+      end
+
+      it "raises a ShutdownError if a the pool is stopped before requesting a store" do
+        subject.stop
+        expect { subject.check_out }.to raise_error Moneta::Pool::ShutdownError
+      end
+    end
+  end
 end