moneta 1.1.1 → 1.2.0

data/lib/rack/moneta_rest.rb

@@ -69,11 +69,11 @@ module Rack
     private
 
     def empty(status)
-      [status, {'Content-Type'=>'application/octet-stream', 'Content-Length' => '0'}, []]
+      [status, { 'Content-Type' => 'application/octet-stream', 'Content-Length' => '0' }, []]
     end
 
     def respond(status, value)
-      [status, {'Content-Type'=>'application/octet-stream', 'Content-Length' => value.bytesize.to_s}, [value]]
+      [status, { 'Content-Type' => 'application/octet-stream', 'Content-Length' => value.bytesize.to_s }, [value]]
     end
   end
 end

data/lib/rack/session/moneta.rb

@@ -1,6 +1,5 @@
 require 'moneta'
 require 'rack/session/abstract/id'
-require 'thread'
 
 module Rack
   module Session
@@ -44,9 +43,10 @@ module Rack
         with_lock(env) do
           unless sid && session = @pool[sid]
             session = {}
-            begin
+            loop do
               sid = generate_sid
-            end until @pool.create(sid, session)
+              break if @pool.create(sid, session)
+            end
           end
           [sid, session]
         end
@@ -76,4 +76,3 @@ module Rack
     end
   end
 end
-

data/moneta.gemspec

@@ -18,4 +18,12 @@ Gem::Specification.new do |s|
   s.require_paths    = %w(lib)
   s.summary          = %{A unified interface to key/value stores, including Redis, Memcached, TokyoCabinet, ActiveRecord and many more}
   s.required_ruby_version = '>= 2.2.2'
+
+  s.add_development_dependency 'rspec', '~> 3.0'
+  s.add_development_dependency 'rspec-retry', '~> 0.6.1'
+  s.add_development_dependency 'rantly', '~> 1.2.0'
+  s.add_development_dependency 'parallel_tests', '~> 2.29.2'
+  s.add_development_dependency 'timecop', '~> 0.9.1'
+  s.add_development_dependency 'rubocop', '~> 0.67.2'
+  s.add_development_dependency 'irb', '~> 1.0.0'
 end

data/script/benchmarks

@@ -7,6 +7,8 @@ require 'fileutils'
 require 'active_support'
 require 'active_support/cache/moneta_store'
 
+require_relative '../spec/restserver.rb'
+
 class String
   def random(n)
     (1..n).map { self[rand(size),1] }.join
@@ -104,7 +106,16 @@ class MonetaBenchmarks
     },
     {name: "Cassandra"},
     {name: "Client (Memory)", adapter: :Client},
-    {name: "Couch"},
+    {
+      name: "Couch",
+      options: {
+        backend: if defined?(JRUBY_VERSION)
+                   require 'faraday/adapter/manticore'
+                   ::Faraday.new("http://127.0.0.1:5984/moneta") { |f| f.adapter :manticore }
+                 end
+      },
+      clear_options: { compact: true, await_compact: true },
+    },
     {
       name: "DBM",
       options: {file: "#{DIR}/dbm"}
@@ -152,7 +163,17 @@ class MonetaBenchmarks
     {name: "MongoOfficial"},
     {name: "PStore", options: { file: "#{DIR}/pstore" }},
     {name: "Redis"},
-    {name: "RestClient", options: { url: 'http://localhost:8808/' }},
+    {
+      name: "RestClient (Memory)",
+      adapter: :RestClient,
+      options: {
+        url: 'http://127.0.0.1:8808/moneta',
+        backend: if defined?(JRUBY_VERSION)
+                   require 'faraday/adapter/manticore'
+                   ::Faraday.new("http://127.0.0.1:8808/moneta") { |f| f.adapter :manticore }
+                 end
+      }
+    },
     {name: "Riak"},
     {
       name: "Sequel (MySQL)",
@@ -214,7 +235,7 @@ class MonetaBenchmarks
     },
     {name: "TDB", options: { file: "#{DIR}/tdb" }},
     {name: "TokyoCabinet", options: { file: "#{DIR}/tokyocabinet" }},
-    {name: "TokyoTyrant"},
+    {name: "TokyoTyrant", options: {port: 10431}},
   ].compact
 
   CONFIGS = {
@@ -349,43 +370,44 @@ class MonetaBenchmarks
   end
 
   def start_servers
-    parallel do
-      begin
-        Moneta::Server.new(Moneta.new(:Memory)).run
-      rescue Exception => ex
-        puts "\e[31mFailed to start Moneta server - #{ex.message}\e[0m"
+    @moneta_server_handle =
+      parallel do
+        begin
+          Moneta::Server.new(Moneta.new(:Memory)).run
+        rescue => ex
+          puts "\e[31mFailed to start Moneta server - #{ex.message}\e[0m"
+        end
       end
-    end
 
-    parallel do
-      begin
-        require 'rack'
-        require 'webrick'
-        require 'rack/moneta_rest'
+    @restserver_handle = start_restserver(8808)
 
-        # Keep webrick quiet
-        ::WEBrick::HTTPServer.class_eval do
-          def access_log(config, req, res); end
-        end
-        ::WEBrick::BasicLog.class_eval do
-          def log(level, data); end
-        end
+    @tokyotyrant_handle = spawn("ttserver -port 10431 -le -log #{DIR}/tokyotyrant.log #{DIR}/tokyotyrant.tch")
 
-        Rack::Server.start(app: Rack::Builder.app do
-                             use Rack::Lint
-                             run Rack::MonetaRest.new(store: :Memory)
-                           end,
-                           environment: :none,
-                           server: :webrick,
-                           Port: 8808)
-      rescue Exception => ex
-        puts "\e[31mFailed to start Rack server - #{ex.message}\e[0m"
-      end
+    Signal.trap "INT" do
+      stop_servers
+      exit
     end
 
     sleep 1 # Wait for servers
   end
 
+  def stop_servers
+    if @restserver_handle
+      stop_restserver(@restserver_handle)
+      @restserver_handle = nil
+    end
+
+    case @moneta_server_handle
+    when Thread
+      Thread.kill @moneta_server_handle
+    when Integer
+      Process.kill "TERM", @moneta_server_handle
+      Process.wait @moneta_server_handle
+    end
+
+    Process.kill "TERM", @tokyotyrant_handle
+  end
+
   def test_stores
     @stores.select! do |spec|
       adapter = spec[:adapter] || spec[:name].to_sym
@@ -480,7 +502,7 @@ class MonetaBenchmarks
       print "%s [%#{2 * @config[:runs]}s] " % [type, state]
 
       @config[:runs].times do |run|
-        store.clear
+        store.clear(spec[:clear_options] || {})
 
         @data.shuffle!
         m1 = Benchmark.measure do
@@ -571,6 +593,7 @@ class MonetaBenchmarks
     generate_data
     run_benchmarks
     print_summary
+    stop_servers
     FileUtils.rm_rf(DIR)
   end
 end

data/script/reconfigure-couchdb

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Reconfigure CouchDB to use delayed commits for speed: http://guide.couchdb.org/draft/performance.html
+sudo sed -i '/\[couchdb\]/a delayed_commits = true' /etc/couchdb/local.ini
+sudo systemctl restart couchdb
+
+# Display some info about CouchDB
+sudo systemctl status couchdb
+until curl http://localhost:5984/; do
+  sleep 1
+done
+

data/script/start-hbase

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 cd $(dirname $(dirname $0))
 

data/script/start-services

@@ -1,18 +1,10 @@
 #!/bin/bash
 
-echo Starting TokyoTyrant ...
-/usr/sbin/ttserver -dmn -pid /tmp/ttserver.pid -log /tmp/ttserver.log -port 1978
-
-# Reconfigure CouchDB to use delayed commits for speed: http://guide.couchdb.org/draft/performance.html
-sudo sed -i '/\[couchdb\]/a delayed_commits = true' /etc/couchdb/local.ini
-sudo service couchdb restart
+# Restart CouchDB
+$(dirname $0)/reconfigure-couchdb
 
 # Start HBase
 $(dirname $0)/start-hbase
 
 # Waiting for servers to start
 sleep 3
-
-echo TokyoTyrant status:
-ps aux | grep ttserver
-cat /tmp/ttserver.log

data/spec/active_support/cache_moneta_store_spec.rb

@@ -2,6 +2,7 @@ require 'moneta'
 require 'active_support'
 require 'active_support/cache/moneta_store'
 require 'ostruct'
+require_relative '../moneta/adapters/memcached_helper.rb'
 
 describe "cache_moneta_store" do
   before(:all) do
@@ -261,7 +262,8 @@ describe "cache_moneta_store" do
 
   describe ActiveSupport::Cache::MemCacheStore do
     let(:store){ described_class.new('127.0.0.1:11213') }
-    start_memcached 11213
+
+    include_context :start_memcached, 11213
 
     include_examples :basic_store
     include_examples :expiry

data/spec/features/concurrent_create.rb

@@ -1,21 +1,42 @@
 shared_examples :concurrent_create do
+  # Each thread attempts to create
   def create_thread(name)
     Thread.new do
       s = new_store
-      1000.times do |i|
-        s[i.to_s].should == name if s.create(i.to_s, name, expires: false)
-        Thread.pass if rand(100) >= 99
+      begin
+        (0...1000).map do |i|
+          s.create(i.to_s, name, expires: false).tap do
+            Thread.pass if rand(100) >= 99
+          end
+        end
+      ensure
+        s.close
       end
-      s.close
     end
   end
 
   it 'have atomic create across multiple threads', isolate: true do
-    a = create_thread('a')
-    b = create_thread('b')
-    c = create_thread('c')
-    a.join
-    b.join
-    c.join
+    names = %w{a b c}
+
+    # Spawn threads and then group results (lists of true/false values) by
+    # store index (0...1000)
+    results = names
+      .map { |name| create_thread(name) }
+      .map(&:value)
+      .transpose.each_with_index
+      .map { |created_values, i| [i.to_s, created_values] }
+      .to_h
+
+    # Just a quick sanity check
+    expect(results.length).to eq 1000
+
+    # Ensure that for each index, one and only one created value is true
+    expect(results.map { |_, created_values| created_values.inject(:^) }).to all(be true)
+
+    # Check that the when a call to create returned true, that the store
+    # contains the correct value as a result
+    expect(store.slice(*results.keys).to_h).to eq(results.map do |i, values|
+      [i, names[values.index(true)]]
+    end.to_h)
   end
 end

data/spec/features/concurrent_increment.rb

@@ -2,31 +2,39 @@ shared_examples :concurrent_increment do
   def increment_thread(name)
     Thread.new do
       s = new_store
-      100.times do |i|
-        100.times do |j|
-          s.increment("counter#{j}", 1, expires: false)
-          Thread.pass if rand(1000) >= 995
+      begin
+        # Create an array where each entry is a list of all the return values
+        # from calling increment for a particular key.
+        increments = (0...100).map { [] }
+        100.times do
+          100.times do |j|
+            increments[j] << s.increment(j.to_s, 1, expires: false).tap do
+              Thread.pass if rand(1000) >= 995
+            end
+          end
         end
-        s.store("#{name}#{i}", i.to_s, expires: false)
+        increments
+      ensure
+        s.close
       end
-      s.close
     end
   end
 
   it 'have atomic increment across multiple threads', isolate: true do
-    a = increment_thread('a')
-    b = increment_thread('b')
-    c = increment_thread('c')
-    a.join
-    b.join
-    c.join
-    100.times do |i|
-      store["a#{i}"].should == i.to_s
-      store["b#{i}"].should == i.to_s
-      store["c#{i}"].should == i.to_s
-    end
-    100.times do |j|
-      store.raw["counter#{j}"].should == 300.to_s
+    results = %w{a b c}
+      .map { |name| increment_thread(name) }
+      .map(&:value)
+      .transpose # Now the array is indexed by store key instead of thread
+
+    # Sanity check
+    expect(results.length).to eq 100
+
+    results.each do |ith_values|
+      # ensure that for each pair in the triple there are no overlapping values
+      expect(ith_values.combination(2).map { |a, b| a & b }).to all be_empty
+
+      # ensure that when joined together they cover the full 1..300 range
+      expect(ith_values.inject(:+)).to contain_exactly(*1..300)
     end
   end
 end

data/spec/features/increment.rb

@@ -1,72 +1,72 @@
 shared_examples :increment do
   it 'initializes in #increment with 1' do
-    store.key?('inckey').should be false
-    store.increment('inckey').should == 1
-    store.key?('inckey').should be true
-    store.raw['inckey'].should == '1'
-    store.raw.load('inckey').should == '1'
-    store.load('inckey', raw: true).should == '1'
+    expect(store.key?('inckey')).to be false
+    expect(store.increment('inckey')).to eq 1
+    expect(store.key?('inckey')).to be true
+    expect(store.raw['inckey'].to_s).to match(/^1\b/)
+    expect(store.raw.load('inckey').to_s).to match(/^1\b/)
+    expect(store.load('inckey', raw: true).to_s).to match(/^1\b/)
 
-    store.delete('inckey', raw: true).should == '1'
-    store.key?('inckey').should be false
+    expect(store.delete('inckey', raw: true).to_s).to match(/^1\b/)
+    expect(store.key?('inckey')).to be false
   end
 
   it 'initializes in #increment with higher value' do
-    store.increment('inckey', 42).should == 42
-    store.key?('inckey').should be true
-    store.raw['inckey'].should == '42'
-    store.delete('inckey', raw: true).should == '42'
+    expect(store.increment('inckey', 42)).to eq 42
+    expect(store.key?('inckey')).to be true
+    expect(store.raw['inckey'].to_s).to match(/^42\b/)
+    expect(store.delete('inckey', raw: true).to_s).to match(/^42\b/)
   end
 
   it 'initializes in #increment with 0' do
-    store.increment('inckey', 0).should == 0
-    store.key?('inckey').should be true
-    store.raw['inckey'].should == '0'
-    store.delete('inckey', raw: true).should == '0'
+    expect(store.increment('inckey', 0)).to eq 0
+    expect(store.key?('inckey')).to be true
+    expect(store.raw['inckey'].to_s).to match(/^0\b/)
+    expect(store.delete('inckey', raw: true).to_s).to match(/^0\b/)
   end
 
   it 'initializes in #decrement with 0' do
-    store.decrement('inckey', 0).should == 0
-    store.raw['inckey'].should == '0'
+    expect(store.decrement('inckey', 0)).to eq 0
+    expect(store.raw['inckey'].to_s).to match(/^0\b/)
   end
 
   it 'initializes in #decrement with negative value' do
-    store.decrement('inckey', -42).should == 42
-    store.raw['inckey'].should == '42'
+    expect(store.decrement('inckey', -42)).to eq 42
+    expect(store.raw['inckey'].to_s).to match(/^42\b/)
   end
 
   it 'supports incrementing existing value by value' do
-    store.increment('inckey').should == 1
-    store.increment('inckey', 42).should == 43
-    store.raw['inckey'].should == '43'
+    expect(store.increment('inckey')).to eq 1
+    expect(store.increment('inckey', 42)).to eq 43
+    expect(store.raw['inckey'].to_s).to match(/^43\b/)
   end
 
   it 'supports decrementing existing value by value' do
-    store.increment('inckey').should == 1
-    store.decrement('inckey').should == 0
-    store.increment('inckey', 42).should == 42
-    store.decrement('inckey', 2).should == 40
-    store.raw['inckey'].should == '40'
+    expect(store.increment('inckey')).to eq 1
+    expect(store.decrement('inckey')).to eq 0
+    expect(store.increment('inckey', 42)).to eq 42
+    expect(store.decrement('inckey', 2)).to eq 40
+    expect(store.raw['inckey'].to_s).to match(/^40\b/)
   end
 
   it 'supports incrementing existing value by 0' do
-    store.increment('inckey').should == 1
-    store.increment('inckey', 0).should == 1
-    store.raw['inckey'].should == '1'
+    expect(store.increment('inckey')).to eq 1
+    expect(store.increment('inckey', 0)).to eq 1
+    expect(store.raw['inckey'].to_s).to match(/^1\b/)
   end
 
   it 'supports decrementing existing value' do
-    store.increment('inckey', 10).should == 10
-    store.increment('inckey', -5).should == 5
-    store.raw['inckey'].should == '5'
-    store.increment('inckey', -5).should == 0
-    store.raw['inckey'].should == '0'
+    expect(store.increment('inckey', 10)).to eq 10
+    expect(store.increment('inckey', -5)).to eq 5
+    expect(store.raw['inckey'].to_s).to match(/^5\b/)
+    expect(store.increment('inckey', -5)).to eq 0
+    expect(store.raw['inckey'].to_s).to match(/^0\b/)
   end
 
   it 'interprets raw value as integer' do
     store.store('inckey', '42', raw: true)
-    store.increment('inckey').should == 43
-    store.raw['inckey'].should == '43'
+    expect(store.increment('inckey')).to eq 43
+    expect(store.raw['inckey'].to_s).to match(/^43\b/)
   end
 
   it 'raises error in #increment on non integer value' do
@@ -88,10 +88,10 @@ shared_examples :increment do
     b = Moneta::Semaphore.new(store, 'semaphore', 2)
     c = Moneta::Semaphore.new(store, 'semaphore', 2)
     a.synchronize do
-      a.locked?.should be true
+      expect(a.locked?).to be true
       b.synchronize do
-        b.locked?.should be true
-        c.try_lock.should be false
+        expect(b.locked?).to be true
+        expect(c.try_lock).to be false
       end
     end
   end