moneta 1.0.0 → 1.5.1

data/script/parallel-tests

@@ -1,105 +1,80 @@
 #!/usr/bin/env ruby
 
-def rspec(spec)
-  if system("rspec #{spec}")
-    true
-  elsif sig = $?.termsig
-    found = Signal.list.to_a.select {|name, id| id == sig }.first
-    puts "\e[31m########## SIG#{found ? found.first : sig} rspec #{spec} ##########\e[0m"
-    false
-  else
-    puts "rspec terminated with #{$?.exitstatus}"
-    false
-  end
-end
-
 ENV['PARALLEL_TESTS'] = 'yes'
 
-specs = Dir['spec/*/*_spec.rb'].sort
+require 'multi_json'
 
-# Shuffle specs to ensure equal distribution over the test groups
-# We have to shuffle with the same seed every time because rake is started
-# multiple times!
-old_seed = srand(43)
-specs.shuffle!
-srand(old_seed)
+def tag_args tags
+  tags.flat_map{ |tag| ['--tag', tag] }
+end
 
-# FIXME:
-#
-# * QuickLZ segfaults because of an assertion
-#   QuickLZ is also not maintained on Github, but on Bitbucket
-#   and I don't know where the issue tracker is.
-#
-# * PStore increment/locking doesn't work correctly on JRuby
-#
-unstable = %w(quicklz riak cassandra)
-unstable += %w(pstore) if defined?(JRUBY_VERSION)
+def example_ids tags, specs
+  json = `bundle exec rspec -f j --dry-run #{tag_args(tags).join(' ')} -- #{specs.join(' ')}`
+  data = MultiJson.load(json)
+  data['examples'].map{ |example| example['id'] }
+end
 
-unstable_re = /#{unstable.join('|')}/
-unstable = specs.select {|s| s =~ unstable_re }
-specs -= unstable
+def run(*args)
+  pid = spawn(*args)
+  Signal.trap("INT") { Process.kill("INT", pid) }
+  Process.wait(pid)
+  $? == 0
+ensure
+  Signal.trap("INT", "DEFAULT")
+end
 
-group = ARGV.first || '1/1'
-case group
-when /^(\d+)\/(\d+)$/
-  n = $1.to_i
-  max = $2.to_i
-  if n == max
-    specs = specs[(n-1)*(specs.size/max)..-1]
+tags = ARGV.take_while { |arg| arg[0] != '-' }
+ARGV.shift(tags.length)
+opts = []
+files = nil
+while arg = ARGV.shift
+  case arg
+  when '--'
+    files = ARGV
+    break
+  when '--remainder'
+    files = Dir['spec/**/*_spec.rb', 'test/**/*_test.rb']
+    existing = File.open('.travis.yml').each_line.flat_map do |line|
+      next unless matches = line.match(%r{((?:test|spec)/(?:[\w\.]+/?)*)})
+      path = matches[1]
+      path[-3..-1] == '.rb' ? path : path + '/**/*.rb'
+    end.compact
+    files -= Dir[*existing]
   else
-    specs = specs[(n-1)*(specs.size/max), specs.size/max]
+    opts << arg
   end
-when 'unstable'
-  specs = unstable
-else
-  puts "Invalid test group #{group}"
-  exit 1
 end
 
-puts "The following specs will be executed:\n\t#{specs.join "\n\t"}\n\n"
+files ||= Dir['spec', 'test/**/*_test.rb']
+specs, tests = files.partition { |file| file.match /^spec/ }
 
-# Memcached and Redis specs cannot be used in parallel
-# because of flushing and lacking namespaces
-parallel = []
-%w(memcached redis client shared riak tokyotyrant couch cassandra).each do |name|
-  serial = specs.select { |s| s.include?(name) }
-  unless serial.empty?
-    specs -= serial
-    parallel << serial
-  end
-end
+puts "The following specs will be executed:\n\t#{specs.join "\n\t"}\n\n" unless specs.empty?
+puts "The following tests will be executed:\n\t#{tests.join "\n\t"}\n\n" unless tests.empty?
 
-# The activesupport cache specs also use memcached
-activesupport_cache_specs = specs.grep(/active_support.+cache/)
-specs -= activesupport_cache_specs
+results = []
+unless specs.empty?
+  # run all non :isolate examples in parallel
+  results << run(*%w{bundle exec parallel_rspec --},
+                 *opts,
+                 *tag_args(tags | %w{~isolate}),
+                 '--',
+                 *specs)
 
-if memcache_specs = parallel.find{ |serial| serial.all?{ |s| s.match /memcached/ } }
-  activesupport_cache_specs.each(&memcache_specs.method(:push))
-else
-  parallel += activesupport_cache_specs
+  # find the example IDs of the isolate examples to be run in serial
+  ids = example_ids(tags, specs) - example_ids(tags | %w{~isolate}, specs)
+  unless ids.empty?
+    results << run(*%w{bundle exec rspec},
+                   *opts,
+                   '--',
+                   *ids)
+  end
 end
 
-# All the left-overs
-parallel += specs.map {|s| [s] }
-
-threads = []
-failed = false
-parallel.each do |serial|
-  threads << Thread.new do
-    begin
-      serial.each do |spec|
-        failed = true unless rspec(spec)
-      end
-    ensure
-      threads.delete Thread.current
-    end
-  end
-  sleep 0.1
-  sleep 0.1 while threads.size >= 5
+tests.each do |test|
+  results << run(*%w{bundle exec ruby}, test)
 end
-sleep 0.1 until threads.empty?
 
-if failed
+if results.any?{ |result| !result }
   puts "\e[31m########## MONETA TESTSUITE FAILED ##########\e[0m"
   exit 1
 end

data/script/start-couchdb

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+# Copied from https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian
+COUCHDB_PASSWORD=password
+echo "couchdb couchdb/mode select standalone
+couchdb couchdb/mode seen true
+couchdb couchdb/bindaddress string 127.0.0.1
+couchdb couchdb/bindaddress seen true
+couchdb couchdb/adminpass password ${COUCHDB_PASSWORD}
+couchdb couchdb/adminpass seen true
+couchdb couchdb/adminpass_again password ${COUCHDB_PASSWORD}
+couchdb couchdb/adminpass_again seen true" | sudo debconf-set-selections
+DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --force-yes couchdb
+
+# Reconfigure CouchDB to use delayed commits for speed: http://guide.couchdb.org/draft/performance.html
+sudo sed -i '/\[couchdb\]/a delayed_commits = true' /etc/couchdb/local.ini
+
+# (Re)start couchdb
+sudo systemctl restart couchdb
+
+# Display some info about CouchDB
+sudo systemctl status couchdb
+until curl http://localhost:5984/; do
+  sleep 1
+done
+

data/script/start-hbase

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -e
+
+cd $(dirname $(dirname $0))
+
+mkdir -p hbase
+mkdir -p zookeeper
+mkdir -p downloads
+root=$(pwd)
+
+version=$(curl -sS https://downloads.apache.org/hbase/stable/RELEASENOTES.md | grep -oP '(?<=# HBASE\s\s)(\d+\.?)+' | head -n1)
+
+echo HBase stable version is $version
+
+if [ ! -f downloads/hbase-$version-bin.tar.gz ]; then
+  echo Downloading HBase ...
+  wget -P downloads https://downloads.apache.org/hbase/stable/hbase-$version-bin.tar.gz
+fi
+
+echo Extracting HBase ...
+cd hbase
+tar -zxf $root/downloads/hbase-$version-bin.tar.gz
+
+echo Configuring HBase ...
+echo "export JAVA_HOME=/usr" >> hbase-$version/conf/hbase-env.sh
+tee <<-EOF > hbase-$version/conf/hbase-site.xml
+<configuration>
+  <property>
+    <name>hbase.rootdir</name>
+    <value>file://$root/hbase</value>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.dataDir</name>
+    <value>$root/zookeeper</value>
+  </property>
+  <property>
+    <name>hbase.unsafe.stream.capability.enforce</name>
+    <value>false</value>
+  </property>
+</configuration>
+EOF
+
+echo Launching HBase ...
+./hbase-$version/bin/start-hbase.sh
+./hbase-$version/bin/hbase-daemon.sh start thrift
+
+cd $root

data/script/start-services

@@ -1,11 +1,10 @@
 #!/bin/bash
 
-echo 'Starting TokyoTyrant'
-/usr/sbin/ttserver -dmn -pid /tmp/ttserver.pid -log /tmp/ttserver.log -port 1978
+# Install and start CouchDB
+$(dirname $0)/start-couchdb
+
+# Install and start HBase
+$(dirname $0)/start-hbase
 
 # Waiting for servers to start
 sleep 3
-
-echo 'TokyoTyrant status'
-ps aux | grep ttserver
-cat /tmp/ttserver.log

data/script/update-feature-matrix

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require 'rubygems'
+require 'bundler/setup'
+Bundler.require(:doc)
+
+require 'yaml'
+
+output = "<table>\n"
+output << <<-TAB.lines.map(&:strip).join
+  <tr>
+    <th>Adapter</th><th>Required gems</th>
+    <th style="writing-mode:tb">MRI support<sup>1</sup></th>
+    <th style="writing-mode:tb">JRuby support<sup>1</sup></th>
+    <th style="writing-mode:tb">Multi-thread safe<sup>2</sup></th>
+    <th style="writing-mode:tb">Multi-process safe<sup>3</sup></th>
+    <th style="writing-mode:tb">Atomic increment<sup>4</sup></th>
+    <th style="writing-mode:tb">Atomic create<sup>5</sup></th>
+    <th style="writing-mode:tb">Native expires<sup>6</sup></th>
+    <th style="writing-mode:tb">Persistent</th>
+    <th style="writing-mode:tb">Key Traversal</th>
+    <th style="writing-mode:tb">Bulk read<sup>7</sup></th>
+    <th style="writing-mode:tb">Bulk write<sup>8</sup></th>
+    <th>Description</th>
+  </tr>
+TAB
+output << "\n\n"
+
+footnotes = {
+  "platform" => <<-EOF,
+  Indicates that the adapter is expected to work on this platform.  Most adapters will at least
+  work on MRI, but some are curently considered unstable, in which case they are not supported
+  on any platform.
+  EOF
+  "multi-thread safe" => <<-EOF,
+  Make adapters thread-safe by using `Moneta::Lock` or by passing the option `threadsafe: true` to
+  `Moneta#new`. There is also `Moneta::Pool` which can be used to share a store between multiple
+  threads if the store is multi-process safe. I recommend to add the option `:threadsafe` to
+  ensure thread-safety since for example under JRuby and Rubinius even the basic datastructures
+  are not thread safe due to the lack of a global interpreter lock (GIL). This differs from MRI
+  where some adapters might appear thread safe already but only due to the GIL.
+  EOF
+  "multi-process safe" => <<-EOF,
+  Share a Moneta store between multiple processes using `Moneta::Shared` (See below).
+  EOF
+  "atomic increment" => <<-EOF,
+  If a store provides atomic increment it can be used with `Moneta::Semaphore`. You can add weak
+  `#increment` support using the `Moneta::WeakIncrement` proxy.
+  EOF
+  "atomic create" => <<-EOF,
+  If a store provides atomic creation it can be used with `Moneta::Mutex`. You can add weak
+  `#create` support using the `Moneta::WeakCreate` proxy.
+  EOF
+  "native expires" => <<-EOF,
+  Add expiration support by using `Moneta::Expires` or by passing the option `expires: true` to
+  `Moneta#new`.
+  EOF
+  "bulk read" => <<-EOF,
+  This indicates that there is some performance gain when fetching
+  multiple values at once using `#values_at`/`#fetch_values` or `#slice`.
+  For instance, the `MGET` instruction in Redis, or the ability to retrieve
+  several rows in one query in SQL.
+  EOF
+  "bulk write" => <<-EOF
+  This indicates that there is some performance gain when storing multiple
+  key/value pairs at once using `#merge!`/`#update`.
+  EOF
+}
+
+YAML.parse_stream(File.read(File.join(File.dirname(File.dirname(__FILE__)), 'feature_matrix.yaml'))) do |document|
+  feature_group = document.to_ruby
+
+  output << %{<tr><th colspan="2">#{feature_group['group']}</th><th colspan="12"></th></tr>\n\n}
+
+  feature_group['notes'].each do |k,v|
+    footnotes[k] = v
+  end
+
+  feature_group['backends'].each do |backend|
+    output << '<tr>'
+    output << "<td>#{backend['adapter']}</td>"
+    output << "<td>#{backend['gems'] || '-'}</td>"
+
+    features = backend['features'] | (feature_group['features'] || [])
+    features += backend['platforms'] || []
+    %w{MRI JRuby threadsafe multiprocess increment create expires persist each_key bulk_read bulk_write}.each do |feature|
+      supported = if features.include? feature
+                    "yes"
+                  elsif backend['unknown'] && backend['unknown'].include?(feature)
+                    "unknown"
+                  else
+                    "no"
+                  end
+
+      note = if backend['notes'] && backend['notes'][feature]
+               "<sup>#{footnotes.keys.index(backend['notes'][feature]) + 1}</sup>"
+             else
+               ''
+             end
+
+      mark = case supported
+              when "yes"
+               "✓"
+              when "no"
+                "✗"
+              when "unknown"
+                '?'
+              end
+
+      colour = case supported
+               when "yes"
+                 '#5F5'
+               when "no"
+                 '#F44'
+               when "unknown"
+                 '#55F'
+               end
+
+      output << %{<td style="text-align:center;background:#{colour}">#{mark}#{note}</td>}
+    end
+
+    html_description = if backend['description']
+                         Kramdown::Document.new(backend['description']).to_html.match('<p>(.*)</p>')[1]
+                       else
+                         ''
+                       end
+    output << "<td>#{html_description}</td>"
+
+    output << '</tr>'
+    output << "\n\n"
+  end
+
+
+end
+
+output << "</table>\n\n"
+
+footnotes.each_value.each_with_index do |note, idx|
+  output << "#{idx+1}. #{note.lines.map(&:strip).join(" ")}\n"
+end
+
+readme = File.open('README.md', 'r+')
+new_readme = readme.read.sub(/(name="backend-matrix".*?\n).*?(------)/m, "\\1\n#{output}\n\\2")
+readme.rewind
+readme << new_readme
+readme.truncate(readme.tell)
+