moneta 1.0.0 → 1.5.1

data/spec/helper.rb

@@ -1,9 +1,15 @@
-require 'rspec'
 require 'rspec/core/formatters/base_text_formatter'
-require 'rspec/retry'
 require 'moneta'
 require 'fileutils'
-require 'monetaspecs'
+require 'tmpdir'
+
+ENV['RANTLY_VERBOSE'] ||= '0'
+
+require 'rspec/retry'
+require 'rantly'
+require 'rantly/rspec_extensions'
+# rantly/shrinks
+require 'timecop'
 
 class MonetaParallelFormatter < RSpec::Core::Formatters::BaseTextFormatter
   def start(*args)
@@ -47,13 +53,335 @@ class MonetaParallelFormatter < RSpec::Core::Formatters::BaseTextFormatter
   end
 end
 
+class MonetaSpecs
+  KEYS = {
+    'nil' => [:choose, nil, 0],
+    'integer' => :integer,
+    'float' => :float,
+    'boolean' => :boolean,
+    'string' => proc{ sized(range 5, 10){ string(:alnum) } },
+    'path' => proc{ array(range 2, 3){ sized(range 5, 10){ string(:alpha) } }.join('/') },
+    'binary' => [:string, :cntrl],
+    'object' => proc{ choose Value.new(:objkey1), Value.new(:objkey2) },
+    'hash' => proc{ dict(2){ sized(range 5, 10){ [string(:alnum), string(:alnum)] } } }
+  }
+
+  VALUES = {
+    'nil' => [:literal, nil],
+    'integer' => :integer,
+    'float' => :float,
+    'boolean' => :boolean,
+    'string' => [:string, :alnum],
+    'binary' => [:string, :cntrl],
+    'object' => proc{ choose Value.new(:objval1), Value.new(:objval2) },
+    'hash' => proc{ dict{ [string(:alnum), array(2){ choose(string(:alnum), integer, dict{ [string(:alnum), integer] }) }] } },
+    'smallhash' => proc{ dict(2){ sized(range 5, 10){ [string(:alnum), string(:alnum)] } } }
+  }
+
+  attr_reader :key, :value, :specs, :features
+
+  def initialize(options = {})
+    @specs = options.delete(:specs).to_a
+    @features = @specs & [:expires, :expires_native, :increment, :each_key, :create]
+    @key = options.delete(:key)     || %w(object string binary hash boolean nil integer float)
+    @value = options.delete(:value) || %w(object string binary hash boolean nil integer float)
+  end
+
+  def new(options)
+    self.class.new({specs: specs, key: key, value: value}.merge(options))
+  end
+
+  def with_keys(*keys)
+    new(key: self.key | keys.map(&:to_s))
+  end
+
+  def without_keys(*keys)
+    new(key: self.key - keys.map(&:to_s))
+  end
+
+  def with_values(*values)
+    new(value: self.value | values.map(&:to_s))
+  end
+
+  def without_values(*values)
+    new(value: self.value - values.map(&:to_s))
+  end
+
+  def without_keys_or_values(*types)
+    without_keys(*types).without_values(*types)
+  end
+
+  def without_path
+    new(key: key - %w(path))
+  end
+
+  def stringvalues_only
+    new(value: %w(string))
+  end
+
+  def simplekeys_only
+    new(key: %w(string hash integer))
+  end
+
+  def simplevalues_only
+    new(value: %w(string hash integer))
+  end
+
+  def without_increment
+    new(specs: specs - [:increment, :concurrent_increment] + [:not_increment])
+  end
+
+  def without_large
+    new(specs: specs - [:store_large]).instance_exec do
+      if value.include? 'hash'
+        without_values(:hash).with_values(:smallhash)
+      else
+        self
+      end
+    end
+  end
+
+  def without_concurrent
+    new(specs: specs - [:concurrent_increment, :concurrent_create])
+  end
+
+  def without_persist
+    new(specs: specs - [:persist, :multiprocess, :concurrent_increment, :concurrent_create] + [:not_persist])
+  end
+
+  def without_multiprocess
+    new(specs: specs - [:multiprocess, :concurrent_increment, :concurrent_create])
+  end
+
+  def with_expires
+    a = specs.dup
+    if a.include?(:transform_value)
+      a.delete(:transform_value)
+      a << :transform_value_expires
+    end
+    a << :create_expires if a.include?(:create)
+    a << :expires
+    new(specs: a)
+  end
+
+  def with_native_expires
+    a = specs.dup
+    a << :create_expires if a.include?(:create)
+    new(specs: a + [:expires])
+  end
+
+  def without_marshallable
+    new(specs: specs - [:marshallable_value, :marshallable_key])
+  end
+
+  def without_transform
+    new(specs: specs - [:marshallable_value, :marshallable_key, :transform_value])
+  end
+
+  def returnsame
+    new(specs: specs - [:returndifferent] + [:returnsame])
+  end
+
+  def without_marshallable_key
+    new(specs: specs - [:marshallable_key])
+  end
+
+  def without_marshallable_value
+    new(specs: specs - [:marshallable_value])
+  end
+
+  def without_store
+    new(specs: specs - [:store, :store_large, :transform_value, :marshallable_value])
+  end
+
+  def with_default_expires
+    new(specs: specs + [:default_expires])
+  end
+
+  def with_each_key
+    new(specs: specs - [:not_each_key] | [:each_key])
+  end
+
+  def without_create
+    new(specs: specs - [:create, :concurrent_create, :create_expires] + [:not_create])
+  end
+end
+
+ADAPTER_SPECS = MonetaSpecs.new(
+  specs: [:null, :store, :returndifferent,
+    :increment, :concurrent_increment, :concurrent_create, :persist, :multiprocess,
+    :create, :features, :store_large, :not_each_key],
+  key: %w(string path),
+  value: %w(string path binary))
+
+NATIVE_EXPIRY_SPECS = MonetaSpecs.new(
+  specs: [:create, :expires, :create_expires],
+  key: %w(string path),
+  value: %w(string path binary))
+
+STANDARD_SPECS = MonetaSpecs.new(
+  specs: [:null, :store, :returndifferent,
+    :marshallable_key, :marshallable_value, :transform_value, :increment,
+    :concurrent_increment, :concurrent_create, :persist, :multiprocess, :create,
+    :features, :store_large, :not_each_key])
+
+TRANSFORMER_SPECS = MonetaSpecs.new(
+  specs: [:null, :store, :returndifferent,
+    :transform_value, :increment, :create, :features, :store_large,
+    :not_each_key])
+
+module MonetaHelpers
+  module ClassMethods
+
+    def moneta_store store_name, options={}, &block
+      name = self.description
+      builder = proc do
+        if block
+          options = instance_exec(&block)
+        end
+
+        Moneta.new(store_name, options.merge(logger: {file: File.join(tempdir, "#{name}.log")}))
+      end
+
+      include_context :setup_moneta_store, builder
+    end
+
+    def moneta_build &block
+      include_context :setup_moneta_store, block
+    end
+
+    def moneta_loader &block
+      before do
+        @moneta_value_loader = block
+      end
+    end
+
+    def moneta_specs specs
+      let(:features){ specs.features }
+      let(:keys_meta) do
+        [:branch, *specs.key.map{ |k| MonetaSpecs::KEYS[k] }.compact]
+      end
+      let(:values_meta) do
+        [:branch, *specs.value.map{ |k| MonetaSpecs::VALUES[k] }.compact]
+      end
+
+      # Used by tests that rely on MySQL.  These env vars can be used if you
+      # want to run the tests but don't want to grant root access to moneta
+      let(:mysql_host) { ENV['MYSQL_HOST'] || 'localhost' }
+      let(:mysql_port) { ENV['MYSQL_TCP_PORT'] || '3306' }
+      let(:mysql_username) { ENV['MONETA_MYSQL_USERNAME'] || 'root' }
+      let(:mysql_password) { ENV['MONETA_MYSQL_PASSWORD'] }
+      let(:mysql_database1) { ENV['MONETA_MYSQL_DATABASE1'] || 'moneta' }
+      let(:mysql_database2) { ENV['MONETA_MYSQL_DATABASE2'] || 'moneta2' }
+
+      let(:postgres_username) { ENV['PGUSER'] || 'postgres' }
+      let(:postgres_password) { ENV['PGPASSWORD'] }
+      let(:postgres_database1) { ENV['MONETA_POSTGRES_DATABSASE1'] || 'moneta1' }
+      let(:postgres_database2) { ENV['MONETA_POSTGRES_DATABSASE1'] || 'moneta2' }
+
+      let(:couch_login) { ENV['COUCH_LOGIN'] || 'admin' }
+      let(:couch_password) { ENV['COUCH_PASSWORD'] || 'password' }
+
+      let(:redis_host) { ENV.fetch('REDIS_HOST', 'localhost') }
+      let(:redis_port) { ENV.fetch('REDIS_PORT', '6379') }
+
+      before do
+        store = new_store
+        store.clear
+        store.close
+      end
+
+      specs.specs.sort.each do |s|
+        context "#{s} feature" do
+          include_examples(s)
+        end
+      end
+    end
+
+    def use_timecop
+      before { @timecop = true }
+      after { Timecop.return }
+    end
+  end
+
+  module InstanceMethods
+    def tempdir
+      @moneta_tempdir ||= Dir.mktmpdir
+    end
+
+    def new_store
+      instance_eval(&@moneta_store_builder)
+    end
+
+    def store
+      @store ||= new_store
+    end
+
+    def load_value value
+      if @moneta_value_loader
+        @moneta_value_loader.call value
+      else
+        Marshal.load(value)
+      end
+    end
+
+    def moneta_property_of(keys: 0, values: 0)
+      keys_meta = self.keys_meta
+      values_meta = self.values_meta
+      property_of do
+        key_values = keys.times.map { call(keys_meta) }
+        guard key_values.uniq.length == key_values.length
+
+        value_values = values.times.map { call(values_meta) }
+        guard value_values.uniq.length == value_values.length
+
+        Struct.new(:keys, :values).new(key_values, value_values)
+      end
+    end
+
+    def advance(seconds)
+      return if seconds < 0
+      if @timecop
+        Timecop.freeze(Time.now + seconds)
+      else
+        sleep seconds
+      end
+    end
+
+    def time_till_next(tick)
+      now = Time.now.to_f
+      tick - (now % tick)
+    end
+
+    # advance to the moment just after a tick. 1e-2 is needed in some
+    # environments (JRuby) to be able to pass the "not in earlier half" test.
+    def advance_next_tick
+      tick = t_res
+      offset = time_till_next(tick) + 1e-2
+      advance offset
+      raise "not in earlier half of tick" unless Time.now.to_f % tick < tick / 2.0
+    end
+  end
+end
+
 RSpec.configure do |config|
   config.verbose_retry = true
   config.color = true
-  config.tty = true
-  config.formatter = ENV['PARALLEL_TESTS'] ? MonetaParallelFormatter : :progress
+  #config.tty = true
+  #config.formatter = ENV['PARALLEL_TESTS'] ? MonetaParallelFormatter : :progress
+  config.silence_filter_announcements = true if ENV['PARALLEL_TESTS']
+
+  # Allow "should" syntax as well as "expect"
+  config.expect_with(:rspec) { |c| c.syntax = [:should, :expect] }
+
+  config.extend MonetaHelpers::ClassMethods
+  config.include MonetaHelpers::InstanceMethods
 end
 
+# FIXME: Get rid of this once raise_error expectations no longer generate
+# warnings
+RSpec::Expectations.configuration.on_potential_false_positives = :nothing
+
 # Disable jruby stdout pollution by memcached
 if defined?(JRUBY_VERSION)
   require 'java'
@@ -82,47 +410,6 @@ class Value
   end
 end
 
-def start_restserver
-  require 'rack'
-  require 'webrick'
-  require 'rack/moneta_rest'
-
-  # Keep webrick quiet
-  ::WEBrick::HTTPServer.class_eval do
-    def access_log(config, req, res); end
-  end
-  ::WEBrick::BasicLog.class_eval do
-    def log(level, data); end
-  end
-
-  Thread.start do
-    Rack::Server.start(:app => Rack::Builder.app do
-                         use Rack::Lint
-                         map '/moneta' do
-                           run Rack::MonetaRest.new(:store => :Memory)
-                         end
-                       end,
-                       :environment => :none,
-                       :server => :webrick,
-                       :Port => 8808)
-  end
-  sleep 1
-end
-
-def start_server(*args)
-  server = Moneta::Server.new(*args)
-  Thread.new { server.run }
-  sleep 0.1 until server.running?
-rescue Exception => ex
-  puts "Failed to start server - #{ex.message}"
-end
-
-def make_tempdir
-  # Expand path since datamapper needs absolute path in setup
-  tempdir = File.expand_path(File.join(File.dirname(__FILE__), 'tmp'))
-  FileUtils.mkpath(tempdir)
-  tempdir
-end
 
 def marshal_error
   # HACK: Marshalling structs in rubinius without class name throws
@@ -134,44 +421,24 @@ def marshal_error
   end
 end
 
-class InitializeStore
-  def initialize(&block)
-    instance_eval(&block)
-    store = new_store
-    store['foo'] = 'bar'
-    store.clear
-    store.close
-  end
-
-  def method_missing(*args)
-  end
-end
-
-def describe_moneta(name, &block)
-  begin
-    InitializeStore.new(&block)
-    describe(name, &block)
-  rescue LoadError => ex
-    puts "\e[31mTest #{name} not executed: #{ex.class} - #{ex.message}\e[0m"
-  rescue Exception => ex
-    puts "\e[31mTest #{name} not executed: #{ex.class} - #{ex.message}\e[0m"
-    puts ex.backtrace.join("\n")
-  end
-end
-
-shared_context 'setup_store' do
-  def store
-    @store ||= new_store
-  end
 
+RSpec.shared_context :setup_moneta_store do |builder|
   before do
-    store.clear
+    @moneta_store_builder = builder
   end
 
   after do
-    if store
-      store.close.should == nil
+    if @store
+      @store.close.should == nil
       @store = nil
     end
   end
+
+  after :all do
+    if @moneta_tempdir
+      FileUtils.remove_dir(@moneta_tempdir)
+    end
+  end
 end
+
+Dir['./spec/features/*.rb'].each{ |rb| require rb }

data/spec/moneta/adapters/activerecord/adapter_activerecord_existing_connection_spec.rb

@@ -0,0 +1,61 @@
+describe 'adapter_activerecord_existing_connection', adapter: :ActiveRecord, mysql: true do
+  before :all do
+    require 'active_record'
+  end
+
+  before do
+    default_env = ActiveRecord::ConnectionHandling::DEFAULT_ENV.call
+    ActiveRecord::Base.configurations = {
+      default_env => {
+        'adapter' => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'),
+        'host' => mysql_host,
+        'port' => mysql_port,
+        'database' => mysql_database1,
+        'username' => mysql_username,
+        'password' => mysql_password
+      }
+    }
+
+    ActiveRecord::Base.establish_connection
+  end
+
+  moneta_build do
+    Moneta::Adapters::ActiveRecord.new(table: 'adapter_activerecord_existing_connection')
+  end
+
+  moneta_specs ADAPTER_SPECS.with_each_key
+
+  # This is based on
+  # https://github.com/jjb/rails/blob/ar-connection-management-guide/guides/source/active_record_connection_management.md
+  it "supports use on a forking web server", unsupported: !Process.respond_to?(:fork) do
+    store['a'] = 'b'
+
+    # Before forking, the connection pool is disconnected so that the
+    # forked processes don't use the same connections.
+    ActiveRecord::Base.connection_pool.disconnect!
+
+    pids = 8.times.map do
+      Process.fork do
+        # Connection is then reestablished in the forked process
+        ActiveRecord::Base.establish_connection
+
+        exit 1 unless store['a'] == 'b'
+
+        store[Process.pid.to_s] = '1'
+        exit 1 unless store[Process.pid.to_s] == '1'
+      end
+    end
+
+    pids.each do |pid|
+      pid2, status = Process.wait2(pid)
+      expect(pid2).to eq pid
+      expect(status.exitstatus).to eq 0
+    end
+
+    # Check that the stores were all operating on the same DB
+    ActiveRecord::Base.establish_connection
+    pids.each do |pid|
+      expect(store[pid.to_s]).to eq '1'
+    end
+  end
+end

data/spec/moneta/adapters/activerecord/adapter_activerecord_spec.rb

@@ -0,0 +1,126 @@
+describe 'adapter_activerecord', adapter: :ActiveRecord do
+  activerecord_specs = ADAPTER_SPECS.with_values(:nil).with_each_key
+
+  shared_examples :adapter_activerecord do |specs|
+    moneta_build do
+      Moneta::Adapters::ActiveRecord.new(
+        table: 'adapter_activerecord',
+        connection: connection1)
+    end
+
+
+    moneta_specs specs
+
+    it 'updates an existing key/value' do
+      store['foo/bar'] = '1'
+      store['foo/bar'] = '2'
+      store.with_connection do |conn|
+        count = conn.select_value \
+          store.table.
+            where(store.table[:k].eq('foo/bar')).
+            project(store.table[:k].count)
+        expect(count).to eq 1
+      end
+    end
+
+    it 'supports different tables same database' do
+      store1 = Moneta::Adapters::ActiveRecord.new(
+        table: 'adapter_activerecord1',
+        connection: connection1)
+      store2 = Moneta::Adapters::ActiveRecord.new(
+        table: 'adapter_activerecord2',
+        connection: connection1)
+
+      store1['key'] = 'value1'
+      store2['key'] = 'value2'
+      store1['key'].should == 'value1'
+      store2['key'].should == 'value2'
+
+      store1.close
+      store2.close
+    end
+
+    it 'supports different databases same table' do
+      store1 = Moneta::Adapters::ActiveRecord.new(
+        table: 'adapter_activerecord',
+        connection: connection1)
+      store2 = Moneta::Adapters::ActiveRecord.new(
+        table: 'adapter_activerecord',
+        connection: connection2)
+
+      store1['key'] = 'value1'
+      store2['key'] = 'value2'
+      store1['key'].should == 'value1'
+      store2['key'].should == 'value2'
+
+      store1.close
+      store2.close
+    end
+  end
+
+  context "with MySQL", mysql: true do
+    let(:connection1) do
+      {
+        adapter: (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'),
+        host: mysql_host,
+        port: mysql_port,
+        database: mysql_database1,
+        username: mysql_username,
+        password: mysql_password
+      }
+    end
+
+    let(:connection2) do
+      {
+        adapter: (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'),
+        host: mysql_host,
+        port: mysql_port,
+        database: mysql_database2,
+        username: mysql_username,
+        password: mysql_password
+      }
+    end
+
+    include_examples :adapter_activerecord, activerecord_specs
+  end
+
+  context "with PostgreSQL", postgres: true do
+    let(:connection1) do
+      {
+        adapter: (defined?(JRUBY_VERSION) ? 'jdbcpostgresql' : 'postgresql'),
+        database: postgres_database1,
+        username: postgres_username,
+        password: postgres_password
+      }
+    end
+
+    let(:connection2) do
+      {
+        adapter: (defined?(JRUBY_VERSION) ? 'jdbcpostgresql' : 'postgresql'),
+        database: postgres_database2,
+        username: postgres_username,
+        password: postgres_password
+      }
+    end
+
+    include_examples :adapter_activerecord, activerecord_specs
+  end
+
+  context "with SQLite", sqlite: true do
+    let(:connection1) do
+      {
+        adapter: (defined?(JRUBY_VERSION) ? 'jdbcsqlite3' : 'sqlite3'),
+        database: File.join(tempdir, 'adapter_activerecord1.db')
+      }
+    end
+
+    let(:connection2) do
+      {
+        adapter: (defined?(JRUBY_VERSION) ? 'jdbcsqlite3' : 'sqlite3'),
+        database: File.join(tempdir, 'adapter_activerecord2.db')
+      }
+    end
+
+    include_examples :adapter_activerecord, activerecord_specs.without_concurrent
+  end
+end

data/spec/moneta/adapters/activerecord/standard_activerecord_spec.rb

@@ -0,0 +1,21 @@
+describe "standard_activerecord", adapter: :ActiveRecord, mysql: true do
+  moneta_store :ActiveRecord do
+    {
+      table: 'standard_activerecord',
+      connection: {
+        adapter: (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'),
+        host: mysql_host,
+        port: mysql_port,
+        database: mysql_database1,
+        username: mysql_username,
+        password: mysql_password
+      }
+    }
+  end
+
+  moneta_loader do |value|
+    ::Marshal.load(value.unpack('m').first)
+  end
+
+  moneta_specs STANDARD_SPECS.with_each_key
+end

data/spec/moneta/adapters/activerecord/standard_activerecord_with_expires_spec.rb

@@ -0,0 +1,27 @@
+describe "standard_activerecord_with_expires", adapter: :ActiveRecord, mysql: true do
+  let(:t_res) { 0.125 }
+  let(:min_ttl) { t_res }
+  use_timecop
+
+  moneta_store :ActiveRecord do
+    {
+      table: 'standard_activerecord_with_expires',
+      connection: {
+        adapter: (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'),
+        host: mysql_host,
+        port: mysql_port,
+        database: mysql_database1,
+        username: mysql_username,
+        password: mysql_password
+      },
+      expires: true
+    }
+  end
+
+
+  moneta_loader do |value|
+    ::Marshal.load(value.unpack('m').first)
+  end
+
+  moneta_specs STANDARD_SPECS.with_expires.with_each_key
+end