moneta 0.6.0 → 0.7.0

data/Rakefile

@@ -1,60 +1,38 @@
-require 'rubygems'
-require 'rake/gempackagetask'
-require 'rubygems/specification'
-require 'spec/rake/spectask'
-require 'date'
-
-GEM = "moneta"
-GEM_VERSION = "0.6.0"
-AUTHOR = "Yehuda Katz"
-EMAIL = "wycats@gmail.com"
-HOMEPAGE = "http://www.yehudakatz.com"
-SUMMARY = "A unified interface to key/value stores"
-
-spec = Gem::Specification.new do |s|
-  s.name = GEM
-  s.version = GEM_VERSION
-  s.platform = Gem::Platform::RUBY
-  s.has_rdoc = true
-  s.extra_rdoc_files = ["README", "LICENSE", 'TODO']
-  s.summary = SUMMARY
-  s.description = s.summary
-  s.author = AUTHOR
-  s.email = EMAIL
-  s.homepage = HOMEPAGE
-  
-  # Uncomment this to add a dependency
-  # s.add_dependency "foo"
-  
-  s.require_path = 'lib'
-  s.autorequire = GEM
-  s.files = %w(LICENSE README Rakefile TODO) + Dir.glob("{lib,specs}/**/*")
+begin
+  require 'bundler'
+  Bundler::GemHelper.install_tasks
+rescue Exception
 end
 
-Rake::GemPackageTask.new(spec) do |pkg|
-  pkg.gem_spec = spec
-end
+task :test => %w(test:parallel test:non_parallel)
 
-desc "install the gem locally"
-task :install => [:package] do
-  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
-end
+# memcached and redis specs cannot be used in parallel
+# because of flushing and namespace lacking in redis
 
-desc "create a gemspec file"
-task :make_spec do
-  File.open("#{GEM}.gemspec", "w") do |file|
-    file.puts spec.to_ruby
+namespace :test do
+  task :parallel do
+    if defined?(JRUBY_VERSION)
+      puts 'No tests executed in parallel in JRuby'
+    else
+      specs = Dir['spec/*/*_spec.rb'].reject {|s| s =~ /memcached|redis|client|shared|riak/ }
+      sh("parallel_rspec -m 5 #{specs.join(' ')}")
+    end
   end
-end
 
-desc "Run all examples (or a specific spec with TASK=xxxx)"
-Spec::Rake::SpecTask.new('spec') do |t|
-  t.spec_opts  = ["-cfs"]
-  t.spec_files = begin
-    if ENV["TASK"] 
-      ENV["TASK"].split(',').map { |task| "spec/**/#{task}_spec.rb" }
+  task :non_parallel do
+    if defined?(JRUBY_VERSION)
+      # Run all tests in jruby non-parallel
+      sh('rspec spec/*/*_spec.rb')
     else
-      FileList['spec/**/*_spec.rb']
+      specs = Dir['spec/*/*_spec.rb'].select {|s| s =~ /memcached|redis|client|shared|riak/ }
+      sh("rspec #{specs.join(' ')}")
     end
   end
-end
+end
+
+task :benchmarks do
+  Dir.chdir('benchmarks')
+  ruby('run.rb')
+end
+
+task :default => :test

data/SPEC.md

@@ -0,0 +1,75 @@
+# Moneta Specification
+
+The purpose of the moneta specification is to create a general-purpose API for interacting with key-value stores. In general, libraries that need to interact with key-value stores should be able to specify that they can use any "moneta-compliant store".
+
+# Moneta Executable Specs
+
+Moneta ships with a set of executable specs which you can use to verify spec-compliance with your moneta adapter.
+
+# Requirements for a Moneta Adapter
+
+(See RFC 2119 for use of MUST, SHOULD, MAY, MUST NOT, and SHOULD NOT)
+
+A Moneta adapter must expose a class with the following characteristics:
+
+## Class Methods
+
+### <code>new(options[Hash] => {}) => Object</code>
+
+Return an instance of the moneta adapter, with the instance methods listed below. The <code>options</code> hash is a required parameter, and the adapter may specify whatever additional requirements it needs to properly instantiate it.
+
+## Instance Methods
+
+### <code>\[\](key[Object]) => Object</code>
+
+Return the value stored in the key-value-store under the provided key. Adapters MUST return a duplicate of the original value, and consumers should expect that adapters might serialize and deserialize the key and value. As a result, both the key and value MUST be objects that can be serialized using Ruby's Marshal system.
+
+### <code>\[\]=(key[Object], value[Object]) => Object(value)</code>
+
+Store the value in the key-value-store under the provided key. Adapters MAY serialize the value using Ruby's Marshal system, and MUST NOT store a reference to the original value in the store, unless Ruby disallows duplication of the original value. Adapters SHOULD NOT simply call <code>dup</code> on the value, unless the value stores no references to other Object. For example, an adapter MAY store a <code>dup</code> of a String, but SHOULD NOT store a <code>dup</code> of <code>["hello", "world"]</code>.
+
+### <code>fetch(key[Object], options[Hash] => {}, &block) => Object</code>
+
+Return the value stored in the key-value-store under the provided key. If no value is stored under the provided key, the adapter MUST yield to the block, and return the value. The adapter MUST NOT store the value returned from the block in the key-value-store.
+
+### <code>fetch(key[Object], value[Object], options[Hash] => {}) => Object</code>
+
+Return the value stored in the key-value-store under the provided key. If no value is stored under the provided key, the adapter MUST return the default value provided. The adapter MUST NOT store the default value in the key-value-store.
+
+### <code>delete(key[Object], options[Hash] => {}) => Object</code>
+
+Delete the value stored in the key-value-store for the key provided, and return the value previously stored there. After this operation, the key-value-store MUST behave as though no value was stored for the provided key.
+
+### <code>key?(key[Object], options[Hash] => {}) => [TrueClass, FalseClass]</code>
+
+Determine whether a value exists in the key-value-store for the key provided. If a value exists, the adapter MUST return <code>true</code>. Otherwise, the adapter MUST return <code>false</code>.
+
+### <code>store(key[Object], value[Object], options[Hash] => {}) => Object(value)</code>
+
+Behaves the same as <code>[]=</code>, but allows the client to send additional options which can be specified by the adapter (and which may be specified by extensions to this specification).
+
+### <code>increment(key[Object], amount[Integer] = 1, options[Hash] => {}) => Integer(value)</code>
+
+Increments a value atomically. This method is not supported by all stores and might raise a <code>NotImplementedError</code>.
+
+### <code>clear(options[Hash] => {})</code>
+
+Completely empty all keys and values from the key-value-store. Adapters MAY allow a namespace during initialization, which can scope this operation to a particular subset of keys. After calling <code>clear</code>, a <code>[]</code> operation MUST return nil for every possible key, and a <code>key?</code> query MUST return false for every possible key.
+
+# Additional Options Hashes
+
+The following methods may all take an additional Hash as a final argument. This allows the client to send additional options which can be specified by the adapter (and which may be specified by extensions to this specification).
+
+* fetch
+* load
+* store
+* delete
+* key?
+* increment
+* clear
+
+In the case of methods with optional arguments, the Hash MUST be provided as the final argument. Keys in this Hash MUST be Symbols.
+
+# Atomicity
+
+The base Moneta specification does not specify any atomicity guarantees. However, extensions to this spec may specify extensions that define additional guarantees for any of the defined operations.

data/benchmarks/run.rb

@@ -0,0 +1,195 @@
+#!/usr/bin/env ruby
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'benchmark'
+require 'moneta'
+
+begin
+  require 'dm-core'
+  DataMapper.setup(:default, :adapter => :in_memory)
+rescue LoadError => ex
+  puts "Failed to load DataMapper - #{ex.message}"
+end
+
+begin
+  server = Moneta::Server.new(Moneta.new(:Memory))
+rescue
+  puts "Failed to start Moneta server - #{ex.message}"
+end
+
+class Array
+  def random_index
+    rand(size)
+  end
+
+  def random_value
+    self[random_index]
+  end
+
+  def random_subset(n)
+    (1..n).map{|x| random_value }
+  end
+end
+
+stores = {
+  :ActiveRecord => { :connection => { :adapter  => 'sqlite3', :database => 'bench.activerecord' } },
+  :Cassandra => {},
+  :Client => {},
+  :Couch => {},
+  :DBM => { :file => 'bench.dbm' },
+  :DataMapper => { :setup => 'sqlite3:bench.datamapper' },
+  :File => { :dir => 'bench.file' },
+  :GDBM => { :file => 'bench.gdbm' },
+  :HBase => {},
+  :HBase => {},
+  :HashFile => { :dir => 'bench.hashfile' },
+  :LRUHash => {},
+  :LevelDB => { :dir => 'bench.leveldb' },
+  :LocalMemCache => { :file => 'bench.lmc' },
+  :MemcachedDalli => {},
+  :MemcachedNative => {},
+  :Memory => {},
+  :Mongo => {},
+  :PStore => { :file => 'bench.pstore' },
+  :Redis => {},
+  :Riak => {},
+  :SDBM => { :file => 'bench.sdbm' },
+  :Sequel => { :db => 'sqlite:/' },
+  :Sqlite => { :file => 'bench.sqlite' },
+  :YAML => { :file => 'bench.yaml' },
+}
+
+stats, keys, data, errors, summary = {}, [], [], [], []
+dict = 'ABCDEFGHIJKLNOPQRSTUVWXYZabcdefghijklnopqrstuvwxyz123456789'.split('')
+vlen_min, vlen_max, vlen_total, vlen_average = 99999, 0, 0, 0
+klen_min, klen_max, klen_total, klen_average = 99999, 0, 0, 0
+
+RUNS = 3
+KEYS = 100
+MIN_KEY_SIZE = 3
+MAX_KEY_SIZE = 64
+MIN_VALUE_SIZE = 1
+MAX_VALUE_SIZE = 1024 * 10
+
+puts '======================================================================'
+puts 'Comparison of write/read between Moneta Stores'
+puts '======================================================================'
+
+stores.each do |name, options|
+  begin
+    cache = Moneta.new(name, options.dup)
+    cache['test'] = 'test'
+  rescue Exception => ex
+    puts "#{name} not benchmarked - #{ex.message}"
+    stores.delete(name)
+  ensure
+    cache.close if cache
+  end
+end
+
+puts 'Data loading...'
+KEYS.times do |x|
+  klen = rand(MAX_KEY_SIZE - MIN_KEY_SIZE) + MIN_KEY_SIZE
+  vlen = rand(MAX_VALUE_SIZE - MIN_VALUE_SIZE) + MIN_VALUE_SIZE
+
+  key = dict.random_subset(klen).join
+  value = dict.random_subset(vlen).join
+  keys << key
+  data << [key, value]
+
+  vlen_min = value.size if value.size < vlen_min
+  vlen_max = value.size if value.size > vlen_max
+  vlen_total = vlen_total + value.size
+
+  klen_min = key.size if key.size < klen_min
+  klen_max = key.size if key.size > klen_max
+  klen_total = klen_total + key.size
+end
+vlen_average = vlen_total / KEYS
+
+puts '----------------------------------------------------------------------'
+puts "Total keys: #{keys.size}, unique: #{keys.uniq.size}"
+puts '----------------------------------------------------------------------'
+puts '                  Minimum    Maximum      Total    Average        xps '
+puts '----------------------------------------------------------------------'
+puts 'Key Length     % 10i % 10i % 10i % 10i ' % [klen_min, klen_max, klen_total, klen_average]
+puts 'Value Length   % 10i % 10i % 10i % 10i ' % [vlen_min, vlen_max, vlen_total, vlen_average]
+
+stores.each do |name, options|
+  begin
+    puts '======================================================================'
+    puts name
+    puts '----------------------------------------------------------------------'
+    cache = Moneta.new(name, options.dup)
+
+    stats[name] = {
+      :writes => [],
+      :reads => [],
+      :totals => [],
+      :averages => [],
+    }
+
+    RUNS.times do |round|
+      cache.clear
+      print "[#{round + 1}] W"
+      m1 = Benchmark.measure do
+        KEYS.times do
+          key, value = data.random_value
+          cache[key] = value
+        end
+      end
+      stats[name][:writes] << m1.real
+      print 'R '
+      m2 = Benchmark.measure do
+        KEYS.times do
+          key, value = data.random_value
+          res = cache[key]
+          errors << [name, key, value, res] unless res == value
+        end
+      end
+      stats[name][:reads] << m2.real
+      stats[name][:totals] << (m1.real + m2.real)
+      stats[name][:averages] << (m1.real + m2.real)
+    end
+    puts ''
+    puts '----------------------------------------------------------------------'
+    puts '                  Minimum    Maximum      Total    Average        xps '
+    puts '----------------------------------------------------------------------'
+    tcmin, tcmax, tctot, tcavg = 99999, 0, 0, 0
+    [:writes, :reads].each do |sname|
+      cmin, cmax, ctot, cavg = 99999, 0, 0, 0
+      stats[name][sname].each do |val|
+        cmin = val if val < cmin
+        tcmin = val if val < tcmin
+        cmax = val if val > cmax
+        tcmax = val if val > tcmax
+        ctot = ctot + val
+        tctot = tctot + val
+      end
+      cavg = ctot / RUNS
+      puts '%-14.14s % 10.4f % 10.4f % 10.4f % 10.4f % 10.4f ' % ["#{name} #{sname}", cmin, cmax, ctot, cavg, KEYS / cavg]
+    end
+    tcavg = tctot / (RUNS * 2)
+    puts '%-14.14s % 10.4f % 10.4f % 10.4f % 10.4f % 10.4f ' % ["#{name} avgs", tcmin, tcmax, tctot, tcavg, KEYS / tcavg]
+    summary << [name, tcmin, tcmax, tctot, tcavg, KEYS / tcavg]
+  rescue Exception => ex
+    puts "Failed to benchmark #{name} - #{ex.message}"
+  ensure
+    cache.close if cache
+  end
+end
+puts '----------------------------------------------------------------------'
+if errors.size > 0
+  puts "Errors : #{errors.size}"
+#  puts errors.inspect
+else
+  puts 'No errors in reading!'
+end
+puts '======================================================================'
+puts "Summary: #{RUNS} runs, #{KEYS} keys"
+puts '======================================================================'
+puts '                  Minimum    Maximum      Total    Average        xps '
+puts '----------------------------------------------------------------------'
+summary.each do |sry|
+  puts '%-14.14s % 10.4f % 10.4f % 10.4f % 10.4f % 10.4f ' % sry
+end

data/lib/action_dispatch/middleware/session/moneta_store.rb

@@ -0,0 +1,11 @@
+require 'rack/session/moneta'
+require 'action_dispatch/middleware/session/abstract_store'
+
+module ActionDispatch
+  module Session
+    class MonetaStore < Rack::Session::Moneta
+      include Compatibility
+      include StaleSessionCheck
+    end
+  end
+end

data/lib/active_support/cache/moneta_store.rb

@@ -0,0 +1,55 @@
+module ActiveSupport
+  module Cache
+    class MonetaStore < Store
+      def initialize(options = nil)
+        raise ArgumentError, 'Option :store is required' unless @store = options.delete(:store)
+        @store = ::Moneta.new(@store, :expires => true) if Symbol === @store
+        super(options)
+        extend Strategy::LocalCache
+      end
+
+      def increment(key, amount = 1, options = nil)
+        instrument(:increment, key, :amount => amount) do
+          @store.increment(key, amount, moneta_options(options))
+        end
+      end
+
+      def decrement(key, amount = 1, options = nil)
+        instrument(:decrement, key, :amount => amount) do
+          @store.increment(key, -amount, moneta_options(options))
+        end
+      end
+
+      def clear(options = nil)
+        instrument(:clear, nil, nil) do
+          @store.clear(moneta_options(options))
+        end
+      end
+
+      protected
+
+      def read_entry(key, options)
+        entry = @store.load(key, moneta_options(options))
+        entry && (ActiveSupport::Cache::Entry === entry ? entry : ActiveSupport::Cache::Entry.new(entry))
+      end
+
+      def write_entry(key, entry, options)
+        @store.store(key, entry, moneta_options(options))
+        true
+      end
+
+      def delete_entry(key, options)
+        @store.delete(key, moneta_options(options))
+        true
+      end
+
+      private
+
+      def moneta_options(options)
+        options ||= {}
+        options[:expires] = options.delete(:expires_in).to_i if options.include?(:expires_in)
+        options
+      end
+    end
+  end
+end

data/lib/moneta.rb

@@ -1,76 +1,130 @@
 module Moneta
-  module Expires
-    def check_expired(key)
-      if @expiration[key] && Time.now > @expiration[key]
-        @expiration.delete(key)
-        self.delete(key)
-      end
-    end
-
-    def key?(key)
-      check_expired(key)
-      super
-    end
-
-    def [](key)
-      check_expired(key)
-      super
-    end
-
-    def fetch(key, default = nil, &blk)
-      check_expired(key)
-      super
-    end
-
-    def delete(key)
-      check_expired(key)
-      super
-    end
+  autoload :Base,              'moneta/base'
+  autoload :Builder,           'moneta/builder'
+  autoload :Cache,             'moneta/cache'
+  autoload :Expires,           'moneta/expires'
+  autoload :Lock,              'moneta/lock'
+  autoload :Logger,            'moneta/logger'
+  autoload :Mixins,            'moneta/mixins'
+  autoload :Net,               'moneta/net'
+  autoload :OptionMerger,      'moneta/optionmerger'
+  autoload :Proxy,             'moneta/proxy'
+  autoload :Server,            'moneta/server'
+  autoload :Shared,            'moneta/shared'
+  autoload :Stack,             'moneta/stack'
+  autoload :Transformer,       'moneta/transformer'
+  autoload :Wrapper,           'moneta/wrapper'
 
-    def update_key(key, options)
-      update_options(key, options)
-    end
-
-    def store(key, value, options = {})
-      ret = super(key, value)
-      update_options(key, options)
-      ret
-    end
-
-    private
-    def update_options(key, options)
-      if options[:expires_in]
-        @expiration[key] = (Time.now + options[:expires_in])
-      end
-    end
+  module Adapters
+    autoload :ActiveRecord,    'moneta/adapters/activerecord'
+    autoload :Cassandra,       'moneta/adapters/cassandra'
+    autoload :Client,          'moneta/adapters/client'
+    autoload :Cookie,          'moneta/adapters/cookie'
+    autoload :Couch,           'moneta/adapters/couch'
+    autoload :DataMapper,      'moneta/adapters/datamapper'
+    autoload :DBM,             'moneta/adapters/dbm'
+    autoload :File,            'moneta/adapters/file'
+    autoload :Fog,             'moneta/adapters/fog'
+    autoload :GDBM,            'moneta/adapters/gdbm'
+    autoload :HBase,           'moneta/adapters/hbase'
+    autoload :LevelDB,         'moneta/adapters/leveldb'
+    autoload :LocalMemCache,   'moneta/adapters/localmemcache'
+    autoload :LRUHash,         'moneta/adapters/lruhash'
+    autoload :Memcached,       'moneta/adapters/memcached'
+    autoload :MemcachedDalli,  'moneta/adapters/memcached_dalli'
+    autoload :MemcachedNative, 'moneta/adapters/memcached_native'
+    autoload :Memory,          'moneta/adapters/memory'
+    autoload :Mongo,           'moneta/adapters/mongo'
+    autoload :Null,            'moneta/adapters/null'
+    autoload :PStore,          'moneta/adapters/pstore'
+    autoload :Redis,           'moneta/adapters/redis'
+    autoload :Riak,            'moneta/adapters/riak'
+    autoload :SDBM,            'moneta/adapters/sdbm'
+    autoload :Sequel,          'moneta/adapters/sequel'
+    autoload :Sqlite,          'moneta/adapters/sqlite'
+    autoload :TokyoCabinet,    'moneta/adapters/tokyocabinet'
+    autoload :YAML,            'moneta/adapters/yaml'
   end
 
-  module StringExpires
-    include Expires
-
-    def check_expired(key)
-      if @expiration[key] && Time.now > Time.at(@expiration[key].to_i)
-        @expiration.delete(key)
-        delete(key)
-      end
+  # Create new Moneta store with default proxies
+  # which works in most cases if you don't want fine
+  # control over the proxy chain. It uses Marshal on the
+  # keys and values. Use Moneta#build if you want to have fine control!
+  #
+  # @param [Symbol] name Name of adapter (See Moneta::Adapters)
+  # @param [Hash] options
+  #
+  # Options:
+  # * :expires - If true or integer, ensure that store supports expiration by inserting
+  #   Moneta::Expires if the underlying adapter doesn't support it natively
+  # * :threadsafe - If true, ensure that the store is thread safe by inserting Moneta::Lock
+  # * :logger - If true or Hash, add logger to chain (Hash is passed to logger as options)
+  # * :compress - If true, compress value with zlib, or specify custom compress, e.g. :quicklz
+  # * :serializer - Serializer used for key and value, disable with nil (default :marshal)
+  # * :key_serializer - Serializer used for key, disable with nil (default options[:serializer] if not provided)
+  # * :value_serializer - Serializer used for key, disable with nil (default options[:serializer] if not provided)
+  # * :prefix - Key prefix used for namespacing (default none)
+  # * All other options passed to the adapter
+  #
+  # Supported adapters:
+  # * :HashFile (Store which spreads the entries using a md5 hash, e.g. cache/42/391dd7535aebef91b823286ac67fcd)
+  # * :File (normal file store)
+  # * :Memcached (Memcached store)
+  # * ... (All other adapters from Moneta::Adapters)
+  def self.new(name, options = {})
+    expires = options.delete(:expires)
+    logger = options.delete(:logger)
+    threadsafe = options.delete(:threadsafe)
+    compress = options.delete(:compress)
+    serializer = options.include?(:serializer) ? options.delete(:serializer) : :marshal
+    key_serializer = options.include?(:key_serializer) ? options.delete(:key_serializer) : serializer
+    value_serializer = options.include?(:value_serializer) ? options.delete(:value_serializer) : serializer
+    transformer = { :key => [key_serializer, :prefix], :value => [value_serializer], :prefix => options.delete(:prefix) }
+    transformer[:value] << (Symbol === compress ? compress : :zlib) if compress
+    raise ArgumentError, 'Name must be Symbol' unless Symbol === name
+    case name
+    when :Sequel, :ActiveRecord, :Couch, :DataMapper
+      # Sequel accept only base64 keys and values
+      # FIXME: Couch should work only with :marshal but this raises an error on 1.9
+      transformer[:key] << :base64
+      transformer[:value] << :base64
+    when :Memcached, :MemcachedDalli, :MemcachedNative
+      # Memcached accept only base64 keys, expires already supported
+      options[:expires] = expires if Integer === expires
+      expires = false
+      transformer[:key] << :base64
+    when :PStore, :YAML, :Null
+      # For PStore and YAML only the key has to be a string
+      transformer.delete(:value) if transformer[:value] == [:marshal]
+    when :HashFile
+      # Use spreading hashes
+      transformer[:key] << :md5 << :spread
+      name = :File
+    when :File
+      # Use escaping
+      transformer[:key] << :escape
+    when :Cassandra, :Redis
+      # Expires already supported
+      options[:expires] = expires if Integer === expires
+      expires = false
     end
-
-    private
-    def update_options(key, options)
-      if options[:expires_in]
-        @expiration[key] = (Time.now + options[:expires_in]).to_i.to_s
-      end
+    build do
+      use :Logger, Hash === logger ? logger : {} if logger
+      use :Expires, :expires => (Integer === expires ? expires : nil) if expires
+      use :Transformer, transformer
+      use :Lock if threadsafe
+      adapter name, options
     end
   end
-  
-  module Defaults
-    def fetch(key, value = nil)
-      value ||= block_given? ? yield(key) : default
-      self[key] || value
-    end
-    
-    def store(key, value, options = {})
-      self[key] = value
-    end        
+
+  # Build your own store chain!
+  #
+  # @example Moneta builder
+  #   Moneta.build do
+  #     use :Expires
+  #     adapter :Memory
+  #   end
+  def self.build(&block)
+    Builder.new(&block).build.last
   end
 end