dkastner-moneta 0.7.0 → 1.0.0

data/.gitignore

@@ -0,0 +1,3 @@
+moneta-*.gem
+.idea
+*.swp

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/README

@@ -1,3 +1,12 @@
+Moneta::Monetas
+===============
+
+  store = Moneta::Monetas.new(:stores => [
+    Moneta::Memory.new,
+    Moneta::BasicFile.new(...)])
+
+  Read operations select a random cache, write operations write to all caches.
+
 Moneta: A unified interface for key/value stores
 ================================================
 
@@ -53,4 +62,46 @@ The API:
 #update_key(key, options)::     updates an existing key with a new :expires_in option.
                                 if the key has already expired, it will not be updated.
 
-#clear::                        clear all keys in this store
+#clear::                        clear all keys in this store
+
+
+Moneta::ActiveRecord
+====================
+You need a migration:
+
+  class AddMonetaTable < ActiveRecord::Migration
+    # Only needed if you want to keep your cache in a separate database
+    class TheStore < ActiveRecord::Base
+      establish_connection "#{Rails.env}_store"
+    end
+
+    def self.connection
+      TheStore.connection
+    end
+
+    def self.up
+      create_table 'moneta_store', :id => false do |t|
+        t.string :key, :null => false
+        t.text :value
+      end
+
+      if mysql?
+        execute "alter table moneta_store modify column `key` varchar(255) primary key"
+      else
+        execute "alter table moneta_store add primary key (key)"
+      end
+    end
+
+    def self.down
+      drop_table 'moneta_store'
+    end
+
+    def self.mysql?
+      ActiveRecord::Base.configurations[Rails.env]['adapter'] == 'mysql'
+    end
+  end
+
+Initialize like this:
+
+  store = Moneta::ActiveRecord.new(:connection => ActiveRecord::Base.configurations["#{Rails.env}_store"])
+

data/Rakefile

@@ -1,60 +1,13 @@
 require 'rubygems'
-require 'rake/gempackagetask'
-require 'rubygems/specification'
-require 'spec/rake/spectask'
-require 'date'
+require 'bundler'
+Bundler.setup
 
-GEM = "moneta"
-GEM_VERSION = "0.5.0"
-AUTHOR = "Yehuda Katz"
-EMAIL = "wycats@gmail.com"
-HOMEPAGE = "http://www.yehudakatz.com"
-SUMMARY = "A unified interface to key/value stores"
+Bundler::GemHelper.install_tasks
 
-spec = Gem::Specification.new do |s|
-  s.name = GEM
-  s.version = GEM_VERSION
-  s.platform = Gem::Platform::RUBY
-  s.has_rdoc = true
-  s.extra_rdoc_files = ["README", "LICENSE", 'TODO']
-  s.summary = SUMMARY
-  s.description = s.summary
-  s.author = AUTHOR
-  s.email = EMAIL
-  s.homepage = HOMEPAGE
-  
-  # Uncomment this to add a dependency
-  # s.add_dependency "foo"
-  
-  s.require_path = 'lib'
-  s.autorequire = GEM
-  s.files = %w(LICENSE README Rakefile TODO) + Dir.glob("{lib,specs}/**/*")
-end
-
-Rake::GemPackageTask.new(spec) do |pkg|
-  pkg.gem_spec = spec
-end
-
-desc "install the gem locally"
-task :install => [:package] do
-  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
-end
-
-desc "create a gemspec file"
-task :make_spec do
-  File.open("#{GEM}.gemspec", "w") do |file|
-    file.puts spec.to_ruby
-  end
+require 'rspec/core/rake_task'
+desc "Run all examples (or a specific spec with TASK=xxxx)"
+RSpec::Core::RakeTask.new(:examples) do |c|
+  c.rspec_opts = '-Ispec'
 end
 
-desc "Run all examples (or a specific spec with TASK=xxxx)"
-Spec::Rake::SpecTask.new('spec') do |t|
-  t.spec_opts  = ["-cfs"]
-  t.spec_files = begin
-    if ENV["TASK"] 
-      ENV["TASK"].split(',').map { |task| "spec/**/#{task}_spec.rb" }
-    else
-      FileList['spec/**/*_spec.rb']
-    end
-  end
-end
+task :default => :examples

data/SPEC.markdown

@@ -0,0 +1,99 @@
+# Moneta Specification
+
+The purpose of the moneta specification is to create a general-purpose API for interacting with key-value stores. In general, libraries that need to interact with key-value stores should be able to specify that they can use any "moneta-compliant store".
+
+# Moneta Executable Specs
+
+Moneta ships with a set of executable specs which you can use to verify spec-compliance with your moneta adapter.
+
+# Moneta Library
+
+Moneta ships with proof-of-concept adapters for over a dozen key-value stores, including in-memory, memcache, database-backed and S3. These adapters are meant as proofs-of-concept, and while the moneta project intends to maintain them (and will accept patches to improve them), you should not consider them the core of the project.
+
+# Requirements for a Moneta Adapter
+
+(See RFC 2119 for use of MUST, SHOULD, MAY, MUST NOT, and SHOULD NOT)
+
+A Moneta adapter must expose a class with the following characteristics:
+
+## Class Methods
+
+### <code>new(options[Hash]) => Object</code>
+
+Return an instance of the moneta adapter, with the instance methods listed below. The <code>options</code> hash is a required parameter, and the adapter may specify whatever additional requirements it needs to properly instantiate it.
+
+## Instance Methods
+
+### <code>\[\](key[Object]) => Object</code>
+
+Return the value stored in the key-value-store under the provided key. Adapters MUST return a duplicate of the original value, and consumers should expect that adapters might serialize and deserialize the key and value. As a result, both the key and value MUST be objects that can be serialized using Ruby's Marshal system.
+
+### <code>\[\]=(key[Object], value[Object]) => Object(value)</code>
+
+Store the value in the key-value-store under the provided key. Adapters MAY serialize the value using Ruby's Marshal system, and MUST NOT store a reference to the original value in the store, unless Ruby disallows duplication of the original value. Adapters SHOULD NOT simply call <code>dup</code> on the value, unless the value stores no references to other Object. For example, an adapter MAY store a <code>dup</code> of a String, but SHOULD NOT store a <code>dup</code> of <code>["hello", "world"]</code>.
+
+### <code>fetch(key[Object]) => Object</code>
+
+Return the value stored in the key-value-store under the provided key. If no value is stored under the provided key, the adapter MUST raise an IndexError.
+
+### <code>fetch(key[Object], &block) => Object</code>
+
+Return the value stored in the key-value-store under the provided key. If no value is stored under the provided key, the adapter MUST yield to the block, and return the value. The adapter MUST NOT store the value returned from the block in the key-value-store.
+
+### <code>fetch(key[Object], value[Object]) => Object</code>
+
+Return the value stored in the key-value-store under the provided key. If no value is stored under the provided key, the adapter MUST return the default value provided. The adapter MUST NOT store the default value in the key-value-store.
+
+### <code>delete(key[Object]) => Object</code>
+
+Delete the value stored in the key-value-store for the key provided, and return the value previously stored there. After this operation, the key-value-store MUST behave as though no value was stored for the provided key.
+
+### <code>key?(key[Object]) => [TrueClass, FalseClass]</code>
+
+Determine whether a value exists in the key-value-store for the key provided. If a value exists, the adapter MUST return <code>true</code>. Otherwise, the adapter MUST return <code>false</code>.
+
+### <code>store(key[Object], value[Object]) => Object(value)</code>
+
+Behaves the same as <code>[]=</code>, but allows the client to send additional options which can be specified by the adapter (and which may be specified by extensions to this specification).
+
+### <code>update_key(key[Object], options[Hash]) => nil</code>
+
+In this specification, this operation does nothing. However, extensions to this specification may specify semantics for certain values of the <code>options</code> Hash.
+
+### <code>clear</code>
+
+Completely empty all keys and values from the key-value-store. Adapters MAY allow a namespace during initialization, which can scope this operation to a particular subset of keys. After calling <code>clear</code>, a <code>[]</code> operation MUST return nil for every possible key, and a <code>key?</code> query MUST return false for every possible key.
+
+# Additional Options Hashes
+
+The following methods may all take an additional Hash as a final argument. This allows the client to send additional options which can be specified by the adapter (and which may be specified by extensions to this specification).
+
+* fetch
+* store
+* delete
+* key?
+* clear
+
+In the case of methods with optional arguments, the Hash MUST be provided as the final argument, and all optional arguments MUST be specified.
+
+Keys in this Hash MUST be Strings or Symbols. If they are Strings, they MUST be prefixed with a unique namespace. Namespaces MUST be separated from the name of the key with a single ".". The namespace SHOULD be the name of the gem that exposes the key.
+
+Keys in this Hash MUST NOT be Symbols unless this specification or an official extension to this specification defines a Symbol key.
+
+# Key Equality
+
+Adapters MUST consider keys as equal to one another if and only if the value of <code>Marshal.dump(keya)</code> is the same (byte-for-byte) as <code>Marshal.dump(keyb)</code>. This does not mean that adapters are required to use <code>Marshal.dump</code> to calculate the key to use for a given key specified by the consumer of the adapter. However, if an adapter does not, it MUST guarantee that the value returned for every key is identical to the value that would be returned if it did a byte-for-byte comparison of the result of <code>Marshal.dump</code> for every operation involving a key.
+
+# Storage and Serialization
+
+In a Moneta-compliant adapter, any Ruby object that can be serialized using Ruby's marshalling system may be used for keys or values.
+
+Adapters MAY use the marshalling system to serialize Ruby objects. Adapters MUST NOT return an Object from a fetch operation that existed on the heap prior to the fetch operation. The intention of this requirement is to prevent adapters that use the heap for persistence to store direct references to Objects passed into the <code>store</code> or <code>[]=</code> methods.
+
+# Atomicity
+
+The base Moneta specification does not specify any atomicity guarantees. However, extensions to this spec may specify extensions that define additional guarantees for any of the defined operations.
+
+# Expiry
+
+The base Moneta specification does not specify any mechanism for time-based expiry. However, extensions to this spec may specify mechanisms (using <code>store</code> and <code>update_key</code>) to provide expiration semantics.

data/benchmarks/various.rb

@@ -0,0 +1,234 @@
+#!/usr/bin/env ruby
+require 'benchmark'
+require "rubygems"
+
+# Hacked arrays
+# Array modifications
+class HackedArray < Array
+  # Random keys/values
+  attr_reader :keys_used
+  def random_key(no_repeat = true, clean_keys_used = false)
+    @keys_used = [] if clean_keys_used or @keys_used.nil? or @keys_used.size == self.size
+    begin key = rand(self.size) end while no_repeat and @keys_used.include?(key)
+    @keys_used << key
+    return key
+  end
+
+  def random_value(no_repeat = true, clean_keys_used = false)
+    values_at(random_key(no_repeat, clean_keys_used)).first
+  end
+  alias_method :random, :random_value
+
+  def random_subset(n, no_repeat = true, clean_keys_used = true)
+    (1..n).map{|x| random_value(no_repeat, (clean_keys_used && x == 1))}
+  end
+
+  def self.new_from_string(str)
+    res = new
+    str.split('').each{|x| res << x}
+    res
+  end
+end
+
+require "../lib/moneta"
+require "moneta/memcache"
+require "moneta/tyrant"
+require "moneta/berkeley"
+
+
+stores = {
+  'Redis' => { },
+  'Memcached' => { :class_name => "Memcache", :server => "localhost:11211", :namespace => 'moneta_bench' },
+  'Tyrant' => { :host => 'localhost', :port => 1978 }, # Breaks for any n > 50 on my machine
+  'MongoDB' => { :host => 'localhost', :port => 27017, :db => 'moneta_bench' },
+  'LMC' => { :filename => "bench.lmc" },
+  'Berkeley' => { :file => "bench.bdb" },
+  'Rufus' => {:file => "bench.rufus"},
+  'Memory' => { },
+  'DataMapper' => { :setup => "sqlite3::memory:" },
+  # 'Couch' => {:db => "couch_test"},
+  'TC (Tyrant)' => 
+    {:name => "test.tieredtyrant", :backup => Moneta::Tyrant.new(:host => "localhost", :port => 1978), :class_name => "TieredCache"},
+  'TC (Memcached)' => 
+    {:name => "test.tieredmc", :backup => Moneta::Memcache.new(:server => "localhost:11211", :namespace => "various"), :class_name => "TieredCache"}
+}
+
+stats, keys, data, errors, summary = {}, [], HackedArray.new, HackedArray.new, HackedArray.new
+dict = HackedArray.new_from_string 'abcdefghijklnopq123456789'
+n = ARGV[0] ? ARGV[0].to_i : 100
+m = ARGV[1] ? ARGV[1].to_i : 10
+c = ARGV[2] ? ARGV[2].to_i : 3
+vlen_min, vlen_max, vlen_ttl, vlen_avg = 99999, 0, 0, 0
+ds = dict.size
+
+puts "======================================================================"
+puts "Comparison of write/read between Moneta Stores"
+puts "======================================================================"
+
+puts "Data loading..."
+n.times do |x|
+  klen = 6 + rand(3)
+  vlen = (rand(m) + 1) * (rand(m) + rand(m) + 1)
+  key = dict.random_subset(klen).join
+  keys << key
+  value = key * vlen
+  data << [key, value]
+  vs = value.size
+  vlen_min = vs if vs < vlen_min
+  vlen_max = vs if vs > vlen_max
+  vlen_ttl = vlen_ttl + vs
+end
+vlen_avg = vlen_ttl / n
+
+puts "----------------------------------------------------------------------"
+#puts data.inspect
+puts "Total keys: #{keys.size}, unique: #{keys.uniq.size}"
+#puts keys.sort.inspect
+
+puts "----------------------------------------------------------------------"
+puts "                  Minimum    Maximum      Total    Average        xps "
+puts "----------------------------------------------------------------------"
+puts "Lenght Stats   % 10i % 10i % 10i % 10i " % [vlen_min, vlen_max, vlen_ttl, vlen_avg]
+
+module Moneta
+  class TieredCache
+    include Moneta::Defaults
+  
+    def initialize(options)
+      @bdb = Moneta::Berkeley.new(:file => File.join(File.dirname(__FILE__), options[:name]))
+      @mc = options[:backup]
+      # @mc = Moneta::Tyrant.new(:host => "localhost", :port => 1978)
+      # @mc  = Moneta::Memcache.new(:server => "localhost:11211", :namespace => options[:name])
+    end
+  
+    def [](key)
+      val = @bdb[key]
+      unless val
+        @bdb[key] = val if val = @mc[key]
+      end
+      val
+    end
+  
+    def []=(key, val)
+      @bdb[key] = val
+      @mc[key]  = val
+    end
+  
+    def store(key, value, options = {})
+      @bdb.store(key, value, options)
+      @mc.store(key, value, options)
+    end
+  
+    def delete(key)
+      bdb_val = @bdb.delete(key)
+      mc_val  = @mc.delete(key)
+      bdb_val || mc_val
+    end
+  
+    def clear
+      @mc.clear
+      @bdb.clear
+    end
+  
+    def update_key(name, options)
+      @mc.update_key(name, options)
+      @bdb.update_key(name, options)
+    end
+  
+    def key?(key)
+      @bdb.key?(key) || @mc.key?(key)
+    end
+  end
+end
+
+stores.each do |name, options|
+  cname = options.delete(:class_name) || name
+  puts "======================================================================"
+  puts name
+  puts "----------------------------------------------------------------------"
+  begin
+    require "../lib/moneta/#{cname.downcase}"
+  rescue LoadError
+  end
+  klass = Moneta.const_get(cname)
+  @cache = klass.new(options)
+  stats[name] = {
+    :writes => [],
+    :reads => [],
+    :totals => [],
+    :avgs => [],
+  }
+  c.times do |round|
+    @cache.clear
+    print "[#{round + 1}] R"
+    m1 = Benchmark.measure do
+      n.times do
+        key, value = data.random
+          
+        @cache[key] = value
+      end
+    end
+    stats[name][:writes] << m1.real
+    print "W "
+    m2 = Benchmark.measure do
+      n.times do
+        key, value = data.random
+        res = @cache[key]
+        errors << [name, key, value, res] unless res == value
+      end
+    end
+    stats[name][:reads] << m2.real
+    stats[name][:totals] << (m1.real + m2.real)
+    stats[name][:avgs] << (m1.real + m2.real)
+  end
+  print "\n"
+  puts "----------------------------------------------------------------------"
+  puts "                  Minimum    Maximum      Total    Average        xps "
+  puts "----------------------------------------------------------------------"
+  tcmin, tcmax, tctot, tcavg = 99999, 0, 0, 0
+  [:writes, :reads].each do |sname|
+    cmin, cmax, ctot, cavg = 99999, 0, 0, 0
+    stats[name][sname].each do |val|
+      cmin = val if val < cmin
+      tcmin = val if val < tcmin
+      cmax = val if val > cmax
+      tcmax = val if val > tcmax
+      ctot = ctot + val
+      tctot = tctot + val
+    end
+    cavg = ctot / c
+    puts "%-14.14s % 10.4f % 10.4f % 10.4f % 10.4f % 10.4f " % ["#{name} #{sname}", cmin, cmax, ctot, cavg, n / cavg]
+  end
+  tcavg = tctot / (c * 2)
+  puts "%-14.14s % 10.4f % 10.4f % 10.4f % 10.4f % 10.4f " % ["#{name} avgs", tcmin, tcmax, tctot, tcavg, n / tcavg]
+  summary << [name, tcmin, tcmax, tctot, tcavg, n / tcavg]
+end
+puts "----------------------------------------------------------------------"
+if errors.size > 0
+  puts "Errors : #{errors.size}"
+#  puts errors.inspect
+else
+  puts "No errors in reading!"
+end
+puts "======================================================================"
+puts "Summary :: #{c} runs, #{n} keys"
+puts "======================================================================"
+puts "                  Minimum    Maximum      Total    Average        xps "
+puts "----------------------------------------------------------------------"
+summary.each do |sry|
+  puts "%-14.14s % 10.4f % 10.4f % 10.4f % 10.4f % 10.4f " % sry
+end
+puts "======================================================================"
+puts "THE END"
+puts "======================================================================"
+
+#======================================================================
+#Summary :: 3 runs, 1000 keys
+#======================================================================
+#                  Minimum    Maximum      Total    Average       xps
+#----------------------------------------------------------------------
+#MemcacheDB         0.6202     2.7850     7.0099     1.1683   855.9366
+#Memcached          0.4483     0.6563     3.3251     0.5542  1804.4385
+#Redis              0.3282     0.5221     2.2965     0.3828  2612.6444
+#MongoDB            0.6660     1.0539     5.1667     0.8611  1161.2745
+#======================================================================

data/dkastner-moneta.gemspec

@@ -0,0 +1,55 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+require 'moneta/version'
+
+Gem::Specification.new do |s|
+  s.name = 'dkastner-moneta'
+  s.version = Moneta::VERSION
+  s.authors = ['Yehuda Katz', 'Derek Kastner']
+  s.email = %w{wycats@gmail.com dkastner@gmail.com}
+  s.date = '2011-02-10'
+  s.description = %q{A unified interface to key/value stores}
+  s.extra_rdoc_files = %w{README LICENSE}
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.homepage = 'http://github.com/dkastner/moneta'
+  s.require_paths = ["lib"]
+  s.summary = %q{A unified interface to key/value stores, including MongoDB, Redis, Tokyo, and ActiveRecord}
+
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec'
+
+  # Adapter-specific requirements:
+  if ENV['ACTIVERECORD']
+    s.add_development_dependency 'activerecord' 
+    s.add_development_dependency 'sqlite3-ruby' 
+  end
+
+  s.add_development_dependency 'couchrest' if ENV['COUCH']
+  
+  if ENV['DM']
+    s.add_development_dependency 'datamapper'
+    s.add_development_dependency 'dm-core'
+    s.add_development_dependency 'dm-migrations'
+  end
+
+  s.add_development_dependency 'fog' if ENV['FOG']
+
+  if ENV['MEMCACHE']
+    s.add_development_dependency 'localmemcache' 
+    s.add_development_dependency 'memcache-client'
+  end
+
+  s.add_development_dependency 'mongo' if ENV['MONGO']
+
+  s.add_development_dependency 'redis' if ENV['REDIS']
+
+  if ENV['TOKYO']
+    s.add_development_dependency 'tokyocabinet'
+    s.add_development_dependency 'tokyotyrant'
+  end
+
+  s.add_development_dependency 'xattr' if ENV['XATTR']
+end

data/lib/moneta/adapters/active_record.rb

@@ -0,0 +1,57 @@
+begin
+  require 'active_record'
+rescue LoadError
+  puts "You need the activerecord gem in order to use the ActiveRecord moneta store"
+end
+
+module Moneta
+  class ActiveRecord
+    include Moneta::Defaults
+
+    def initialize(options = {})
+      @options = options
+      unless self.class.const_defined?('Store')
+        self.class.const_set('Store', Class.new(::ActiveRecord::Base)) # this prevents loading issues when active_record gem is unavailable
+      end
+      Store.establish_connection(@options[:connection] || raise("Must specify :connection"))
+      Store.set_table_name(@options[:table] || 'moneta_store')
+    end
+
+    def migrate
+      unless Store.table_exists?
+        Store.connection.create_table Store.table_name, :id => false do |t|
+          t.string   'key', :primary => :true
+          t.string   'value'
+        end
+      end
+    end
+
+    def key?(key)
+      record = Store.find_by_key key_for(key)
+      !record.nil?
+    end
+
+    def [](key)
+      record = Store.find_by_key key_for(key)
+      record ? deserialize(record.value) : nil
+    end
+
+    def delete(key)
+      record = Store.find_by_key key_for(key)
+      if record
+        Store.where(:key => key_for(record.key)).delete_all
+        deserialize record.value
+      end
+    end
+
+    def store(key, value, options = {})
+      record = Store.new :key => key_for(key), :value => serialize(value)
+      record.save!
+      value
+    end
+
+    def clear
+      Store.delete_all
+    end
+  end
+end

data/lib/moneta/adapters/memcache.rb

@@ -2,10 +2,11 @@ begin
   require "memcached"
   MemCache = Memcached
 rescue LoadError
-  require "memcache"
-rescue
-  puts "You need either the `memcached` or `memcache-client` gem to use the Memcache moneta store"
-  exit
+  begin
+    require "memcache"
+  rescue LoadError
+    puts "You need either the `memcached` or `memcache-client` gem to use the Memcache moneta store"
+  end
 end
 
 module Moneta

data/lib/moneta/adapters/pstore.rb

@@ -6,7 +6,7 @@ module Moneta
       include Moneta::Defaults
 
       def initialize(options = {})
-        options[:path] ||= File.join Dir.pwd, 'data.pstore'
+        options[:path] ||= ::File.join Dir.pwd, 'data.pstore'
         super options[:path]
       end
 

data/lib/moneta/adapters/yaml.rb

@@ -7,9 +7,9 @@ module Moneta
       include Moneta::Defaults
 
       def initialize(options = {})
-        @file = File.expand_path(options[:path])
+        @file = ::File.expand_path(options[:path])
         unless ::File.exists?(@file)
-          File.open(@file, "w") { |file| file << {}.to_yaml }
+          ::File.open(@file, "w") { |file| file << {}.to_yaml }
         end
       end