tokyo_store 0.3.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,6 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+*.tch

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Marcos Piccinini
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,51 @@
+= TokyoStore
+
+Rack based Tokyo stored (FAST!) Rails session store.
+
+Code:: http://github.com/nofxx/tokyo_store
+Demo:: http://github.com/nofxx/tokyo_webapps under rails/tokyo_store
+
+
+== Require
+
+Tokyo Cabinet and/or Tyrant:: http://tokyocabinet.sourceforge.net
+
+Choose an adapter:
+
+Tyrant - Pure Ruby:: http://tokyocabinet.sourceforge.net/tyrantrubydoc
+Cabinet - C Bindings:: http://tokyocabinet.sourceforge.net/rubydoc
+RufusTyrant - Rufus FFI:: http://github.com/jmettraux/rufus-tokyo
+
+
+== Install
+
+  gem install nofxx-tokyo_store
+
+Rails (enviroment.rb)
+
+  config.gem 'nofxx-tokyo_store', :lib => 'tokyo_store'
+  ActionController::Base.session_store = Rack::Session::<ADAPTER (Tyrant, Cabinet or RufusTyrant)>
+
+
+== Tyrant
+
+Start the server if you are using a Tyrant based adapter:
+
+  ttserver data.tch
+
+Some rake tasks: http://gist.github.com/147413
+
+
+Have fun!
+
+
+== Thanks
+
+Mikio Hirabayashi:: tokyo products
+John Mettraux:: rufus-tokyo
+Luca Guidi:: redis-store
+
+
+== Copyright
+
+Copyright (c) 2009 Marcos Piccinini. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,44 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "tokyo_store"
+    gem.summary = "Tokyo Tyrant rails session store"
+    gem.email = "x@nofxx.com"
+    gem.homepage = "http://github.com/nofxx/tokyo_store"
+    gem.authors = ["Marcos Piccinini"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION').chomp
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "foo #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.3.0

data/benchmark/cache.rb

@@ -0,0 +1,134 @@
+#
+# Tokyo Store
+#
+# Benchmark vs MemCacheStore on memcached and tyrant
+#
+$: << File.join(File.dirname(__FILE__), "/../lib")
+require 'rubygems'
+require 'active_support'
+require 'cache/tokyo_store'
+require 'redis-store'
+require 'cache/rails/redis_store'
+
+P = "x" * 100
+M = P * 10
+G = M * 10
+A = [P, M, G]
+X = { :small => P, :medium => M, :big => G }
+class User; attr_accessor :name, :info; end
+u = User.new; u.name = P; u.info = G
+O = u
+T = ARGV[0].to_i || 10000
+
+#TODO: Cabinet & memcached C bindings
+TEST = {
+ "TokyoStore" => ActiveSupport::Cache.lookup_store(:tokyo_store, "localhost:1978"),
+ "MemTokyo"   => ActiveSupport::Cache.lookup_store(:mem_cache_store, "localhost:1978"),
+ "RedisStore" => ActiveSupport::Cache.lookup_store(:redis_store, "localhost:6379"),
+ "MemCache"   => ActiveSupport::Cache.lookup_store(:mem_cache_store, "localhost:11211"),
+}
+
+puts " Write"
+puts "----------"
+Benchmark.bmbm do |b|
+  TEST.each_pair do |k,s|
+    b.report("#{k} P") { T.times { |i| s.write i.to_s, P }}
+    b.report("#{k} M") { T.times { |i| s.write i.to_s, M }}
+    b.report("#{k} G") { T.times { |i| s.write i.to_s, G }}
+    b.report("#{k} Obj") { T.times { |i| s.write i.to_s, O }}
+    b.report("#{k} Hash") { T.times { |i| s.write i.to_s, X }}
+    b.report("#{k} Array") { T.times { |i| s.write i.to_s, A }}
+    b.report("#{k} Delete") { T.times { |i| s.delete i.to_s }}
+    b.report("#{k} +") { T.times { |i| s.increment i.to_s }}
+    b.report("#{k} -") { T.times { |i| s.decrement i.to_s }}
+  end
+end
+
+puts " Read"
+puts "----------"
+TEST.each { |p| 10_000.times { |i| p[1].write i.to_s, G }}
+#TODO: implement read with diff data.
+Benchmark.bmbm do |b|
+  TEST.each_pair do |k,s|
+    k = s.class.to_s.split("::")[-1]
+    b.report("#{k} Seq") { T.times { |i| s.read rand(i).to_s }}
+    b.report("#{k} Rand") { T.times { |i| s.read rand(i).to_s }}
+    b.report("#{k} Exist") { T.times { |i| s.exist? i.to_s }}
+  end
+end
+
+puts
+thr = []
+Benchmark.bmbm do |b|
+  TEST.each_pair do |k,s|
+    b.report("#{k} TW") { (T/2).times { |j| thr << Thread.new { (T/2).times { |i| s.write "#{j}-#{i}", X }}};  thr.each { |t| t.join }; thr = [] }
+    b.report("#{k} TR") { (T/2).times { |j| thr << Thread.new { (T/2).times { |i| s.read "#{j}-#{i}" }}};  thr.each { |t| t.join }; thr = [] }
+  end
+end
+
+__END__
+
+*NOTE: Redis and Memcache support native expiration, Tokyo doesn't.
+Wondering the impact of this feature written in ruby...
+
+Core 2 Duo 8500 - 3.16Ghz
+------------------------------------------------------------
+ Write
+-------------
+TokyoStore# P    0.140000   0.090000   0.230000 (  0.367587)
+RedisStore# P    0.340000   0.090000   0.430000 (  0.542441)
+MemCacheD # P    0.570000   0.160000   0.730000 (  0.829167)
+MemCacheT # P    0.560000   0.170000   0.730000 (  0.897084)
+
+TokyoStore# M    0.170000   0.100000   0.270000 (  0.448071)
+RedisStore# M    0.360000   0.120000   0.480000 (  0.641083)
+MemCacheD # M    0.610000   0.140000   0.750000 (  0.878559)
+MemCacheT # M    0.630000   0.140000   0.770000 (  0.951748)
+
+TokyoStore# G    0.410000   0.090000   0.500000 (  0.976746)
+RedisStore# G    0.930000   0.170000   1.100000 (  1.572024)
+MemCacheD # G    1.000000   0.200000   1.200000 (  1.429635)
+MemCacheT # G    1.060000   0.170000   1.230000 (  1.558731)
+
+TokyoStore# D    0.220000   0.170000   0.390000 (  0.707877)
+RedisStore# D    0.240000   0.110000   0.350000 (  0.474265)
+MemCacheD # D    0.560000   0.110000   0.670000 (  0.775865)
+MemCacheT # D    0.560000   0.140000   0.700000 (  0.860964)
+
+TokyoStore# +    0.230000   0.190000   0.420000 (  0.654690)
+RedisStore# +    0.200000   0.120000   0.320000 (  0.494333)
+MemCacheD # +    0.570000   0.130000   0.700000 (  0.801407)
+MemCacheT # +    0.540000   0.190000   0.730000 (  0.937882)
+
+TokyoStore# -    0.200000   0.210000   0.410000 (  0.669899)
+RedisStore# -    0.240000   0.120000   0.360000 (  0.511279)
+MemCacheD # -    0.600000   0.100000   0.700000 (  0.778212)
+MemCacheT # -    0.600000   0.140000   0.740000 (  0.983345)
+
+ Read
+--------------
+TokyoStore# Seq     0.440000   0.150000   0.590000 (  0.858501)
+RedisStore# Seq     0.660000   0.130000   0.790000 (  1.039310)
+MemCacheD # Seq     0.830000   0.220000   1.050000 (  1.135630)
+MemCacheT # Seq     1.210000   0.190000   1.400000 (  1.754004)
+
+TokyoStore# Rand    0.490000   0.200000   0.690000 (  1.003466)
+RedisStore# Rand    0.580000   0.210000   0.790000 (  1.877162)
+MemCacheD # Rand    0.870000   0.170000   1.040000 (  1.135382)
+MemCacheT # Rand    1.410000   0.230000   1.640000 (  1.971372)
+
+TokyoStore# Exist   0.460000   0.140000   0.600000 (  0.845920)
+RedisStore# Exist   0.550000   0.250000   0.800000 (  1.658580)
+MemCacheD # Exist   0.810000   0.220000   1.030000 (  1.138310)
+MemCacheT # Exist   1.150000   0.250000   1.400000 (  1.760770)
+
+
+Tokyo # W   0.510000   0.150000   0.660000 (  1.023739)
+Redis # W   0.540000   0.140000   0.680000 (  1.032881)
+MemCa # W   2.530000   0.210000   2.740000 (  3.072881)
+MemTo # W   2.600000   0.280000   2.880000 (  3.344125)
+
+Tokyo # R   0.680000   0.210000   0.890000 (  2.547828)
+Redis # R   0.710000   0.220000   0.930000 (  2.599269)
+MemCa # R   2.190000   0.240000   2.430000 (  2.977408)
+MemTo # R   3.210000   0.330000   3.540000 (  4.298882)

data/benchmark/session.rb

@@ -0,0 +1,75 @@
+#
+# Tokyo Store
+#
+# Session Benchmark vs Redis Store
+#
+# TODO: memcache
+#
+$: << File.join(File.dirname(__FILE__), "/../lib")
+require 'rubygems'
+require 'benchmark'
+require 'rack'
+require 'tokyo_store'
+require 'redis-store'
+#S = [Rack::Session::Tokyo::DEFAULT_OPTIONS[:key], { :test => "foo"}]
+T = ARGV[0].to_i || 10000
+
+rack = lambda do |env|
+  env["rack.session"]["counter"] ||= 0
+  env["rack.session"]["counter"] += 1
+  Rack::Response.new(env["rack.session"].inspect).to_a
+end
+
+TEST = {
+  "Tokyo" =>  Rack::Session::Tokyo.new(rack),
+  "Redis" => Rack::Session::Redis.new(rack),
+}
+bar = "_" * 45
+
+puts "\n#{bar}  SET"
+Benchmark.bmbm do |b|
+  TEST.each_pair do |n,s|
+    b.report(n) {  T.times {  Rack::MockRequest.new(s).get("/") }}
+  end
+end
+
+puts "\n#{bar}  GET"
+Benchmark.bmbm do |b|
+  TEST.each_pair do |n,s|
+    b.report(n) do T.times do
+      req = Rack::MockRequest.new(s)
+      cookie = req.get("/")["Set-Cookie"]
+      req.get("/", "HTTP_COOKIE" => cookie)
+    end end
+  end
+end
+
+puts "\n#{bar} EXIST"
+Benchmark.bmbm do |b|
+  TEST.each_pair do |n,s|
+    b.report(n) do T.times do
+      Rack::MockRequest.new(s).get("/", "HTTP_COOKIE" => "rack.session=badbadcookie")
+    end end
+  end
+end
+
+
+__END__
+
+*Note: Tokyo uses FFI adapter, while redis a pure ruby.
+Both comparable to memcache pr too. Check the cache benchmark.
+
+_____________________________________________  GET
+Tokyo   6.310000   1.030000   7.340000 (  8.511138)
+Redis   7.300000   1.010000   8.310000 (  9.325441)
+------------------------------- total: 15.650000sec
+
+_____________________________________________  SET
+Tokyo   3.340000   0.540000   3.880000 (  4.562920)
+Redis   3.960000   0.540000   4.500000 (  5.030627)
+-------------------------------- total: 8.380000sec
+
+_____________________________________________ EXIST
+Tokyo   4.090000   0.650000   4.740000 (  5.537898)
+Redis   4.700000   0.700000   5.400000 (  6.061131)
+------------------------------- total: 10.140000sec

data/lib/cache/tokyo_store.rb

@@ -0,0 +1,139 @@
+require 'rufus/tokyo/tyrant'
+# require 'tokyocabinet'
+module ActiveSupport
+  module Cache
+
+    # A cache store implementation which stores data in Tokyo Cabinet
+    #
+    # Special features:
+    # - Substring match
+    # - Clustering and load balancing. TODO
+    # - Time-based expiry support. TODO (Lua)
+    # - Per-request in memory cache for all communication with the Tokyo server(s).
+    class TokyoStore < Store
+
+      def self.build_tokyo(*store)
+        store = store.flatten
+        options = store.extract_options!
+        #TODO: multiple instances
+        store = store.empty? ? ["localhost", 1978]  : store[0].split(":")
+
+        #TODO: Auto choice between tyrant ffi x tyrant pure ruby x cabinet C
+        # Tyrant FFI
+        Rufus::Tokyo::Tyrant.new(store[0], store[1].to_i)
+
+        # Cabinet C
+        #hdb = HDB.new
+        # if !hdb.open(store[0], HDB::OWRITER | HDB::OCREAT)
+        #   ecode = hdb.ecode
+        #   STDERR.printf("open error: %s\n", hdb.errmsg(ecode))
+        # end
+        # hdb
+      end
+
+      # Creates a new TokyoStore object, with the given tyrant server
+      # addresses. Each address is either a host name, or a host-with-port string
+      # in the form of "host_name:port". For example:
+      #
+      #   ActiveSupport::Cache::TokyoStore.new("localhost", "server-downstairs.localnetwork:8229")
+      #
+      # If no addresses are specified, then TokyoStore will connect to
+      # localhost port 1978 (the default tyrant port).
+      def initialize(*store)
+        if store.first.respond_to?(:get)
+          @data = store.first
+        else
+          @data = self.class.build_tokyo(*store)
+        end
+
+        extend Strategy::LocalCache
+      end
+
+      # Reads multiple keys from the cache.
+      def read_multi(*keys)
+        #keys.inject({ }){ |h,k| h.merge({ k => read(k)}) }
+        @data.lget(keys).inject({ }) { |h, k| h.merge({ k[0] => Marshal.load(k[1])})} #
+      end
+
+      def read(key, options = nil) # :nodoc:
+        # TODO: benchmark [key] vs .get(key)
+        super
+        return nil unless val = @data[key]
+        val = Marshal.load(val) unless raw?(options)
+        val
+        # if str = @data.get(key)
+        #   Marshal.load str
+        #   else
+        #   STDERR.printf("get error: %s\n", @data.errmsg(@data.ecode))
+        #   end
+        # logger.error("TokyoError (#{e}): #{e.message}")
+        # nil
+      end
+
+      # Writes a value to the cache.
+      #
+      # Possible options:
+      # - +:unless_exist+ - set to true if you don't want to update the cache
+      #   if the key is already set.
+      def write(key, value, options = nil)
+        super
+        method = options && options[:unless_exist] ? :add : :set
+        # will break the connection if you send it an integer
+        # in raw mode, so we convert it to a string to be sure it continues working.
+        value = raw?(options) ? value.to_s : Marshal.dump(value) # if value.instance_of? Hash
+
+        @data[key] = value
+        ###response = @data.put(key, value) || STDERR.printf("get error: %s\n", @data.errmsg(@data.ecode))#, expires_in(options), raw?(options))
+        # logger.error("TokyoError (#{e}): #{e.message}")
+        # false
+      end
+
+      def delete(key, options = nil) # :nodoc:
+        super
+        @data.delete(key) #= nil #, expires_in(options))
+      end
+
+      def exist?(key, options = nil) # :nodoc:
+        # Local cache is checked first?
+       !read(key, options).nil?
+      end
+
+      def increment(key, amount = 1) # :nodoc:
+        #NATIVE breaks...rufus integer prob?
+        # @data.incr(key, amount)
+        @data[key] = (@data[key].to_i + amount).to_s
+      end
+
+      def decrement(key, amount = 1) # :nodoc:
+        # @data.incr(key, -amount)
+        increment(key, -amount)
+      end
+
+      def delete_matched(matcher, options = nil) # :nodoc:
+        super
+        @data.delete_keys_with_prefix(matcher)
+      end
+
+      def clear
+        @data.clear
+      end
+
+      def stats
+        @data.stat
+      end
+
+      private
+        #TODO
+        # def expires_in(options)
+        #   (options && options[:expires_in]) || 0
+        # end
+
+        def raw?(options)
+          options && options[:raw]
+        end
+
+
+    end
+
+  end
+end