nofxx-tokyo_store 0.1.8 → 0.1.9

data/README.rdoc

@@ -21,12 +21,12 @@ Rails (enviroment.rb)
   config.gem 'nofxx-tokyo_store', :lib => 'tokyo_store'
   ActionController::Base.session_store = Rack::Session::Tokyo
 
+Start Tyrant
 
-== Run
+  ttserver data.tch
 
-Just start tyrant
 
-  ttserver data.tch
+Have fun!
 
 
 == Thanks

data/Rakefile

@@ -16,12 +16,29 @@ rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
 end
 
-require 'rake/testtask'
-Rake::TestTask.new(:spec) do |spec|
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
   spec.libs << 'lib' << 'spec'
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.verbose = true
+  spec.spec_files = FileList['spec/**/*_spec.rb']
 end
 
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
 task :default => :spec
 
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION').chomp
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "foo #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -1 +1 @@
-0.1.8
+0.1.9

data/benchmark/cache.rb

@@ -3,118 +3,132 @@
 #
 # Benchmark vs MemCacheStore on memcached and tyrant
 #
+$: << File.join(File.dirname(__FILE__), "/../lib")
 require 'rubygems'
 require 'active_support'
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'cache/tokyo_store'
-#include TokyoCabinet
+require 'redis-store'
+require 'cache/rails/redis_store'
 
 P = "x" * 100
 M = P * 10
 G = M * 10
-X = { :small => P, :medium => M, :big => G}
+A = [P, M, G]
+X = { :small => P, :medium => M, :big => G }
+class User; attr_accessor :name, :info; end
+u = User.new; u.name = P; u.info = G
+O = u
+T = ARGV[0].to_i || 10000
 
 #TODO: Cabinet & memcached C bindings
-@tokyo = ActiveSupport::Cache.lookup_store :tokyo_store, "localhost:1978"
-@memca = ActiveSupport::Cache.lookup_store :mem_cache_store, "localhost:11211"
-@memto = ActiveSupport::Cache.lookup_store :mem_cache_store, "localhost:1978"
+TEST = {
+ "TokyoStore" => ActiveSupport::Cache.lookup_store(:tokyo_store, "localhost:1978"),
+ "MemTokyo"   => ActiveSupport::Cache.lookup_store(:mem_cache_store, "localhost:1978"),
+ "RedisStore" => ActiveSupport::Cache.lookup_store(:redis_store, "localhost:6379"),
+ "MemCache"   => ActiveSupport::Cache.lookup_store(:mem_cache_store, "localhost:11211"),
+}
 
 puts " Write"
-puts "--------------"
+puts "----------"
 Benchmark.bmbm do |b|
-  b.report("TokyoStore# P") { 10_000.times { |i| @tokyo.write i.to_s, P }}
-  b.report("MemCacheD # P") { 10_000.times { |i| @memca.write i.to_s, P }}
-  b.report("MemCacheT # P") { 10_000.times { |i| @memto.write i.to_s, P }}
-  b.report("TokyoStore# M") { 10_000.times { |i| @tokyo.write i.to_s, M }}
-  b.report("MemCacheD # M") { 10_000.times { |i| @memca.write i.to_s, M }}
-  b.report("MemCacheT # M") { 10_000.times { |i| @memto.write i.to_s, M }}
-  b.report("TokyoStore# G") { 10_000.times { |i| @tokyo.write i.to_s, G }}
-  b.report("MemCacheD # G") { 10_000.times { |i| @memca.write i.to_s, G }}
-  b.report("MemCacheT # G") { 10_000.times { |i| @memto.write i.to_s, G }}
-  b.report("TokyoStore# D") { 10_000.times { |i| @tokyo.delete i.to_s }}
-  b.report("MemCacheD # D") { 10_000.times { |i| @memca.delete i.to_s }}
-  b.report("MemCacheT # D") { 10_000.times { |i| @memto.delete i.to_s }}
-  b.report("TokyoStore# +") { 10_000.times { |i| @tokyo.increment i.to_s }}
-  b.report("MemCacheD # +") { 10_000.times { |i| @memca.increment i.to_s }}
-  b.report("MemCacheT # +") { 10_000.times { |i| @memto.increment i.to_s }}
-  b.report("TokyoStore# -") { 10_000.times { |i| @tokyo.decrement i.to_s }}
-  b.report("MemCacheD # -") { 10_000.times { |i| @memca.decrement i.to_s }}
-  b.report("MemCacheT # -") { 10_000.times { |i| @memto.decrement i.to_s }}
+  TEST.each_pair do |k,s|
+    b.report("#{k} P") { T.times { |i| s.write i.to_s, P }}
+    b.report("#{k} M") { T.times { |i| s.write i.to_s, M }}
+    b.report("#{k} G") { T.times { |i| s.write i.to_s, G }}
+    b.report("#{k} Obj") { T.times { |i| s.write i.to_s, O }}
+    b.report("#{k} Hash") { T.times { |i| s.write i.to_s, X }}
+    b.report("#{k} Array") { T.times { |i| s.write i.to_s, A }}
+    b.report("#{k} Delete") { T.times { |i| s.delete i.to_s }}
+    b.report("#{k} +") { T.times { |i| s.increment i.to_s }}
+    b.report("#{k} -") { T.times { |i| s.decrement i.to_s }}
+  end
 end
 
 puts " Read"
-puts "--------------"
-#TODO: implement read with diff data. ALl should probably be 0 here
+puts "----------"
+TEST.each { |p| 10_000.times { |i| p[1].write i.to_s, G }}
+#TODO: implement read with diff data.
 Benchmark.bmbm do |b|
-  b.report("TokyoStore# Seq") { 10_000.times { |i| @tokyo.read i.to_s }}
-  b.report("MemCacheD # Seq") { 10_000.times { |i| @memca.read i.to_s }}
-  b.report("MemCacheT # Seq") { 10_000.times { |i| @memto.read i.to_s }}
-  b.report("TokyoStore# Rand") { 10_000.times { |i| @tokyo.read rand(i).to_s }}
-  b.report("MemCacheD # Rand") { 10_000.times { |i| @memca.read rand(i).to_s }}
-  b.report("MemCacheT # Rand") { 10_000.times { |i| @memto.read rand(i).to_s }}
-  b.report("TokyoStore# Exist") { 10_000.times { |i| @tokyo.exist? i.to_s }}
-  b.report("MemCacheD # Exist") { 10_000.times { |i| @memca.exist? i.to_s }}
-  b.report("MemCacheT # Exist") { 10_000.times { |i| @memto.exist? i.to_s }}
+  TEST.each_pair do |k,s|
+    k = s.class.to_s.split("::")[-1]
+    b.report("#{k} Seq") { T.times { |i| s.read rand(i).to_s }}
+    b.report("#{k} Rand") { T.times { |i| s.read rand(i).to_s }}
+    b.report("#{k} Exist") { T.times { |i| s.exist? i.to_s }}
+  end
 end
 
 puts
 thr = []
 Benchmark.bmbm do |b|
-  b.report("Tokyo # W") {  100.times { |j| thr << Thread.new { 100.times { |i| @tokyo.write "#{j}-#{i}", X }}};  thr.each { |t| t.join }; thr = [] }
-  b.report("MemCa # W") {  100.times { |j| thr << Thread.new { 100.times { |i| @memca.write "#{j}-#{i}", X}}};  thr.each { |t| t.join }; thr = [] }
-  b.report("MemTo # W") {  100.times { |j| thr << Thread.new { 100.times { |i| @memto.write "#{j}-#{i}", X}}};  thr.each { |t| t.join }; thr = [] }
-  b.report("Tokyo # R") {  100.times { |j| thr << Thread.new { 100.times { |i| @tokyo.read "#{j}-#{i}" }}};  thr.each { |t| t.join }; thr = [] }
-  b.report("MemCa # R") {  100.times { |j| thr << Thread.new { 100.times { |i| @memca.read "#{j}-#{i}"}}};  thr.each { |t| t.join }; thr = [] }
-  b.report("MemTo # R") {  100.times { |j| thr << Thread.new { 100.times { |i| @memto.read "#{j}-#{i}"}}};  thr.each { |t| t.join }; thr = [] }
+  TEST.each_pair do |k,s|
+    b.report("#{k} TW") { (T/2).times { |j| thr << Thread.new { (T/2).times { |i| s.write "#{j}-#{i}", X }}};  thr.each { |t| t.join }; thr = [] }
+    b.report("#{k} TR") { (T/2).times { |j| thr << Thread.new { (T/2).times { |i| s.read "#{j}-#{i}" }}};  thr.each { |t| t.join }; thr = [] }
+  end
 end
 
 __END__
 
-
+*NOTE: Redis and Memcache support native expiration, Tokyo doesn't.
+Wondering the impact of this feature written in ruby...
 
 Core 2 Duo 8500 - 3.16Ghz
 ------------------------------------------------------------
  Write
 -------------
 TokyoStore# P    0.140000   0.090000   0.230000 (  0.367587)
+RedisStore# P    0.340000   0.090000   0.430000 (  0.542441)
 MemCacheD # P    0.570000   0.160000   0.730000 (  0.829167)
 MemCacheT # P    0.560000   0.170000   0.730000 (  0.897084)
+
 TokyoStore# M    0.170000   0.100000   0.270000 (  0.448071)
+RedisStore# M    0.360000   0.120000   0.480000 (  0.641083)
 MemCacheD # M    0.610000   0.140000   0.750000 (  0.878559)
 MemCacheT # M    0.630000   0.140000   0.770000 (  0.951748)
+
 TokyoStore# G    0.410000   0.090000   0.500000 (  0.976746)
+RedisStore# G    0.930000   0.170000   1.100000 (  1.572024)
 MemCacheD # G    1.000000   0.200000   1.200000 (  1.429635)
 MemCacheT # G    1.060000   0.170000   1.230000 (  1.558731)
-TokyoStore# X    0.480000   0.130000   0.610000 (  1.299556)
-MemCacheD # X    1.460000   0.280000   1.740000 (  1.980678)
-MemCacheT # X    1.230000   0.230000   1.460000 (  1.856561)
+
 TokyoStore# D    0.220000   0.170000   0.390000 (  0.707877)
+RedisStore# D    0.240000   0.110000   0.350000 (  0.474265)
 MemCacheD # D    0.560000   0.110000   0.670000 (  0.775865)
 MemCacheT # D    0.560000   0.140000   0.700000 (  0.860964)
+
 TokyoStore# +    0.230000   0.190000   0.420000 (  0.654690)
+RedisStore# +    0.200000   0.120000   0.320000 (  0.494333)
 MemCacheD # +    0.570000   0.130000   0.700000 (  0.801407)
 MemCacheT # +    0.540000   0.190000   0.730000 (  0.937882)
+
 TokyoStore# -    0.200000   0.210000   0.410000 (  0.669899)
+RedisStore# -    0.240000   0.120000   0.360000 (  0.511279)
 MemCacheD # -    0.600000   0.100000   0.700000 (  0.778212)
 MemCacheT # -    0.600000   0.140000   0.740000 (  0.983345)
 
  Read
 --------------
 TokyoStore# Seq     0.440000   0.150000   0.590000 (  0.858501)
+RedisStore# Seq     0.660000   0.130000   0.790000 (  1.039310)
 MemCacheD # Seq     0.830000   0.220000   1.050000 (  1.135630)
 MemCacheT # Seq     1.210000   0.190000   1.400000 (  1.754004)
+
 TokyoStore# Rand    0.490000   0.200000   0.690000 (  1.003466)
+RedisStore# Rand    0.580000   0.210000   0.790000 (  1.877162)
 MemCacheD # Rand    0.870000   0.170000   1.040000 (  1.135382)
 MemCacheT # Rand    1.410000   0.230000   1.640000 (  1.971372)
+
 TokyoStore# Exist   0.460000   0.140000   0.600000 (  0.845920)
+RedisStore# Exist   0.550000   0.250000   0.800000 (  1.658580)
 MemCacheD # Exist   0.810000   0.220000   1.030000 (  1.138310)
 MemCacheT # Exist   1.150000   0.250000   1.400000 (  1.760770)
 
 
 Tokyo # W   0.510000   0.150000   0.660000 (  1.023739)
+Redis # W   0.540000   0.140000   0.680000 (  1.032881)
 MemCa # W   2.530000   0.210000   2.740000 (  3.072881)
 MemTo # W   2.600000   0.280000   2.880000 (  3.344125)
+
 Tokyo # R   0.680000   0.210000   0.890000 (  2.547828)
+Redis # R   0.710000   0.220000   0.930000 (  2.599269)
 MemCa # R   2.190000   0.240000   2.430000 (  2.977408)
 MemTo # R   3.210000   0.330000   3.540000 (  4.298882)

data/benchmark/session.rb

@@ -0,0 +1,75 @@
+#
+# Tokyo Store
+#
+# Session Benchmark vs Redis Store
+#
+# TODO: memcache
+#
+$: << File.join(File.dirname(__FILE__), "/../lib")
+require 'rubygems'
+require 'benchmark'
+require 'rack'
+require 'tokyo_store'
+require 'redis-store'
+#S = [Rack::Session::Tokyo::DEFAULT_OPTIONS[:key], { :test => "foo"}]
+T = ARGV[0].to_i || 10000
+
+rack = lambda do |env|
+  env["rack.session"]["counter"] ||= 0
+  env["rack.session"]["counter"] += 1
+  Rack::Response.new(env["rack.session"].inspect).to_a
+end
+
+TEST = {
+  "Tokyo" =>  Rack::Session::Tokyo.new(rack),
+  "Redis" => Rack::Session::Redis.new(rack),
+}
+bar = "_" * 45
+
+puts "\n#{bar}  GET"
+Benchmark.bmbm do |b|
+  TEST.each_pair do |n,s|
+    b.report(n) do T.times do
+      req = Rack::MockRequest.new(s)
+      cookie = req.get("/")["Set-Cookie"]
+      req.get("/", "HTTP_COOKIE" => cookie)
+    end end
+  end
+end
+
+puts "\n#{bar}  SET"
+Benchmark.bmbm do |b|
+  TEST.each_pair do |n,s|
+    b.report(n) {  T.times {  Rack::MockRequest.new(s).get("/") }}
+  end
+end
+
+puts "\n#{bar} EXIST"
+Benchmark.bmbm do |b|
+  TEST.each_pair do |n,s|
+    b.report(n) do T.times do
+      Rack::MockRequest.new(s).get("/", "HTTP_COOKIE" => "rack.session=badbadcookie")
+    end end
+  end
+end
+
+
+__END__
+
+*Note: Tokyo uses FFI adapter, while redis a pure ruby.
+Both comparable to memcache pr too. Check the cache benchmark.
+
+_____________________________________________  GET
+Tokyo   6.310000   1.030000   7.340000 (  8.511138)
+Redis   7.300000   1.010000   8.310000 (  9.325441)
+------------------------------- total: 15.650000sec
+
+_____________________________________________  SET
+Tokyo   3.340000   0.540000   3.880000 (  4.562920)
+Redis   3.960000   0.540000   4.500000 (  5.030627)
+-------------------------------- total: 8.380000sec
+
+_____________________________________________ EXIST
+Tokyo   4.090000   0.650000   4.740000 (  5.537898)
+Redis   4.700000   0.700000   5.400000 (  6.061131)
+------------------------------- total: 10.140000sec

data/lib/cache/tokyo_store.rb

@@ -6,6 +6,7 @@ module ActiveSupport
     # A cache store implementation which stores data in Tokyo Cabinet
     #
     # Special features:
+    # - Substring match
     # - Clustering and load balancing. TODO
     # - Time-based expiry support. TODO (Lua)
     # - Per-request in memory cache for all communication with the Tokyo server(s).
@@ -109,7 +110,8 @@ module ActiveSupport
       end
 
       def delete_matched(matcher, options = nil) # :nodoc:
-        #TODO @data.ldelete?
+        super
+        @data.delete_keys_with_prefix(matcher)
       end
 
       def clear

data/lib/rack/session/tokyo.rb

@@ -15,13 +15,6 @@ module Rack
         end
       end
 
-      def generate_sid
-        loop do
-          sid = super
-          break sid unless @pool[sid]
-        end
-      end
-
       private
       def get_session(env, sid)
         @mutex.lock if env['rack.multithread']
@@ -43,7 +36,7 @@ module Rack
 
       def set_session(env, sid, new_session, options)
         @mutex.lock if env['rack.multithread']
-        session = Marshal.load(session) if session = @pool[sid]
+        session = Marshal.load(session) rescue session if session = @pool[sid]
         if options[:renew] || options[:drop]
           @pool.delete sid
           return false if options[:drop]
@@ -55,9 +48,13 @@ module Rack
         @pool[sid] = options && options[:raw] ? session : Marshal.dump(session)
         return sid
       rescue => e
+        if e =~ /refused/
         warn "#{self} is unable to find server, error: #{e}"
         warn $!.inspect
         return false
+        else
+          raise e
+        end
       ensure
         @mutex.unlock if env['rack.multithread']
       end
@@ -79,6 +76,13 @@ module Rack
         cur
       end
 
+      def generate_sid
+        loop do
+          sid = super
+          break sid unless @pool[sid]
+        end
+      end
+
     end
 
   end

data/spec/cache/tokyo_store_spec.rb

@@ -141,10 +141,13 @@ describe "TokyoStore" do
     end
 
     it "should delete matched" do
-      @cache.write("val", 1)
-      @cache.write("value", 1)
-      @cache.write("not", 1)
+      ["val", "value", "vall", "val/1", "vla", "xla", "xla/1"].each do |v|
+        @cache.write(v, 1)
+      end
+
       @cache.delete_matched('val')
+      ["val", "value", "vall", "val/1"].each { |v| @cache.read(v).should be_nil }
+      ["vla", "xla", "xla/1"].each { |v| @cache.read(v).should eql(1) }
     end
 
   end

data/tokyo_store.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{tokyo_store}
-  s.version = "0.1.8"
+  s.version = "0.1.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Marcos Piccinini"]
-  s.date = %q{2009-07-04}
+  s.date = %q{2009-07-05}
   s.email = %q{x@nofxx.com}
   s.extra_rdoc_files = [
     "LICENSE",
@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
      "Rakefile",
      "VERSION",
      "benchmark/cache.rb",
+     "benchmark/session.rb",
      "lib/cache/tokyo_store.rb",
      "lib/rack/cache/tokyo_entitystore.rb",
      "lib/rack/cache/tokyo_metastore.rb",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nofxx-tokyo_store
 version: !ruby/object:Gem::Version 
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors: 
 - Marcos Piccinini
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-04 00:00:00 -07:00
+date: 2009-07-05 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -30,6 +30,7 @@ files:
 - Rakefile
 - VERSION
 - benchmark/cache.rb
+- benchmark/session.rb
 - lib/cache/tokyo_store.rb
 - lib/rack/cache/tokyo_entitystore.rb
 - lib/rack/cache/tokyo_metastore.rb