moneta 0.7.13 → 0.7.14

data/.travis.yml

@@ -16,6 +16,8 @@ before_install:
 install: 'echo "Bundle installed"'
 before_script:
   - mysql -e 'create database moneta;'
+  - mysql -e 'create database moneta_activerecord1;'
+  - mysql -e 'create database moneta_activerecord2;'
 env:
   global:
     - secure: "B0vx1g1CB1A6mM3B/iy2ATicfS4OXT80bb2RVe8mSRsPzez1B4q4Q4hJcaMI\nrMARONN8Krtnti+IqvmDnB0Z0AKYMEyIc+zT37zJOCjLdkLJl+x/thuU/MbC\nvlLVwjMf6JE2EUzTfORDRFYc5ycCqfsfgNk1Go0D2CPT6P9u9uQ="

data/CHANGES

@@ -1,3 +1,8 @@
+0.7.14
+
+* Adapters::ActiveRecord, Adapters::Sequel: store values as blobs
+* Adapters::ActiveRecord fixed and improved
+
 0.7.13
 
 * Adapters::ActiveRecord: Use connection_pool

data/Rakefile

@@ -35,8 +35,13 @@ task :test do
   #
   # * Cassandra and Riak fail spuriously (An expert has to check the adapters!)
   #
-  # * ActiveRecord has some connection problems (An expert has to check the adapter!)
-  unstable = specs.select {|s| s =~ /quicklz|cassandra|riak|activerecord/ }
+  # * PStore and File increment/locking doesn't work correctly on JRuby
+  #
+  unstable = %w(quicklz cassandra riak)
+  unstable += %w(file pstore) if defined?(JRUBY_VERSION)
+
+  unstable_re = /#{unstable.join('|')}/
+  unstable = specs.select {|s| s =~ unstable_re }
   specs -= unstable
 
   if group =~ /^(\d+)\/(\d+)$/

data/lib/moneta/adapters/activerecord.rb

@@ -1,4 +1,5 @@
 require 'active_record'
+require 'thread'
 
 module Moneta
   module Adapters
@@ -10,44 +11,78 @@ module Moneta
       supports :create, :increment
       attr_reader :table
 
-      def self.tables
-        @tables ||= {}
-      end
+      @table_mutex = ::Mutex.new
+      @table_refcount = {}
 
-      # @param [Hash] options
-      # @option options [String] :table ('moneta') Table name
-      # @option options [Hash]   :connection ActiveRecord connection configuration
-      def initialize(options = {})
-        table = options[:table] || 'moneta'
-        @table = self.class.tables[table] ||=
-          begin
-            c = Class.new(::ActiveRecord::Base)
-            c.table_name = table
-            c.primary_key = :k
-            c
+      class << self
+        def release(table)
+          @table_mutex.synchronize do
+            if (@table_refcount[table] -= 1) <= 0
+              remove_const(table.name.sub(/^.*::/, ''))
+              @table_refcount.delete(table)
+            end
           end
+        end
 
-        if options[:connection]
-          begin
-            @table.establish_connection(options[:connection])
-          rescue
-            tries ||= 0
-            (tries += 1) < 3 ? retry : raise
+        def get(options)
+          name = 'Table_' << options.inspect.gsub(/[^\w]+/) do
+            $&.unpack('H2' * $&.bytesize).join.upcase
+          end
+          @table_mutex.synchronize do
+            table =
+              if const_defined?(name)
+                const_get(name)
+              else
+                create(name, options)
+              end
+            @table_refcount[table] ||= 0
+            @table_refcount[table] += 1
+            table
           end
         end
 
-        @table.connection_pool.with_connection do |conn|
-          unless @table.table_exists?
-            conn.create_table(table, :id => false) do |t|
-              # Do not use binary columns (Issue #17)
-              t.string :k, :null => false
-              t.string :v
+        private
+
+        def create(name, options)
+          table = Class.new(::ActiveRecord::Base)
+          const_set(name, table)
+          table.table_name = options[:table] || 'moneta'
+          table.primary_key = :k
+
+          if options[:connection]
+            begin
+              table.establish_connection(options[:connection])
+            rescue
+              tries ||= 0
+              (tries += 1) < 3 ? retry : raise
+            end
+          end
+
+          table.connection_pool.with_connection do |conn|
+            unless table.table_exists?
+              conn.create_table(table.table_name, :id => false) do |t|
+                # Do not use binary key (Issue #17)
+                t.string :k, :null => false
+                t.binary :v
+              end
+              conn.add_index(table.table_name, :k, :unique => true)
             end
-            conn.add_index(table, :k, :unique => true)
           end
+
+          table
+        rescue
+          remove_const(name)
+          raise
         end
       end
 
+      # @param [Hash] options
+      # @option options [String] :table ('moneta') Table name
+      # @option options [Hash]   :connection ActiveRecord connection configuration
+      def initialize(options = {})
+        @table = self.class.get(options)
+      end
+
       # (see Proxy#key?)
       def key?(key, options = {})
         @table.connection_pool.with_connection do
@@ -129,6 +164,12 @@ module Moneta
         end
         self
       end
+
+      # (see Proxy#close)
+      def close
+        self.class.release(@table)
+        @table = nil
+      end
     end
   end
 end

data/lib/moneta/adapters/file.rb

@@ -32,7 +32,7 @@ module Moneta
         path = store_path(key)
         temp_file = ::File.join(@dir, "value-#{$$}-#{Thread.current.object_id}")
         FileUtils.mkpath(::File.dirname(path))
-        ::File.open(temp_file, 'wb') {|file| file.write(value) }
+        ::File.open(temp_file, 'wb') {|f| f.write(value) }
         ::File.rename(temp_file, path)
         value
       end
@@ -63,9 +63,11 @@ module Moneta
         existed = ::File.exists?(path)
         ::File.open(path, 'ab+') do |f|
           Thread.pass until f.flock(::File::LOCK_EX)
+          # FIXME: JRuby needs synchronous mode, otherwise f.read might return wrong value
+          f.sync = true if defined?(JRUBY_VERSION)
           content = f.read
-          f.truncate(0)
           amount += Utils.to_int(content) if existed || !content.empty?
+          f.truncate(0)
           f.write(amount.to_s)
           amount
         end
@@ -75,9 +77,9 @@ module Moneta
       def create(key, value, options = {})
         path = store_path(key)
         FileUtils.mkpath(::File.dirname(path))
-        ::File.open(path, ::File::WRONLY | ::File::CREAT | ::File::EXCL) do |file|
-          file.binmode
-          file.write(value)
+        ::File.open(path, ::File::WRONLY | ::File::CREAT | ::File::EXCL) do |f|
+          f.binmode
+          f.write(value)
         end
         true
       rescue Errno::EEXIST

data/lib/moneta/adapters/sequel.rb

@@ -24,7 +24,7 @@ module Moneta
           end
         @backend.create_table?(table) do
           String :k, :null => false, :primary_key => true
-          String :v
+          Blob :v
         end
         @table = @backend[table]
       end

data/lib/moneta/transformer/config.rb

@@ -18,8 +18,8 @@ module Moneta
       :snappy   => [ :compress,  '::Snappy.inflate(%s)',         '::Snappy.deflate(%s)',           'snappy'        ],
       :quicklz  => [ :compress,  '::QuickLZ.decompress(%s)',     '::QuickLZ.compress(%s)',         'qlzruby'       ],
       :zlib     => [ :compress,  '::Zlib::Inflate.inflate(%s)',  '::Zlib::Deflate.deflate(%s)',    'zlib'          ],
-      :qp       => [ :encode,    "%s.unpack('M').first",         "[%s].pack('M').strip"                            ],
-      :uuencode => [ :encode,    "%s.unpack('u').first",         "[%s].pack('u').strip"                            ],
+      :qp       => [ :encode,    "%s.unpack('M').first",         "[%s].pack('M')"                                  ],
+      :uuencode => [ :encode,    "%s.unpack('u').first",         "[%s].pack('u')"                                  ],
       :base64   => RUBY_VERSION > '1.9' ?
                    [ :encode,    "%s.unpack('m0').first",        "[%s].pack('m0')"                                 ] :
                    [ :encode,    "%s.unpack('m').first",         "[%s].pack('m').gsub(\"\n\", '')"                 ],

data/lib/moneta/version.rb

@@ -1,5 +1,5 @@
 module Moneta
   # Moneta version number
   # @api public
-  VERSION = '0.7.13'
+  VERSION = '0.7.14'
 end

data/script/benchmarks

@@ -31,8 +31,9 @@ class MonetaBenchmarks
   FileUtils.mkpath(DIR)
 
   STORES = {
-    # SDBM is unstable, YAML is too slow
+    # SDBM accepts only very short key/value pairs (1k for both)
     # :SDBM => { :file => "#{DIR}/sdbm" },
+    # YAML is too slow
     # :YAML => { :file => "#{DIR}/yaml" },
     :ActiveRecord => { :table => 'activerecord', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :username => 'root', :database => 'moneta' } },
     :Cassandra => {},

data/script/generate-specs

@@ -41,6 +41,10 @@ class Specs
     new(:specs => specs - [:increment, :concurrent_increment] + [:not_increment])
   end
 
+  def without_large
+    new(:specs => specs - [:store_large])
+  end
+
   def without_concurrent
     new(:specs => specs - [:concurrent_increment, :concurrent_create])
   end
@@ -91,7 +95,7 @@ class Specs
   end
 
   def without_store
-    new(:specs => specs - [:store, :transform_value, :marshallable_value])
+    new(:specs => specs - [:store, :store_large, :transform_value, :marshallable_value])
   end
 
   def with_default_expires
@@ -103,9 +107,9 @@ class Specs
   end
 end
 
-ADAPTER_SPECS = Specs.new(:specs => [:null, :store, :returndifferent, :increment, :concurrent_increment, :concurrent_create, :persist, :multiprocess, :create, :features], :key => %w(string), :value => %w(string))
-STANDARD_SPECS = Specs.new(:specs => [:null, :store, :returndifferent, :marshallable_key, :marshallable_value, :transform_value, :increment, :concurrent_increment, :concurrent_create, :persist, :multiprocess, :create, :features])
-TRANSFORMER_SPECS = Specs.new(:specs => [:null, :store, :returndifferent, :transform_value, :increment, :create, :features])
+ADAPTER_SPECS = Specs.new(:specs => [:null, :store, :returndifferent, :increment, :concurrent_increment, :concurrent_create, :persist, :multiprocess, :create, :features, :store_large], :key => %w(string), :value => %w(string))
+STANDARD_SPECS = Specs.new(:specs => [:null, :store, :returndifferent, :marshallable_key, :marshallable_value, :transform_value, :increment, :concurrent_increment, :concurrent_create, :persist, :multiprocess, :create, :features, :store_large])
+TRANSFORMER_SPECS = Specs.new(:specs => [:null, :store, :returndifferent, :transform_value, :increment, :create, :features, :store_large])
 
 header = "# Generated by #{File.basename(__FILE__)}\n"
 
@@ -273,12 +277,12 @@ end
   'simple_sdbm' => {
     :store => :SDBM,
     :options => ':file => File.join(make_tempdir, "simple_sdbm")',
-    :specs => STANDARD_SPECS.without_multiprocess
+    :specs => STANDARD_SPECS.without_multiprocess.without_large
   },
   'simple_sdbm_with_expires' => {
     :store => :SDBM,
     :options => ':file => File.join(make_tempdir, "simple_sdbm_with_expires"), :expires => true',
-    :specs => STANDARD_SPECS.without_multiprocess.with_expires
+    :specs => STANDARD_SPECS.without_multiprocess.with_expires.without_large
   },
   'simple_leveldb' => {
     :store => :LevelDB,
@@ -1111,13 +1115,36 @@ it 'updates an existing key/value' do
   records.count.should == 1
 end
 
-it 'uses an existing connection' do
-  ActiveRecord::Base.establish_connection :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta', :username => 'root'
+it 'supports different tables same database' do
+  store1 = Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord1', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta', :username => 'root' })
+  store2 = Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord2', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta', :username => 'root' })
+
+  store1['key'] = 'value1'
+  store2['key'] = 'value2'
+  store1['key'].should == 'value1'
+  store2['key'].should == 'value2'
 
-  store = Moneta::Adapters::ActiveRecord.new(:table => 'activerecord_existing')
-  store.table.should be_table_exists
+  store1.close
+  store2.close
 end
-}
+
+it 'supports different databases same table' do
+  store1 = Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta_activerecord1', :username => 'root' })
+  store2 = Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta_activerecord2', :username => 'root' })
+
+  store1['key'] = 'value1'
+  store2['key'] = 'value2'
+  store1['key'].should == 'value1'
+  store2['key'].should == 'value2'
+
+  store1.close
+  store2.close
+end}
+  },
+  'adapter_activerecord_exisiting_connection' => {
+    :preamble => "require 'active_record'\nActiveRecord::Base.establish_connection :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta', :username => 'root'\n",
+    :build => "Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord_existing_connection')",
+    :specs => ADAPTER_SPECS
   },
   'adapter_client' => {
     :preamble => "start_server(Moneta::Adapters::Memory.new)\n",
@@ -1301,7 +1328,7 @@ end}
   },
   'adapter_sdbm' => {
     :build => 'Moneta::Adapters::SDBM.new(:file => File.join(make_tempdir, "adapter_sdbm"))',
-    :specs => ADAPTER_SPECS.without_multiprocess
+    :specs => ADAPTER_SPECS.without_multiprocess.without_large
   },
   'adapter_leveldb' => {
     :build => 'Moneta::Adapters::LevelDB.new(:dir => File.join(make_tempdir, "adapter_leveldb"))',
@@ -1720,6 +1747,18 @@ end}
   end
 end
 
+SPECS['store_large'] = %{it 'should store values up to 32k' do
+  value = 'x' * (32 * 1024)
+  store['large'] = value
+  store['large'].should == value
+end
+
+it 'should store keys up to 128 bytes' do
+  key = 'x' * 128
+  store[key] = 'value'
+  store[key].should == 'value'
+end}
+
 SPECS['not_persist'] = %{it 'does not persist values' do
   store['key'] = 'val'
   store.close
@@ -1985,17 +2024,20 @@ end
 it 'have atomic increment across multiple processes' do
   a = increment_thread('a')
   b = increment_thread('b')
+  c = increment_thread('c')
   a.join
   b.join
+  c.join
   1000.times do |i|
     store["a\#{i}"].should == i.to_s
     store["b\#{i}"].should == i.to_s
+    store["c\#{i}"].should == i.to_s
   end
-  store.raw['counter'].should == 2000.to_s
+  store.raw['counter'].should == 3000.to_s
 end}
 
 SPECS['concurrent_create'] = %{def create_thread(name)
-  a = Thread.new do
+  Thread.new do
     s = new_store
     1000.times do |i|
       s[i.to_s].should == name if s.create(i.to_s, name, :expires => false)
@@ -2008,8 +2050,10 @@ end
 it 'have atomic create across multiple processes' do
   a = create_thread('a')
   b = create_thread('b')
+  c = create_thread('c')
   a.join
   b.join
+  c.join
 end}
 
 SPECS['increment'] = %{it 'initializes in #increment with 1' do
@@ -2338,11 +2382,11 @@ TESTS.each do |name, options|
   code = %{#{header}require 'helper'
 
 describe_moneta #{name.inspect} do
-  def features
+  #{preamble}def features
     #{specs.features.to_a.inspect}
   end
 
-  #{preamble}def new_store
+  def new_store
     #{build.gsub("\n", "\n    ")}
   end
 

data/spec/moneta/adapter_activerecord_exisiting_connection_spec.rb

@@ -0,0 +1,31 @@
+# Generated by generate-specs
+require 'helper'
+
+describe_moneta "adapter_activerecord_exisiting_connection" do
+  require 'active_record'
+  ActiveRecord::Base.establish_connection :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta', :username => 'root'
+  def features
+    [:create, :increment]
+  end
+
+  def new_store
+    Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord_existing_connection')
+  end
+
+  def load_value(value)
+    Marshal.load(value)
+  end
+
+  include_context 'setup_store'
+  it_should_behave_like 'concurrent_create'
+  it_should_behave_like 'concurrent_increment'
+  it_should_behave_like 'create'
+  it_should_behave_like 'features'
+  it_should_behave_like 'increment'
+  it_should_behave_like 'multiprocess'
+  it_should_behave_like 'null_stringkey_stringvalue'
+  it_should_behave_like 'persist_stringkey_stringvalue'
+  it_should_behave_like 'returndifferent_stringkey_stringvalue'
+  it_should_behave_like 'store_stringkey_stringvalue'
+  it_should_behave_like 'store_large'
+end

data/spec/moneta/adapter_activerecord_spec.rb

@@ -25,6 +25,7 @@ describe_moneta "adapter_activerecord" do
   it_should_behave_like 'persist_stringkey_stringvalue'
   it_should_behave_like 'returndifferent_stringkey_stringvalue'
   it_should_behave_like 'store_stringkey_stringvalue'
+  it_should_behave_like 'store_large'
   it 'updates an existing key/value' do
     store['foo/bar'] = '1'
     store['foo/bar'] = '2'
@@ -32,11 +33,29 @@ describe_moneta "adapter_activerecord" do
     records.count.should == 1
   end
 
-  it 'uses an existing connection' do
-    ActiveRecord::Base.establish_connection :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta', :username => 'root'
+  it 'supports different tables same database' do
+    store1 = Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord1', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta', :username => 'root' })
+    store2 = Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord2', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta', :username => 'root' })
 
-    store = Moneta::Adapters::ActiveRecord.new(:table => 'activerecord_existing')
-    store.table.should be_table_exists
+    store1['key'] = 'value1'
+    store2['key'] = 'value2'
+    store1['key'].should == 'value1'
+    store2['key'].should == 'value2'
+
+    store1.close
+    store2.close
   end
 
+  it 'supports different databases same table' do
+    store1 = Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta_activerecord1', :username => 'root' })
+    store2 = Moneta::Adapters::ActiveRecord.new(:table => 'adapter_activerecord', :connection => { :adapter => (defined?(JRUBY_VERSION) ? 'jdbcmysql' : 'mysql2'), :database => 'moneta_activerecord2', :username => 'root' })
+
+    store1['key'] = 'value1'
+    store2['key'] = 'value2'
+    store1['key'].should == 'value1'
+    store2['key'].should == 'value2'
+
+    store1.close
+    store2.close
+  end
 end