moneta 0.7.19 → 0.7.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 54ab5446cbb72cf10100e2e72680c4624113aaa0
-  data.tar.gz: c3337a129506fdeab6624542b37ada15a9c16c20
+  metadata.gz: d992dc7597746374044079fd618a13b1e988766a
+  data.tar.gz: fec8939551039410e2412294f2fa19755413edfe
 SHA512:
-  metadata.gz: dc94d4af7c904ad023cee82ee3ec408e9685273362a94a6cf620ba0d108ea69b660ea8a737bd91491faacd10c09d0e5b1942f429b19fb010df806980e9f1f88b
-  data.tar.gz: 50a2df5e71906df99c2954ed1efb8ca2986234db51f21565c53bbbbdf93e2b07f123d041c6c165a0b24e9876852cbc23d2b4877e94261cf0f8b4bda946da53c1
+  metadata.gz: 13fa047663d93ec878871a9aaac054589dbc3135994bb320575b2c55185d88a3b395658350b2d140052ca9217a2b5558d17513f1354a409759fb9e8b399778ab
+  data.tar.gz: a93d886f2b8ca3ddb1ab258603ba3b3b9ceb318d814b99022c107e9aa138041ccd564f679e90ea0e18b440ef4421a46f405b578f854388f5fcf0503881b7cdb6

data/.travis.yml

@@ -8,7 +8,6 @@ rvm:
   - rbx-19mode
   - rbx-18mode
 before_install:
-  - script/kill-travis
   #- script/install-kyotocabinet
   - sudo apt-get install -qq libtokyocabinet8 libtokyocabinet-dev liblzo2-dev libtdb-dev libleveldb-dev tokyotyrant
   - script/start-services

data/CHANGES

@@ -1,3 +1,14 @@
+0.7.20
+
+* Adapters::LRUHash: add option :max_value
+* Moneta.new(:Couch, :Riak, :RestClient): use urlencode instead of base64 for key encoding)
+* Transformer: Add :hex encoder
+* Transformer: Don't wrap object in array for JSON serialization
+* Transformer: Add :php serializer
+* Moneta.new(:Sequel) - Don't encode blob data using base64
+* Moneta::Adapters::LMDB added (Symas Lightning Memory-Mapped Database)
+* Moneta::Adapters::Sequel - Fix for https://github.com/jeremyevans/sequel/issues/715
+
 0.7.19
 
 * ActionDispatch::Session::MonetaStore fixed for Rails 4

data/Gemfile

@@ -22,6 +22,7 @@ gem 'ox', :platforms => :ruby
 gem 'msgpack', :platforms => :ruby
 gem 'msgpack-jruby', :platforms => :jruby
 gem 'bert', :platforms => :ruby
+gem 'php_serialize'
 
 # Compressors used by Transformer
 if RUBY_VERSION < '2.0'
@@ -57,6 +58,9 @@ gem 'tokyotyrant'
 #gem 'localmemcache'
 gem 'tdb', :platforms => :ruby
 gem 'leveldb-ruby', :platforms => :ruby
+if RUBY_VERSION >= '1.9'
+  gem 'lmdb', :platforms => :mri
+end
 if RUBY_VERSION < '2.0'
   gem 'tokyocabinet', :platforms => :ruby
 end
@@ -82,3 +86,8 @@ gem 'rack-cache'
 # Rails integration testing
 gem 'actionpack', *rails_version
 gem 'minitest', '~> 4.7.4'
+
+# Fix versions for old ruby 1.8
+if RUBY_VERSION < '1.9'
+  gem 'nokogiri', '< 1.6'
+end

data/README.md

@@ -105,6 +105,7 @@ Out of the box, it supports the following backends. Use the backend name symbol
     * [GDBM](http://www.ruby-doc.org/stdlib/libdoc/gdbm/rdoc/GDBM.html) (`:GDBM`)
     * [HBase](http://hbase.apache.org/) (`:HBase`)
     * [LevelDB](http://code.google.com/p/leveldb/) (`:LevelDB`)
+    * [LMDB](http://symas.com/mdb/) (`:LMDB`)
     * [Redis](http://redis.io/) (`:Redis`)
     * [Riak](http://docs.basho.com/) (`:Riak`)
     * [SDBM](http://www.ruby-doc.org/stdlib/libdoc/sdbm/rdoc/SDBM.html) (`:SDBM`)
@@ -145,6 +146,8 @@ __NOTE:__ <a name="backend-matrix">The backend matrix</a> is much more readable
 
 <tr><td>File</td><td>-</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#F44">✗</td><td style="text-align:center;background:#5F5">✓</td><td>File store</td></tr>
 
+<tr><td>LMDB</td><td>lmdb</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#F44">✗</td><td style="text-align:center;background:#5F5">✓</td><td><a href="http://symas.com/mdb/">Symas Lightning Memory-Mapped Database (LMDB)</a></td></tr>
+
 <tr><td>Sequel</td><td>sequel</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#F44">✗</td><td style="text-align:center;background:#5F5">✓</td><td><a href="http://sequel.rubyforge.org/">Sequel</a> ORM</td></tr>
 
 <tr><td>TokyoTyrant</td><td>tokyotyrant or ruby-tokyotyrant</td><td style="text-align:center;background:#F44">✗</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#5F5">✓</td><td style="text-align:center;background:#F44">✗</td><td style="text-align:center;background:#5F5">✓</td><td><a href="http://fallabs.com/tokyotyrant/">TokyoTyrant</a> database</td></tr>
@@ -210,7 +213,7 @@ __NOTE:__ <a name="backend-matrix">The backend matrix</a> is much more readable
 * [1]: Make adapters thread-safe by using `Moneta::Lock` or by passing the option `:threadsafe => true` to `Moneta#new`. There is also `Moneta::Pool` which can be used to share a store between multiple threads if the store is multi-process safe. I recommend to add the option `:threadsafe` to ensure thread-safety since for example under JRuby and Rubinius even the basic datastructures are not thread safe due to the lack of a global interpreter lock (GIL). This differs from MRI where some adapters might appear thread safe already but only due to the GIL.
 * [2]: Share a Moneta store between multiple processes using `Moneta::Shared` (See below).
 * [3]: Add expiration support by using `Moneta::Expires` or by passing the option `:expires => true` to `Moneta#new`.
-* [4]: There are some servers which use the memcached protocol but which are persistent (e.g. [MemcacheDB](http://memcachedb.org/), [Kai](http://sourceforge.net/apps/mediawiki/kai), [IronCache](http://dev.iron.io/cache/reference/memcache/), [Roma](https://github.com/roma/roma/tree) and [Flare](http://labs.gree.jp/Top/OpenSource/Flare-en.html))
+* [4]: There are some servers which use the memcached protocol but which are persistent (e.g. [MemcacheDB](http://memcachedb.org/), [Kai](http://sourceforge.net/apps/mediawiki/kai), [IronCache](http://dev.iron.io/cache/reference/memcache/), [Roma](https://github.com/roma/roma/tree), [Flare](http://labs.gree.jp/Top/OpenSource/Flare-en.html) and [Kumofs](https://github.com/etolabo/kumofs))
 * [5]: Depends on server
 * [6]: Store is multi-process safe because it is an in-memory store, values are not shared between multiple processes
 * [7]: Store is multi-process safe, but not synchronized automatically between multiple processes
@@ -229,7 +232,7 @@ add additional features to storage backends:
 * `Moneta::Expires` to add expiration support to stores which don't support it natively. Add it in the builder using `use :Expires`.
 * `Moneta::Stack` to stack multiple stores (Read returns result from first where the key is found, writes go to all stores). Add it in the builder using `use(:Stack) {}`.
 * `Moneta::Transformer` transforms keys and values (Marshal, YAML, JSON, Base64, MD5, ...). Add it in the builder using `use :Transformer`.
-* `Moneta::Cache` combine two stores, one as backend and one as cache (e.g. `Moneta::Adapters::File` + `Moneta::Adapters::Memory`). Add it in the builder using `use(:Cache) {}`.
+* `Moneta::Cache` combine two stores, one as backend and one as cache (e.g. `Moneta::Adapters::File` + `Moneta::Adapters::LRUHash`). Add it in the builder using `use(:Cache) {}`.
 * `Moneta::Lock` to make store thread safe. Add it in the builder using `use :Lock`.
 * `Moneta::Pool` to create a pool of stores as a means of making the store thread safe. Add it in the builder using `use(:Pool) {}`.
 * `Moneta::Logger` to log database accesses. Add it in the builder using `use :Logger`.
@@ -247,11 +250,13 @@ Supported serializers:
 * Marshal (`:marshal`)
 * MessagePack (`:msgpack`)
 * Ox (`:ox`)
+* PHP (`:php`)
 * TNetStrings (`:tnet`)
 * YAML (`:yaml`)
 
 Supported value compressors:
 
+* Bzip2 (`:bzip2`)
 * LZ4 (`:lz4`)
 * LZMA (`:lzma`)
 * LZO (`:lzo`)
@@ -259,6 +264,14 @@ Supported value compressors:
 * QuickLZ (`:quicklz`)
 * Zlib (`:zlib`)
 
+Supported encoders:
+
+* Base64 (`:base64`)
+* Url escape (`:escape`)
+* Hexadecimal (`:hex`)
+* QP (`:qp`)
+* UUEncode (`:uuencode`)
+
 Special transformers:
 
 * Digests (MD5, Shas, CityHash, ...)

data/lib/moneta.rb

@@ -40,6 +40,7 @@ module Moneta
     autoload :LRUHash,         'moneta/adapters/lruhash'
     autoload :KyotoCabinet,    'moneta/adapters/kyotocabinet'
     autoload :LevelDB,         'moneta/adapters/leveldb'
+    autoload :LMDB,            'moneta/adapters/lmdb'
     autoload :LocalMemCache,   'moneta/adapters/localmemcache'
     autoload :Memcached,       'moneta/adapters/memcached'
     autoload :MemcachedDalli,  'moneta/adapters/memcached/dalli'
@@ -101,15 +102,13 @@ module Moneta
     transformer[:value] << (Symbol === compress ? compress : :zlib) if compress
     raise ArgumentError, 'Name must be Symbol' unless Symbol === name
     case name
-    when :Sequel, :ActiveRecord, :Couch, :DataMapper
-      # Sequel, DataMapper and AR accept only base64 keys and values
-      # FIXME: Couch should work only with :marshal but this raises an error on 1.9
+    when :Sequel
+      # Sequel accept only base64 keys
       transformer[:key] << :base64
-      transformer[:value] << :base64
-    when :Riak, :RestClient
-      # Riak accepts only utf-8 keys over the http interface
-      # We use base64 encoding therefore.
+    when :ActiveRecord, :DataMapper, :Couch
+      # DataMapper and AR accept only base64 keys and values
       transformer[:key] << :base64
+      transformer[:value] << :base64
     when :PStore, :YAML, :Null
       # For PStore and YAML only the key has to be a string
       transformer.delete(:value) if transformer[:value] == [:marshal]
@@ -117,8 +116,8 @@ module Moneta
       # Use spreading hashes
       transformer[:key] << :md5 << :spread
       name = :File
-    when :File
-      # Use escaping
+    when :File, :Riak, :RestClient
+      # Use escaping for file and HTTP interfaces
       transformer[:key] << :escape
     end
     a = Adapters.const_get(name).new(options)

data/lib/moneta/adapters/cassandra.rb

@@ -18,14 +18,20 @@ module Moneta
       # @option options [Integer] :port (9160) Server port
       # @option options [Integer] :expires Default expiration time
       # @option options [::Cassandra] :backend Use existing backend instance
+      # @option options Other options passed to `Cassandra#new`
       def initialize(options = {})
-        self.default_expires = options[:expires]
-        @cf = (options[:column_family] || 'moneta').to_sym
+        self.default_expires = options.delete(:expires)
+        @cf = (options.delete(:column_family) || 'moneta').to_sym
         if options[:backend]
           @backend = options[:backend]
         else
-          keyspace = options[:keyspace] || 'moneta'
-          @backend = ::Cassandra.new('system', "#{options[:host] || '127.0.0.1'}:#{options[:port] || 9160}")
+          keyspace = options.delete(:keyspace) || 'moneta'
+          options[:retries] ||= 3
+          options[:connect_timeout] ||= 10
+          options[:timeout] ||= 10
+          @backend = ::Cassandra.new('system',
+                                     "#{options[:host] || '127.0.0.1'}:#{options[:port] || 9160}",
+                                     options)
           unless @backend.keyspaces.include?(keyspace)
             cf_def = ::Cassandra::ColumnFamily.new(:keyspace => keyspace, :name => @cf.to_s)
             ks_def = ::Cassandra::Keyspace.new(:name => keyspace,
@@ -71,10 +77,6 @@ module Moneta
       def store(key, value, options = {})
         @backend.insert(@cf, key, {'value' => value}, :ttl => expires_value(options) || nil)
         value
-      rescue
-        # FIXME: We get spurious cassandra transport exceptions
-        tries ||= 0
-        (tries += 1) < 10 ? retry : raise
       end
 
       # (see Proxy#delete)
@@ -87,9 +89,7 @@ module Moneta
 
       # (see Proxy#clear)
       def clear(options = {})
-        @backend.each_key(@cf) do |key|
-          delete(key)
-        end
+        @backend.clear_column_family!(@cf)
         self
       end
 

data/lib/moneta/adapters/lmdb.rb

@@ -0,0 +1,93 @@
+require 'lmdb'
+require 'fileutils'
+
+module Moneta
+  module Adapters
+    # LMDB backend
+    # @api public
+    class LMDB
+      include Defaults
+
+      supports :create, :increment
+      attr_reader :backend, :db
+
+      PUT_FLAGS = [:nooverwrite, :nodupdata, :current, :append, :appenddup]
+
+      # @param [Hash] options
+      # @option options [String] :dir Environment directory
+      # @option options [::LMDB::Environment] :backend Use existing backend instance
+      # @option options [String or nil] :db Database name
+      def initialize(options)
+        db = options.delete(:db)
+        @backend = options.delete(:backend) ||
+          begin
+            raise ArgumentError, 'Option :dir is required' unless dir = options.delete(:dir)
+            FileUtils.mkpath(dir)
+            ::LMDB.new(dir, options)
+          end
+
+        @db = @backend.database(db, :create => true)
+      end
+
+      # (see Proxy#key?)
+      def key?(key, options = {})
+        !@db.get(key).nil?
+      end
+
+      # (see Proxy#load)
+      def load(key, options = {})
+        @db.get(key)
+      end
+
+      # (see Proxy#store)
+      def store(key, value, options = {})
+        @db.put(key, value, Utils.only(options, *PUT_FLAGS))
+        value
+      end
+
+      # (see Proxy#delete)
+      def delete(key, options = {})
+        @backend.transaction do
+          if value = @db.get(key)
+            @db.delete(key)
+            value
+          end
+        end
+      end
+
+      # (see Proxy#clear)
+      def clear(options = {})
+        @db.clear
+        self
+      end
+
+      # (see Proxy#increment)
+      def increment(key, amount = 1, options = {})
+        @backend.transaction do
+          value = @db.get(key)
+          value = Utils.to_int(value) + amount
+          @db.put(key, value.to_s, Utils.only(options, *PUT_FLAGS))
+          value
+        end
+      end
+
+      # (see Defaults#create)
+      def create(key, value, options = {})
+        @backend.transaction do
+          if @db.get(key)
+            false
+          else
+            @db.put(key, value, Utils.only(options, *PUT_FLAGS))
+            true
+          end
+        end
+      end
+
+      # (see Proxy#close)
+      def close
+        @backend.close
+        nil
+      end
+    end
+  end
+end

data/lib/moneta/adapters/lruhash.rb

@@ -11,11 +11,13 @@ module Moneta
       include CreateSupport
 
       # @param [Hash] options
-      # @option options [Integer] :max_size (1024000) Maximum total byte size of hash values
-      # @option options [Integer] :max_count (10240) Maximum number of hash values
+      # @option options [Integer] :max_size (1024000) Maximum byte size of all values
+      # @option options [Integer] :max_value (options[:max_size]) Maximum byte size of one value
+      # @option options [Integer] :max_count (10240) Maximum number of values
       def initialize(options = {})
         @max_size = options[:max_size] || 1024000
         @max_count = options[:max_count] || 10240
+        @max_value = options[:max_value] || @max_size
         clear
       end
 
@@ -34,16 +36,20 @@ module Moneta
 
       # (see Proxy#store)
       def store(key, value, options = {})
-        if entry = @entry[key]
-          @size -= entry.value.bytesize
+        if value.bytesize > @max_value
+          delete(key)
         else
-          @entry[key] = entry = Entry.new
-          entry.key = key
+          if entry = @entry[key]
+            @size -= entry.value.bytesize
+          else
+            @entry[key] = entry = Entry.new
+            entry.key = key
+          end
+          entry.value = value
+          @size += entry.value.bytesize
+          entry.insert_after(@list)
+          delete(@list.prev.key) while @list.next != @list.prev && (@size > @max_size || @entry.size > @max_count)
         end
-        entry.value = value
-        @size += entry.value.bytesize
-        entry.insert_after(@list)
-        delete(@list.prev.key) while @list.next != @list.prev && (@size > @max_size || @entry.size > @max_count)
         value
       end
 

data/lib/moneta/adapters/mongo.rb

@@ -101,9 +101,8 @@ module Moneta
         key = to_binary(key)
         @collection.insert(value_to_doc(key, value, options))
         true
-      rescue ::Mongo::OperationFailure
-        # FIXME: This catches too many errors
-        # it should only catch a not-unique-exception
+      rescue ::Mongo::OperationFailure => ex
+        raise if ex.error_code != 11000 # duplicate key error
         false
       end
 

data/lib/moneta/adapters/sequel.rb

@@ -47,9 +47,9 @@ module Moneta
       # (see Proxy#store)
       def store(key, value, options = {})
         begin
-          @table.insert(:k => key, :v => value)
+          @table.insert(:k => key, :v => blob(value))
         rescue UniqueConstraintViolation
-          @table.where(:k => key).update(:v => value)
+          @table.where(:k => key).update(:v => blob(value))
         end
         value
       rescue ::Sequel::DatabaseError
@@ -59,7 +59,7 @@ module Moneta
 
       # (see Proxy#store)
       def create(key, value, options = {})
-        @table.insert(:k => key, :v => value)
+        @table.insert(:k => key, :v => blob(value))
         true
       rescue UniqueConstraintViolation
         false
@@ -71,10 +71,10 @@ module Moneta
           locked_table = @table.for_update
           if record = locked_table[:k => key]
             value = Utils.to_int(record[:v]) + amount
-            locked_table.where(:k => key).update(:v => value.to_s)
+            locked_table.where(:k => key).update(:v => blob(value.to_s))
             value
           else
-            locked_table.insert(:k => key, :v => amount.to_s)
+            locked_table.insert(:k => key, :v => blob(amount.to_s))
             amount
           end
         end
@@ -103,6 +103,13 @@ module Moneta
         @backend.disconnect
         nil
       end
+
+      private
+
+      # See https://github.com/jeremyevans/sequel/issues/715
+      def blob(s)
+        s.empty? ? '' : ::Sequel.blob(s)
+      end
     end
   end
 end

data/lib/moneta/cache.rb

@@ -53,8 +53,7 @@ module Moneta
 
     # (see Proxy#load)
     def load(key, options = {})
-      value = @cache.load(key, options)
-      if value == nil
+      if options[:sync] || (value = @cache.load(key, options)) == nil
         value = @adapter.load(key, options)
         @cache.store(key, value, options) if value != nil
       end

data/lib/moneta/proxy.rb

@@ -39,6 +39,7 @@ module Moneta
     # @option options [Integer] :expires Update expiration time (See {Expires})
     # @option options [Boolean] :raw Raw access without value transformation (See {Transformer})
     # @option options [String] :prefix Prefix key (See {Transformer})
+    # @option options [Boolean] :sync Synchronized load ({Cache} reloads from adapter, {Adapters::Daybreak} syncs with file)
     # @option options Other options as defined by the adapters or middleware
     # @return [Object] value
     # @api public

data/lib/moneta/synchronize.rb

@@ -31,11 +31,10 @@ module Moneta
     # @param [Number] wait Sleep time between tries to acquire lock
     # @return [Boolean] true if the lock was aquired
     def enter(timeout = nil, wait = 0.01)
-      total = 0
-      while !timeout || total < timeout
+      time_at_timeout = Time.now + timeout if timeout
+      while !timeout || Time.now < time_at_timeout
         return true if try_enter
         sleep(wait)
-        total += wait
       end
       false
     end

data/lib/moneta/transformer/config.rb

@@ -6,10 +6,11 @@ module Moneta
       :bencode  => [ :serialize, '::BEncode.load(%s)',           '::BEncode.dump(%s)',              'bencode'       ],
       :bert     => [ :serialize, '::BERT.decode(%s)',            '::BERT.encode(%s)',               'bert'          ],
       :bson     => [ :serialize, "::BSON.deserialize(%s)['v']",  "::BSON.serialize('v'=>%s).to_s",  'bson'          ],
-      :json     => [ :serialize, '::MultiJson.load(%s).first',   '::MultiJson.dump([%s])',          'multi_json'    ],
+      :json     => [ :serialize, '::MultiJson.load(%s)',         '::MultiJson.dump(%s)',            'multi_json'    ],
       :marshal  => [ :serialize, '::Marshal.load(%s)',           '::Marshal.dump(%s)'                               ],
       :msgpack  => [ :serialize, '::MessagePack.unpack(%s)',     '::MessagePack.pack(%s)',          'msgpack'       ],
       :ox       => [ :serialize, '::Ox.parse_obj(%s)',           '::Ox.dump(%s)',                   'ox'            ],
+      :php      => [ :serialize, '::PHP.unserialize(%s)',        '::PHP.serialize(%s)',           'php_serialize' ],
       :tnet     => [ :serialize, '::TNetstring.parse(%s).first', '::TNetstring.dump(%s)',           'tnetstring'    ],
       :yaml     => [ :serialize, '::YAML.load(%s)',              '::YAML.dump(%s)',                 'yaml'          ],
       :bzip2    => [ :compress,  '::Bzip2.uncompress(%s)',       '::Bzip2.compress(%s)',            'bzip2'         ],
@@ -19,12 +20,13 @@ module Moneta
       :snappy   => [ :compress,  '::Snappy.inflate(%s)',         '::Snappy.deflate(%s)',            'snappy'        ],
       :quicklz  => [ :compress,  '::QuickLZ.decompress(%s)',     '::QuickLZ.compress(%s)',          'qlzruby'       ],
       :zlib     => [ :compress,  '::Zlib::Inflate.inflate(%s)',  '::Zlib::Deflate.deflate(%s)',     'zlib'          ],
-      :qp       => [ :encode,    "%s.unpack('M').first",         "[%s].pack('M')"                                   ],
-      :uuencode => [ :encode,    "%s.unpack('u').first",         "[%s].pack('u')"                                   ],
       :base64   => RUBY_VERSION > '1.9' ?
                    [ :encode,    "%s.unpack('m0').first",        "[%s].pack('m0')"                                  ] :
                    [ :encode,    "%s.unpack('m').first",         "[%s].pack('m').gsub(\"\n\", '')"                  ],
       :escape   => [ :encode,    'Helper.unescape(%s)',          'Helper.escape(%s)'                                ],
+      :hex      => [ :encode,    "[%s].pack('H*')",              "%s.unpack('H*').first"                            ],
+      :qp       => [ :encode,    "%s.unpack('M').first",         "[%s].pack('M')"                                   ],
+      :uuencode => [ :encode,    "%s.unpack('u').first",         "[%s].pack('u')"                                   ],
       :hmac     => [ :hmac,      'Helper.hmacverify(%s, options[:secret] || @secret)',
                                  'Helper.hmacsign(%s, options[:secret] || @secret)',                'openssl'       ],
       :truncate => [ :truncate,  nil,                            'Helper.truncate(%s, @maxlen)',    'digest/md5'    ],