moneta 1.0.0 → 1.5.1

data/lib/moneta/adapters/sequel.rb

@@ -4,61 +4,89 @@ module Moneta
   module Adapters
     # Sequel backend
     # @api public
-    class Sequel
-      include Defaults
+    class Sequel < Adapter
+      autoload :MySQL, 'moneta/adapters/sequel/mysql'
+      autoload :Postgres, 'moneta/adapters/sequel/postgres'
+      autoload :PostgresHStore, 'moneta/adapters/sequel/postgres_hstore'
+      autoload :SQLite, 'moneta/adapters/sequel/sqlite'
 
-      # Sequel::UniqueConstraintViolation is defined since sequel 3.44.0
-      # older versions raise a Sequel::DatabaseError.
-      UniqueConstraintViolation = defined?(::Sequel::UniqueConstraintViolation) ? ::Sequel::UniqueConstraintViolation : ::Sequel::DatabaseError
+      supports :create, :increment, :each_key
 
-      supports :create, :increment
-      attr_reader :backend
+      config :table, default: :moneta, coerce: :to_sym
+      config :optimize, default: true
+      config :create_table, default: true
+      config :key_column, default: :k
+      config :value_column, default: :v
+      config :hstore, coerce: :to_s
+      config :each_key_server
+
+      backend do |db:, extensions: [], connection_validation_timeout: nil, **options|
+        ::Sequel.connect(db, options).tap do |backend|
+          extensions.map(&:to_sym).each(&backend.method(:extension))
+
+          if connection_validation_timeout
+            backend.pool.connection_validation_timeout = connection_validation_timeout
+          end
+        end
+      end
 
       # @param [Hash] options
       # @option options [String] :db Sequel database
-      # @option options [String/Symbol] :table (:moneta) Table name
+      # @option options [String, Symbol] :table (:moneta) Table name
       # @option options [Array] :extensions ([]) List of Sequel extensions
       # @option options [Integer] :connection_validation_timeout (nil) Sequel connection_validation_timeout
+      # @option options [Sequel::Database] :backend Use existing backend instance
+      # @option options [Boolean] :optimize (true) Set to false to prevent database-specific optimisations
+      # @option options [Proc, Boolean] :create_table (true) Provide a Proc for creating the table, or
+      #   set to false to disable table creation all together.  If a Proc is given, it will be
+      #   called regardless of whether the table exists already.
+      # @option options [Symbol] :key_column (:k) The name of the key column
+      # @option options [Symbol] :value_column (:v) The name of the value column
+      # @option options [String] :hstore If using Postgres, keys and values are stored in a single
+      #   row of the table in the value_column using the hstore format.  The row to use is
+      #   the one where the value_column is equal to the value of this option, and will be created
+      #   if it doesn't exist.
+      # @option options [Symbol] :each_key_server Some adapters are unable to do
+      #   multiple operations with a single connection. For these, it is
+      #   possible to specify a separate connection to use for `#each_key`.  Use
+      #   in conjunction with Sequel's `:servers` option
       # @option options All other options passed to `Sequel#connect`
-      # @option options [Sequel connection] :backend Use existing backend instance
       def initialize(options = {})
-        table = (options.delete(:table) || :moneta).to_sym
-        extensions = options.delete(:extensions) || []
-        raise ArgumentError, 'Option :extensions must be an Array' unless extensions.is_a?(Array)
-        connection_validation_timeout = options.delete(:connection_validation_timeout)
-        @backend = options[:backend] ||
-          begin
-            raise ArgumentError, 'Option :db is required' unless db = options.delete(:db)
-            ::Sequel.connect(db, options)
-          end
-        extensions.each do |extension|
-          @backend.extension(extension.to_sym)
+        super
+
+        if config.hstore
+          extend Sequel::PostgresHStore
+        elsif config.optimize
+          add_optimizations
         end
-        @backend.pool.connection_validation_timeout = connection_validation_timeout if connection_validation_timeout
-        @backend.create_table?(table) do
-          String :k, null: false, primary_key: true
-          File :v
+
+        if config.create_table.respond_to?(:call)
+          config.create_table.call(backend)
+        elsif config.create_table
+          create_table
         end
-        @table = @backend[table]
+
+        @table = backend[config.table]
+        prepare_statements
       end
 
       # (see Proxy#key?)
       def key?(key, options = {})
-        @table[k: key] != nil
+        @key.call(key: key) != nil
       end
 
       # (see Proxy#load)
       def load(key, options = {})
-        record = @table[k: key]
-        record && record[:v]
+        if row = @load.call(key: key)
+          row[config.value_column]
+        end
       end
 
       # (see Proxy#store)
       def store(key, value, options = {})
-        begin
-          @table.insert(k: key, v: blob(value))
-        rescue UniqueConstraintViolation
-          @table.where(k: key).update(v: blob(value))
+        blob_value = blob(value)
+        unless @store_update.call(key: key, value: blob_value) == 1
+          @create.call(key: key, value: blob_value)
         end
         value
       rescue ::Sequel::DatabaseError
@@ -66,26 +94,29 @@ module Moneta
         (tries += 1) < 10 ? retry : raise
       end
 
-      # (see Proxy#store)
+      # (see Proxy#create)
       def create(key, value, options = {})
-        @table.insert(k: key, v: blob(value))
+        @create.call(key: key, value: blob(value))
         true
-      rescue UniqueConstraintViolation
+      rescue ::Sequel::ConstraintViolation
         false
       end
 
       # (see Proxy#increment)
       def increment(key, amount = 1, options = {})
-        @backend.transaction do
-          locked_table = @table.for_update
-          if record = locked_table[k: key]
-            value = Utils.to_int(record[:v]) + amount
-            locked_table.where(k: key).update(v: blob(value.to_s))
-            value
+        backend.transaction do
+          if existing = @load_for_update.call(key: key)
+            existing_value = existing[config.value_column]
+            amount += Integer(existing_value)
+            raise IncrementError, "no update" unless @increment_update.call(
+              key: key,
+              value: existing_value,
+              new_value: blob(amount.to_s)
+            ) == 1
           else
-            locked_table.insert(k: key, v: blob(amount.to_s))
-            amount
+            @create.call(key: key, value: blob(amount.to_s))
           end
+          amount
         end
       rescue ::Sequel::DatabaseError
         # Concurrent modification might throw a bunch of different errors
@@ -95,10 +126,9 @@ module Moneta
 
       # (see Proxy#delete)
       def delete(key, options = {})
-        if value = load(key, options)
-          @table.filter(k: key).delete
-          value
-        end
+        value = load(key, options)
+        @delete.call(key: key)
+        value
       end
 
       # (see Proxy#clear)
@@ -109,16 +139,187 @@ module Moneta
 
       # (see Proxy#close)
       def close
-        @backend.disconnect
+        backend.disconnect
         nil
       end
 
-      private
+      # (see Proxy#slice)
+      def slice(*keys, **options)
+        @slice.all(keys).map! { |row| [row[config.key_column], row[config.value_column]] }
+      end
+
+      # (see Proxy#values_at)
+      def values_at(*keys, **options)
+        pairs = Hash[slice(*keys, **options)]
+        keys.map { |key| pairs[key] }
+      end
+
+      # (see Proxy#fetch_values)
+      def fetch_values(*keys, **options)
+        return values_at(*keys, **options) unless block_given?
+        existing = Hash[slice(*keys, **options)]
+        keys.map do |key|
+          if existing.key? key
+            existing[key]
+          else
+            yield key
+          end
+        end
+      end
+
+      # (see Proxy#merge!)
+      def merge!(pairs, options = {})
+        backend.transaction do
+          existing = Hash[slice_for_update(pairs)]
+          update_pairs, insert_pairs = pairs.partition { |k, _| existing.key?(k) }
+          @table.import([config.key_column, config.value_column], blob_pairs(insert_pairs))
+
+          if block_given?
+            update_pairs.map! do |key, new_value|
+              [key, yield(key, existing[key], new_value)]
+            end
+          end
+
+          update_pairs.each do |key, value|
+            @store_update.call(key: key, value: blob(value))
+          end
+        end
+
+        self
+      end
+
+      # (see Proxy#each_key)
+      def each_key
+        return enum_for(:each_key) { @table.count } unless block_given?
+
+        key_column = config.key_column
+        if config.each_key_server
+          @table.server(config.each_key_server).order(key_column).select(key_column).paged_each do |row|
+            yield row[key_column]
+          end
+        else
+          @table.select(key_column).order(key_column).paged_each(stream: false) do |row|
+            yield row[key_column]
+          end
+        end
+        self
+      end
+
+      protected
 
-      # See https://github.com/jeremyevans/sequel/issues/715
-      def blob(s)
-        s.empty? ? '' : ::Sequel.blob(s)
+      # @api private
+      def add_optimizations
+        case backend.database_type
+        when :mysql
+          extend Sequel::MySQL
+        when :postgres
+          if matches = backend.get(::Sequel[:version].function).match(/PostgreSQL (\d+)\.(\d+)/)
+            # Our optimisations only work on Postgres 9.5+
+            major, minor = matches[1..2].map(&:to_i)
+            extend Sequel::Postgres if major > 9 || (major == 9 && minor >= 5)
+          end
+        when :sqlite
+          extend Sequel::SQLite
+        end
+      end
+
+      def blob(str)
+        ::Sequel.blob(str) unless str == nil
+      end
+
+      def blob_pairs(pairs)
+        pairs.map do |key, value|
+          [key, blob(value)]
+        end
+      end
+
+      def create_table
+        key_column = config.key_column
+        value_column = config.value_column
+        backend.create_table?(config.table) do
+          String key_column, null: false, primary_key: true
+          File value_column
+        end
+      end
+
+      def slice_for_update(pairs)
+        @slice_for_update.all(pairs.map { |k, _| k }.to_a).map! do |row|
+          [row[config.key_column], row[config.value_column]]
+        end
+      end
+
+      def yield_merge_pairs(pairs)
+        existing = Hash[slice_for_update(pairs)]
+        pairs.map do |key, new_value|
+          new_value = yield(key, existing[key], new_value) if existing.key?(key)
+          [key, new_value]
+        end
+      end
+
+      def statement_id(id)
+        "moneta_#{config.table}_#{id}".to_sym
+      end
+
+      def prepare_statements
+        prepare_key
+        prepare_load
+        prepare_store
+        prepare_create
+        prepare_increment
+        prepare_delete
+        prepare_slice
+      end
+
+      def prepare_key
+        @key = @table
+          .where(config.key_column => :$key).select(1)
+          .prepare(:first, statement_id(:key))
+      end
+
+      def prepare_load
+        @load = @table
+          .where(config.key_column => :$key).select(config.value_column)
+          .prepare(:first, statement_id(:load))
       end
+
+      def prepare_store
+        @store_update = @table
+          .where(config.key_column => :$key)
+          .prepare(:update, statement_id(:store_update), config.value_column => :$value)
+      end
+
+      def prepare_create
+        @create = @table
+          .prepare(:insert, statement_id(:create), config.key_column => :$key, config.value_column => :$value)
+      end
+
+      def prepare_increment
+        @load_for_update = @table
+          .where(config.key_column => :$key).for_update
+          .select(config.value_column)
+          .prepare(:first, statement_id(:load_for_update))
+        @increment_update ||= @table
+          .where(config.key_column => :$key, config.value_column => :$value)
+          .prepare(:update, statement_id(:increment_update), config.value_column => :$new_value)
+      end
+
+      def prepare_delete
+        @delete = @table.where(config.key_column => :$key)
+          .prepare(:delete, statement_id(:delete))
+      end
+
+      def prepare_slice
+        @slice_for_update = ::Sequel::Dataset::PlaceholderLiteralizer.loader(@table) do |pl, ds|
+          ds.filter(config.key_column => pl.arg).select(config.key_column, config.value_column).for_update
+        end
+
+        @slice = ::Sequel::Dataset::PlaceholderLiteralizer.loader(@table) do |pl, ds|
+          ds.filter(config.key_column => pl.arg).select(config.key_column, config.value_column)
+        end
+      end
+
+      # @api private
+      class IncrementError < ::Sequel::DatabaseError; end
     end
   end
 end

data/lib/moneta/adapters/sqlite.rb

@@ -4,34 +4,52 @@ module Moneta
   module Adapters
     # Sqlite3 backend
     # @api public
-    class Sqlite
-      include Defaults
+    class Sqlite < Adapter
       include IncrementSupport
 
-      supports :create
-      attr_reader :backend
+      supports :create, :each_key
+
+      config :table, default: 'moneta'
+      config :busy_timeout, default: 1000
+      config :journal_mode
+
+      backend { |file:| ::SQLite3::Database.new(file) }
 
       # @param [Hash] options
       # @option options [String] :file Database file
       # @option options [String] :table ('moneta') Table name
-      # @option options [Fixnum] :busy_timeout (1000) Sqlite timeout if database is busy
+      # @option options [Integer] :busy_timeout (1000) Sqlite timeout if database is busy
       # @option options [::Sqlite3::Database] :backend Use existing backend instance
+      # @option options [String, Symbol] :journal_mode Set the journal mode for the connection
       def initialize(options = {})
-        table = options[:table] || 'moneta'
-        @backend = options[:backend] ||
-          begin
-            raise ArgumentError, 'Option :file is required' unless options[:file]
-            ::SQLite3::Database.new(options[:file])
-          end
-        @backend.busy_timeout(options[:busy_timeout] || 1000)
-        @backend.execute("create table if not exists #{table} (k blob not null primary key, v blob)")
+        super
+
+        backend.busy_timeout(config.busy_timeout)
+        backend.execute("create table if not exists #{config.table} (k blob not null primary key, v blob)")
+
+        if journal_mode = config.journal_mode
+          backend.journal_mode = journal_mode.to_s
+        end
+
         @stmts =
-          [@exists  = @backend.prepare("select exists(select 1 from #{table} where k = ?)"),
-           @select  = @backend.prepare("select v from #{table} where k = ?"),
-           @replace = @backend.prepare("replace into #{table} values (?, ?)"),
-           @delete  = @backend.prepare("delete from #{table} where k = ?"),
-           @clear   = @backend.prepare("delete from #{table}"),
-           @create  = @backend.prepare("insert into #{table} values (?, ?)")]
+          [@exists = backend.prepare("select exists(select 1 from #{config.table} where k = ?)"),
+           @select = backend.prepare("select v from #{config.table} where k = ?"),
+           @replace = backend.prepare("replace into #{config.table} values (?, ?)"),
+           @delete = backend.prepare("delete from #{config.table} where k = ?"),
+           @clear = backend.prepare("delete from #{config.table}"),
+           @create = backend.prepare("insert into #{config.table} values (?, ?)"),
+           @keys = backend.prepare("select k from #{config.table}"),
+           @count = backend.prepare("select count(*) from #{config.table}")]
+
+        version = backend.execute("select sqlite_version()").first.first
+        if @can_upsert = ::Gem::Version.new(version) >= ::Gem::Version.new('3.24.0')
+          @stmts << (@increment = backend.prepare <<-SQL)
+            insert into #{config.table} values (?, ?)
+            on conflict (k)
+            do update set v = cast(cast(v as integer) + ? as blob)
+            where v = '0' or v = X'30' or cast(v as integer) != 0
+          SQL
+        end
       end
 
       # (see Proxy#key?)
@@ -60,7 +78,11 @@ module Moneta
 
       # (see Proxy#increment)
       def increment(key, amount = 1, options = {})
-        @backend.transaction(:exclusive) { return super }
+        backend.transaction(:exclusive) { return super } unless @can_upsert
+        backend.transaction do
+          @increment.execute!(key, amount.to_s, amount)
+          return Integer(load(key))
+        end
       end
 
       # (see Proxy#clear)
@@ -71,7 +93,7 @@ module Moneta
 
       # (see Default#create)
       def create(key, value, options = {})
-        @create.execute!(key,value)
+        @create.execute!(key, value)
         true
       rescue SQLite3::ConstraintException
         # If you know a better way to detect whether an insert-ignore
@@ -82,10 +104,68 @@ module Moneta
 
       # (see Proxy#close)
       def close
-        @stmts.each {|s| s.close }
-        @backend.close
+        @stmts.each { |s| s.close }
+        backend.close
         nil
       end
+
+      # (see Proxy#slice)
+      def slice(*keys, **options)
+        query = "select k, v from #{config.table} where k in (#{(['?'] * keys.length).join(',')})"
+        backend.execute(query, keys)
+      end
+
+      # (see Proxy#values_at)
+      def values_at(*keys, **options)
+        hash = Hash[slice(*keys, **options)]
+        keys.map { |key| hash[key] }
+      end
+
+      # (see Proxy#fetch_values)
+      def fetch_values(*keys, **options)
+        return values_at(*keys, **options) unless block_given?
+        hash = Hash[slice(*keys, **options)]
+        keys.map do |key|
+          if hash.key?(key)
+            hash[key]
+          else
+            yield key
+          end
+        end
+      end
+
+      # (see Proxy#merge!)
+      def merge!(pairs, options = {})
+        transaction = backend.transaction if block_given?
+
+        if block_given?
+          existing = Hash[slice(*pairs.map { |k, _| k }.to_a)]
+          pairs = pairs.map do |key, new_value|
+            new_value = yield(key, existing[key], new_value) if existing.key?(key)
+            [key, new_value]
+          end.to_a
+        else
+          pairs = pairs.to_a
+        end
+
+        query = "replace into #{config.table} (k, v) values" + (['(?, ?)'] * pairs.length).join(',')
+        backend.query(query, pairs.flatten).close
+      rescue
+        backend.rollback if transaction
+        raise
+      else
+        backend.commit if transaction
+        self
+      end
+
+      # (see Proxy#each_key)
+      def each_key
+        return enum_for(:each_key) { @count.execute!.first.first } unless block_given?
+        @keys.execute!.each do |row|
+          yield row.first
+        end
+        self
+      end
     end
   end
 end

data/lib/moneta/adapters/tdb.rb

@@ -4,23 +4,32 @@ module Moneta
   module Adapters
     # TDB backend
     # @api public
-    class TDB < Memory
-      # @param [Hash] options
-      # @option options [String] :file Database file
-      # @option options [::TDB] :backend Use existing backend instance
-      def initialize(options)
-        @backend = options[:backend] ||
-          begin
-            raise ArgumentError, 'Option :file is required' unless file = options.delete(:file)
-            ::TDB.new(file, options)
-          end
-      end
+    class TDB < Adapter
+      include HashAdapter
+      include IncrementSupport
+      include EachKeySupport
+
+      supports :create
+
+      # @!method initialize(options = {})
+      #   @param [Hash] options
+      #   @option options [String] :file Database file
+      #   @option options [::TDB] :backend Use existing backend instance
+      backend { |file:, **options| ::TDB.new(file, options) }
 
       # (see Proxy#close)
       def close
-        @backend.close
+        backend.close
         nil
       end
+
+      # (see Proxy#create)
+      def create(key, value, options = {})
+        backend.insert!(key, value)
+        true
+      rescue ::TDB::ERR::EXISTS
+        false
+      end
     end
   end
 end

data/lib/moneta/adapters/tokyocabinet.rb

@@ -4,23 +4,31 @@ module Moneta
   module Adapters
     # TokyoCabinet backend
     # @api public
-    class TokyoCabinet < Memory
-      # @param [Hash] options
-      # @option options [String] :file Database file
-      # @option options [Symbol] :type (:hdb) Database type (:bdb and :hdb possible)
-      # @option options [::TokyoCabinet::*DB] :backend Use existing backend instance
-      def initialize(options = {})
-        if options[:backend]
-          @backend = options[:backend]
+    class TokyoCabinet < Adapter
+      include HashAdapter
+      include IncrementSupport
+      include CreateSupport
+      include EachKeySupport
+
+      # @!method initialize(options = {})
+      #   @param [Hash] options
+      #   @option options [String] :file Database file
+      #   @option options [Symbol] :type (:hdb) Database type (:bdb and :hdb possible)
+      #   @option options [::TokyoCabinet::*DB] :backend Use existing backend instance
+      backend do |file:, type: :hdb|
+        case type
+        when :bdb
+          ::TokyoCabinet::BDB.new.tap do |backend|
+            backend.open(file, ::TokyoCabinet::BDB::OWRITER | ::TokyoCabinet::BDB::OCREAT) or
+              raise backend.errmsg(backend.ecode)
+          end
+        when :hdb
+          ::TokyoCabinet::HDB.new.tap do |backend|
+            backend.open(file, ::TokyoCabinet::HDB::OWRITER | ::TokyoCabinet::HDB::OCREAT) or
+              raise backend.errmsg(backend.ecode)
+          end
         else
-          raise ArgumentError, 'Option :file is required' unless options[:file]
-          if options[:type] == :bdb
-            @backend = ::TokyoCabinet::BDB.new
-            @backend.open(options[:file], ::TokyoCabinet::BDB::OWRITER | ::TokyoCabinet::BDB::OCREAT)
-          else
-            @backend = ::TokyoCabinet::HDB.new
-            @backend.open(options[:file], ::TokyoCabinet::HDB::OWRITER | ::TokyoCabinet::HDB::OCREAT)
-          end or raise @backend.errmsg(@backend.ecode)
+          raise ArgumentError, ":type must be :bdb or :hdb"
         end
       end