moneta 1.1.0 → 1.4.0

data/lib/moneta/adapters/client.rb

@@ -12,51 +12,55 @@ module Moneta
       # @option options [String] :host ('127.0.0.1') Hostname
       # @option options [String] :socket Unix socket file name as alternative to `:port` and `:host`
       def initialize(options = {})
-        @socket = options[:socket] ? UNIXSocket.open(options[:socket]) :
-          TCPSocket.open(options[:host] || '127.0.0.1', options[:port] || 9000)
+        @socket =
+          if options[:socket]
+            UNIXSocket.open(options[:socket])
+          else
+            TCPSocket.open(options[:host] || '127.0.0.1', options[:port] || 9000)
+          end
       end
 
       # (see Proxy#key?)
       def key?(key, options = {})
         write(:key?, key, options)
-        read
+        read_msg
       end
 
       # (see Proxy#load)
       def load(key, options = {})
         write(:load, key, options)
-        read
+        read_msg
       end
 
       # (see Proxy#store)
       def store(key, value, options = {})
         write(:store, key, value, options)
-        read
+        read_msg
         value
       end
 
       # (see Proxy#delete)
       def delete(key, options = {})
         write(:delete, key, options)
-        read
+        read_msg
       end
 
       # (see Proxy#increment)
       def increment(key, amount = 1, options = {})
         write(:increment, key, amount, options)
-        read
+        read_msg
       end
 
       # (see Proxy#create)
       def create(key, value, options = {})
         write(:create, key, value, options)
-        read
+        read_msg
       end
 
       # (see Proxy#clear)
       def clear(options = {})
         write(:clear, options)
-        read
+        read_msg
         self
       end
 
@@ -66,12 +70,43 @@ module Moneta
         nil
       end
 
+      # (see Proxy#each_key)
+      def each_key
+        raise NotImplementedError, 'each_key is not supported' unless supports?(:each_key)
+        return enum_for(:each_key) unless block_given?
+
+        begin
+          write(:each_key)
+          yield_break = false
+
+          loop do
+            write('NEXT')
+
+            # A StopIteration error will be raised by this call if the server
+            # reached the end of the enumeration.  This will stop the loop
+            # automatically.
+            result = read_msg
+
+            # yield_break will be true in the ensure block (below) if anything
+            # happened during the yield to stop further enumeration.
+            yield_break = true
+            yield result
+            yield_break = false
+          end
+        ensure
+          write('BREAK') if yield_break
+          read_msg # nil return from each_key
+        end
+
+        self
+      end
+
       # (see Default#features)
       def features
         @features ||=
           begin
             write(:features)
-            read.freeze
+            read_msg.freeze
           end
       end
 
@@ -82,18 +117,24 @@ module Moneta
         @socket.write([s.bytesize].pack('N') << s)
       end
 
-      def read
-        size = @socket.
-          recv(4).tap do |bytes|
-            raise EOFError.new('failed to read size') unless bytes.bytesize == 4
-          end.
-          unpack('N').
-          first
-
-        result = Marshal.load(@socket.recv(size).tap do |bytes|
-          raise EOFError.new("Not enough bytes read") unless bytes.bytesize == size
-        end)
+      # JRuby doesn't support socket#recv with flags
+      if defined?(JRUBY_VERSION)
+        def read(bytes)
+          received = @socket.read(bytes)
+          raise EOFError, "Server closed socket" unless received && received.bytesize == bytes
+          received
+        end
+      else
+        def read(bytes)
+          received = @socket.recv(bytes, Socket::MSG_WAITALL)
+          raise EOFError, "Server closed socket" unless received && received.bytesize == bytes
+          received
+        end
+      end
 
+      def read_msg
+        size = read(4).unpack('N').first
+        result = Marshal.load(read(size))
         raise result if Exception === result
         result
       end

data/lib/moneta/adapters/couch.rb

@@ -1,6 +1,5 @@
 require 'faraday'
 require 'multi_json'
-require 'uri'
 
 module Moneta
   module Adapters
@@ -16,6 +15,19 @@ module Moneta
     class Couch
       include Defaults
 
+      # @api private
+      class HTTPError < StandardError
+        attr_reader :status, :request_method, :key
+
+        def initialize(status, request_method, key)
+          @status = status
+          @request_method = request_method.to_sym
+          @key = key
+
+          super "HTTP Error: #{@status} (#{@request_method.to_s.upcase} #{@key})"
+        end
+      end
+
       attr_reader :backend
 
       supports :create, :each_key
@@ -24,14 +36,31 @@ module Moneta
       # @option options [String] :host ('127.0.0.1') Couch host
       # @option options [String] :port (5984) Couch port
       # @option options [String] :db ('moneta') Couch database
+      # @option options [String] :scheme ('http') HTTP scheme to use
       # @option options [String] :value_field ('value') Document field to store value
       # @option options [String] :type_field ('type') Document field to store value type
-      # @option options [Faraday connection] :backend Use existing backend instance
+      # @option options [String] :login Login name to use for HTTP basic authentication
+      # @option options [String] :password Password to use for HTTP basic authentication
+      # @option options [Symbol] :adapter Adapter to use with Faraday
+      # @option options [Faraday::Connecton] :backend Use existing backend instance
+      # @option options Other options passed to {Faraday::new} (unless
+      #   :backend option is provided).
       def initialize(options = {})
-        @value_field = options[:value_field] || 'value'
-        @type_field = options[:type_field] || 'type'
-        url = "http://#{options[:host] || '127.0.0.1'}:#{options[:port] || 5984}/#{options[:db] || 'moneta'}"
-        @backend = options[:backend] || ::Faraday.new(url: url)
+        @value_field = options.delete(:value_field) || 'value'
+        @type_field = options.delete(:type_field) || 'type'
+        login = options.delete(:login)
+        password = options.delete(:password)
+        @backend = options.delete(:backend) || begin
+          host = options.delete(:host) || '127.0.0.1'
+          port = options.delete(:port) || 5984
+          db = options.delete(:db) || 'moneta'
+          scheme = options.delete(:scheme) || 'http'
+          block = if faraday_adapter = options.delete(:adapter)
+                    proc { |faraday| faraday.adapter(faraday_adapter) }
+                  end
+          ::Faraday.new("#{scheme}://#{host}:#{port}/#{db}", options, &block)
+        end
+        @backend.basic_auth(login, password) if login && password
         @rev_cache = Moneta.build do
           use :Lock
           adapter :LRUHash
@@ -40,68 +69,121 @@ module Moneta
       end
 
       # (see Proxy#key?)
+      # @option options [Boolean] :cache_rev (true) Whether to cache the rev of
+      #   the document for faster updating
       def key?(key, options = {})
-        response = @backend.head(key)
-        update_rev_cache(key, response)
-        response.status == 200
+        cache_rev = options[:cache_rev] != false
+        head(key, cache_rev: cache_rev)
       end
 
       # (see Proxy#load)
+      # @option (see #key?)
       def load(key, options = {})
-        response = @backend.get(key)
-        update_rev_cache(key, response)
-        response.status == 200 ? body_to_value(response.body) : nil
+        cache_rev = options[:cache_rev] != false
+        doc = get(key, cache_rev: cache_rev)
+        doc ? doc_to_value(doc) : nil
       end
 
       # (see Proxy#store)
+      # @option (see #key?)
+      # @option options [Boolean] :batch (false) Whether to do a
+      #   {https://docs.couchdb.org/en/stable/api/database/common.html#api-doc-batch-writes
+      #    batch mode write}
+      # @option options [Boolean] :full_commit (nil) Set to `true` or `false`
+      #   to override the server's
+      #   {https://docs.couchdb.org/en/stable/config/couchdb.html#couchdb/delayed_commits
+      #    commit policy}
       def store(key, value, options = {})
-        body = value_to_body(value, rev(key))
-        response = @backend.put(key, body, 'Content-Type' => 'application/json')
-        update_rev_cache(key, response)
-        raise "HTTP error #{response.status} (PUT /#{key})" unless response.status == 201
+        put(key, value_to_doc(value, rev(key)),
+            headers: full_commit_header(options[:full_commit]),
+            query: options[:batch] ? { batch: 'ok' } : {},
+            cache_rev: options[:cache_rev] != false,
+            expect: options[:batch] ? 202 : 201)
         value
-      rescue
+      rescue HTTPError
         tries ||= 0
         (tries += 1) < 10 ? retry : raise
       end
 
       # (see Proxy#delete)
+      # @option options [Boolean] :batch (false) Whether to do a
+      #   {https://docs.couchdb.org/en/stable/api/database/common.html#api-doc-batch-writes
+      #    batch mode write}
+      # @option options [Boolean] :full_commit (nil) Set to `true` or `false`
+      #   to override the server's
+      #   {https://docs.couchdb.org/en/stable/config/couchdb.html#couchdb/delayed_commits
+      #    commit policy}
       def delete(key, options = {})
-        clear_rev_cache(key)
-        get_response = @backend.get(key)
-        if get_response.status == 200
-          existing_rev = get_response['etag'][1..-2]
+        get_response = get(key, returns: :response)
+        if get_response.success?
           value = body_to_value(get_response.body)
-          delete_response = @backend.delete("#{key}?rev=#{existing_rev}")
-          raise "HTTP error #{response.status}" unless delete_response.status == 200
+          existing_rev = parse_rev(get_response)
+          query = { rev: existing_rev }
+          query[:batch] = 'ok' if options[:batch]
+          request(:delete, key,
+                  headers: full_commit_header(options[:full_commit]),
+                  query: query,
+                  expect: options[:batch] ? 202 : 200)
+          delete_cached_rev(key)
           value
         end
-      rescue
+      rescue HTTPError
         tries ||= 0
         (tries += 1) < 10 ? retry : raise
       end
 
       # (see Proxy#clear)
+      # @option options [Boolean] :compact (false) Whether to compact the database after clearing
+      # @option options [Boolean] :await_compact (false) Whether to wait for compaction to complete
+      #   before returning.
+      # @option options [Boolean] :full_commit (nil) Set to `true` or `false`
+      #   to override the server's
+      #   {https://docs.couchdb.org/en/stable/config/couchdb.html#couchdb/delayed_commits
+      #    commit policy}
       def clear(options = {})
-        @backend.delete ''
-        create_db
+        loop do
+          docs = all_docs(limit: 10_000)
+          break if docs['rows'].empty?
+          deletions = docs['rows'].map do |row|
+            { _id: row['id'], _rev: row['value']['rev'], _deleted: true }
+          end
+          bulk_docs(deletions, full_commit: options[:full_commit])
+        end
+
+        # Compact the database unless told not to
+        if options[:compact]
+          post('_compact', expect: 202)
+
+          # Performance won't be great while compaction is happening, so by default we wait for it
+          if options[:await_compact]
+            loop do
+              db_info = get('', expect: 200)
+              break unless db_info['compact_running']
+
+              # wait before checking again
+              sleep 1
+            end
+          end
+        end
+
         self
       end
 
       # (see Proxy#create)
+      # @option (see #key?)
       def create(key, value, options = {})
-        body = value_to_body(value, nil)
-        response = @backend.put(key, body, 'Content-Type' => 'application/json')
-        update_rev_cache(key, response)
+        cache_rev = options[:cache_rev] != false
+        doc = value_to_doc(value, nil)
+        response = put(key, doc, cache_rev: cache_rev, returns: :response)
         case response.status
         when 201
           true
         when 409
           false
         else
-          raise "HTTP error #{response.status}"
+          raise HTTPError.new(response.status, :put, @backend.create_url(key))
         end
-      rescue
+      rescue HTTPError
         tries ||= 0
         (tries += 1) < 10 ? retry : raise
       end
@@ -112,21 +194,14 @@ module Moneta
 
         skip = 0
         limit = 1000
-        total_rows = 1
-        while total_rows > skip do
-          response = @backend.get("_all_docs?" + encode_query(limit: limit, skip: skip))
-          case response.status
-          when 200
-            result = MultiJson.load(response.body)
-            total_rows = result['total_rows']
-            skip += result['rows'].length
-            result['rows'].each do |row|
-              key = row['id']
-              @rev_cache[key] = row['value']['rev']
-              yield key
-            end
-          else
-            raise "HTTP error #{response.status}"
+        loop do
+          docs = all_docs(limit: limit, skip: skip)
+          break if docs['rows'].empty?
+          skip += docs['rows'].length
+          docs['rows'].each do |row|
+            key = row['id']
+            @rev_cache[key] = row['value']['rev']
+            yield key
           end
         end
         self
@@ -140,16 +215,18 @@ module Moneta
 
       # (see Proxy#slice)
       def slice(*keys, **options)
-        response = @backend.get('_all_docs?' + encode_query(keys: keys, include_docs: true))
-        raise "HTTP error #{response.status}" unless response.status == 200
-        docs = MultiJson.load(response.body)
+        docs = all_docs(keys: keys, include_docs: true)
         docs["rows"].map do |row|
-          next unless row['doc']
-          [row['id'], doc_to_value(row['doc'])]
+          next unless doc = row['doc']
+          [row['id'], doc_to_value(doc)]
         end.compact
       end
 
       # (see Proxy#merge!)
+      # @option options [Boolean] :full_commit (nil) Set to `true` or `false`
+      #   to override the server's
+      #   {https://docs.couchdb.org/en/stable/config/couchdb.html#couchdb/delayed_commits
+      #    commit policy}
       def merge!(pairs, options = {})
         keys = pairs.map { |key, _| key }.to_a
         cache_revs(*keys.reject { |key| @rev_cache[key] })
@@ -169,16 +246,14 @@ module Moneta
         end
 
         docs = pairs.map { |key, value| value_to_doc(value, @rev_cache[key], key) }.to_a
-        body = MultiJson.dump(docs: docs)
-        response = @backend.post('_bulk_docs', body, "Content-Type" => "application/json")
-        raise "HTTP error #{response.status}" unless response.status == 201
-        retries = []
-        MultiJson.load(response.body).each do |row|
-          if row['ok'] == true
-            @rev_cache[row['id']] = row['rev']
+        results = bulk_docs(docs, full_commit: options[:full_commit], returns: :doc)
+        retries = results.each_with_object([]) do |row, retries|
+          ok, id = row.values_at('ok', 'id')
+          if ok
+            @rev_cache[id] = row['rev']
           elsif row['error'] == 'conflict'
-            clear_rev_cache(row['id'])
-            retries << pairs.find { |key, _| key == row['id'] }
+            delete_cached_rev(id)
+            retries << pairs.find { |key,| key == id }
           else
             raise "Unrecognised response: #{row}"
           end
@@ -194,6 +269,10 @@ module Moneta
 
       private
 
+      def full_commit_header(full_commit)
+        full_commit == nil ? {} : { 'X-Couch-Full-Commit' => (!!full_commit).to_s }
+      end
+
       def body_to_value(body)
         doc_to_value(MultiJson.load(body))
       end
@@ -228,61 +307,127 @@ module Moneta
         doc
       end
 
-      def value_to_body(value, rev)
-        MultiJson.dump(value_to_doc(value, rev))
-      end
-
       def create_db
-        100.times do
-          response = @backend.put('', '')
+        loop do
+          response = put('', returns: :response)
           case response.status
           when 201
             break
           when 412
             # Make sure the database really does exist
-            break if @backend.head('').status == 200
+            # See https://github.com/apache/couchdb/issues/2073
+            break if head('')
           else
-            raise "HTTP error #{response.status}"
+            raise HTTPError.new(response.status, :put, '')
           end
 
           # Wait before trying again
           sleep 1
         end
+
+        self
       end
 
       def cache_revs(*keys)
-        response = @backend.get('_all_docs?' + encode_query(keys: keys))
-        raise "HTTP error #{response.status}" unless response.status == 200
-        docs = MultiJson.load(response.body)
+        docs = all_docs(keys: keys)
         docs['rows'].each do |row|
-          next unless row['value']
+          next if !row['value'] || row['value']['deleted']
           @rev_cache[row['id']] = row['value']['rev']
         end
       end
 
-      def update_rev_cache(key, response)
+      def parse_rev(response)
+        response['etag'][1..-2]
+      end
+
+      def cache_response_rev(key, response)
         case response.status
         when 200, 201
-          @rev_cache[key] = response['etag'][1..-2]
+          @rev_cache[key] = parse_rev(response)
         else
-          clear_rev_cache(key)
+          delete_cached_rev(key)
           nil
         end
       end
 
-      def clear_rev_cache(key)
+      def delete_cached_rev(key)
         @rev_cache.delete(key)
       end
 
       def rev(key)
-        @rev_cache[key] || (
-          response = @backend.head(key) and
-          update_rev_cache(key, response)).tap do |rev|
+        @rev_cache[key] || begin
+          response = @backend.head(key)
+          cache_response_rev(key, response)
         end
       end
 
       def encode_query(query)
-        URI.encode_www_form(query.map { |key, value| [key, MultiJson.dump(value)] })
+        query.map { |key, value| [key, MultiJson.dump(value)] }
+      end
+
+      def request(method, key, body = nil, returns: :nil, cache_rev: false, expect: nil, query: nil, headers: {})
+        url = @backend.build_url(key, query)
+        headers['Content-Type'] = 'application/json' if %i{put post}.include?(method)
+        response = @backend.run_request(method, url, body || '', headers)
+
+        if cache_rev
+          cache_response_rev(key, response)
+        end
+
+        if expect
+          raise HTTPError.new(response.status, method, url) unless response.status == expect
+        end
+
+        case returns
+        when :response
+          response
+        when :success
+          response.success?
+        when :doc
+          response.success? ? MultiJson.load(response.body) : nil
+        when :nil
+          nil
+        else
+          raise "Unknown returns param: #{returns.inspect}"
+        end
+      end
+
+      def get(key, returns: :doc, **options)
+        request(:get, key, returns: returns, **options)
+      end
+
+      def head(key, returns: :success, **options)
+        request(:head, key, returns: returns, **options)
+      end
+
+      def put(key, doc = nil, returns: :success, **options)
+        body = doc == nil ? '' : MultiJson.dump(doc)
+        request(:put, key, body, returns: returns, **options)
+      end
+
+      def post(key, doc = nil, returns: :success, **options)
+        body = doc == nil ? '' : MultiJson.dump(doc)
+        request(:post, key, body, returns: returns, **options)
+      end
+
+      def all_docs(sorted: false, **params)
+        keys = params.delete(:keys)
+        query = encode_query(params.merge(sorted: sorted))
+        if keys
+          post('_all_docs', { keys: keys },
+               query: query,
+               expect: 200,
+               returns: :doc)
+        else
+          get('_all_docs', query: query, expect: 200)
+        end
+      end
+
+      def bulk_docs(docs, returns: :success, full_commit: nil)
+        post('_bulk_docs', { docs: docs },
+             headers: full_commit_header(full_commit),
+             returns: returns,
+             expect: 201)
       end
     end
   end