moped 2.0.3 → 2.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c42ef9279d08c76ecf69539b067b93ab0878f4e
-  data.tar.gz: 3a12fc6c37618f64b7554483f186bac840df90bb
+  metadata.gz: eda09d3428a61f892b3d983486b0bcacb4d22c1f
+  data.tar.gz: 0d496810ace691461b3443f1163102a76f85ec5b
 SHA512:
-  metadata.gz: a636e60201e811f5ea5334ae6e2d3418e2c3c483ae992183172abadadf5ffa0b1682acbac4e03684e4d2dede020d3f562e3a512dbfe6395b19615746fc121707
-  data.tar.gz: b733ee7605f748665b060621b7a65bc1ee2ca610f6079ca96c76563329b9140f25468c618e96986abda5be9e963aa52881c6f3982f98d99c5245ae415170c428
+  metadata.gz: 951ba327262a778af25d190b5f80332c10bac5f8186194c399091bb924bcab9c625f4b317d3807390a2fc233f3c2f2a7cfb1f499045363743ec2e6a789581043
+  data.tar.gz: 8a5fcdd582261c79a7d1fd241d17ffd437bd4232e3a663e040c316fa8c2bc29e15c56399fa2e677e8971c1687cbbafd8743ba4fc6af6b1c97eb888ca1a4d6c56

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Overview
 
+## 2.0.7
+
+* Improved Moped Failover (John Hyman)
+
+## 2.0.6
+
+* Relaxing BSON dependency to allow for 3.1.0 upgrade.
+
+## 2.0.5
+
+### Resolved Issues
+
+* \#351
+  Fixing retries with authentication. (Wandenberg Peixoto)
+
+* Bump BSON dependency to 3.0.4
+
+## 2.0.4
+
+### Resolved Issues
+
+* \#345
+  Writes fail with ConnectionPool::PoolShuttingDownError. (fedenusy, dblock)
+
 ## 2.0.3
 
 ### Resolved Issues

data/lib/moped/address.rb

@@ -45,9 +45,13 @@ module Moped
     #
     # @since 2.0.0
     def resolve(node)
+      return @resolved if @resolved
+      start = Time.now
+      retries = 0
       begin
-        return @resolved if @resolved
-        Timeout::timeout(@timeout) do
+        # This timeout should be very large since Timeout::timeout plays very badly with multithreaded code
+        # TODO: Remove this Timeout entirely
+        Timeout::timeout(@timeout * 10) do
           Resolv.each_address(host) do |ip|
             if ip =~ Resolv::IPv4::Regex
               @ip ||= ip
@@ -57,9 +61,19 @@ module Moped
           raise Resolv::ResolvError unless @ip
         end
         @resolved = "#{ip}:#{port}"
-      rescue Timeout::Error, Resolv::ResolvError, SocketError
-        Loggable.warn("  MOPED:", "Could not resolve IP for: #{original}", "n/a")
-        node.down! and false
+      rescue Timeout::Error, Resolv::ResolvError, SocketError => e
+        msg = ["  MOPED:", "Could not resolve IP for: #{original}, delta is #{Time.now - start}, error class is #{e.inspect}, retries is #{retries}. Node is #{node.inspect}", "n/a"]
+        if retries == 0
+          Loggable.info(*msg)
+        else
+          Loggable.warn(*msg)
+        end
+        if retries < 2
+          retries += 1
+          retry
+        else
+          node.down! and false
+        end
       end
     end
   end

data/lib/moped/collection.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "moped/query"
+require "moped/retryable"
 
 module Moped
 
@@ -8,6 +9,7 @@ module Moped
   # @since 1.0.0
   class Collection
     include Readable
+    include Retryable
 
     # @!attribute database
     #   @return [ Database ] The database for the collection.
@@ -120,9 +122,11 @@ module Moped
     #
     # @since 1.0.0
     def insert(documents, flags = nil)
-      docs = documents.is_a?(Array) ? documents : [ documents ]
-      cluster.with_primary do |node|
-        node.insert(database.name, name, docs, write_concern, flags: flags || [])
+      with_retry(cluster) do
+        docs = documents.is_a?(Array) ? documents : [ documents ]
+        cluster.with_primary do |node|
+          node.insert(database.name, name, docs, write_concern, flags: flags || [])
+        end
       end
     end
 

data/lib/moped/connection/manager.rb

@@ -48,10 +48,17 @@ module Moped
       #
       # @since 2.0.3
       def shutdown(node)
+        pool = nil
         MUTEX.synchronize do
           pool = pools.delete(node.address.resolved)
-          pool.shutdown{ |conn| conn.disconnect } if pool
-          nil
+        end
+        pool.shutdown{ |conn| conn.disconnect } if pool
+        nil
+      end
+
+      def delete_pool(node)
+        MUTEX.synchronize do
+          pools.delete(node.address.resolved)
         end
       end
 

data/lib/moped/connection.rb

@@ -3,7 +3,6 @@ require "moped/connection/manager"
 require "moped/connection/sockets"
 
 module Moped
-
   # This class contains behaviour of database socket connections.
   #
   # @since 2.0.0
@@ -15,6 +14,8 @@ module Moped
     # @since 2.0.0
     TIMEOUT = 5
 
+    REPLY_DECODE_STR = 'l<5q<l<2'
+
     # @!attribute host
     #   @return [ String ] The ip address of the host.
     # @!attribute options
@@ -114,15 +115,15 @@ module Moped
       with_connection do |socket|
         reply = Protocol::Reply.allocate
         data = read_data(socket, 36)
-        response = data.unpack('l<5q<l<2')
+        response = data.unpack(REPLY_DECODE_STR)
         reply.length,
-            reply.request_id,
-            reply.response_to,
-            reply.op_code,
-            reply.flags,
-            reply.cursor_id,
-            reply.offset,
-            reply.count = response
+          reply.request_id,
+          reply.response_to,
+          reply.op_code,
+          reply.flags,
+          reply.cursor_id,
+          reply.offset,
+          reply.count = response
 
         if reply.count == 0
           reply.documents = []

data/lib/moped/cursor.rb

@@ -6,6 +6,7 @@ module Moped
   class Cursor
     include Readable
     include Enumerable
+    include Retryable
 
     # @attribute [r] get_more_op The get more message.
     # @attribute [r] kill_cursor_op The kill cursor message.
@@ -43,10 +44,12 @@ module Moped
     #
     # @since 1.0.0
     def get_more
-      reply = @node.get_more @database, @collection, @cursor_id, request_limit
-      @limit -= reply.count if limited?
-      @cursor_id = reply.cursor_id
-      reply.documents
+      with_retry(session.cluster) do
+        reply = @node.get_more @database, @collection, @cursor_id, request_limit
+        @limit -= reply.count if limited?
+        @cursor_id = reply.cursor_id
+        reply.documents
+      end
     end
 
     # Determine the request limit for the query

data/lib/moped/errors.rb

@@ -112,10 +112,10 @@ module Moped
     class PotentialReconfiguration < MongoError
 
       # Not master error codes.
-      NOT_MASTER = [ 13435, 13436, 10009 ]
+      NOT_MASTER = [ 13435, 13436, 10009, 15986, 83 ]
 
       # Error codes received around reconfiguration
-      CONNECTION_ERRORS_RECONFIGURATION = [ 15988, 10276, 11600, 9001, 13639, 10009, 11002 ]
+      CONNECTION_ERRORS_RECONFIGURATION = [ 15988, 10276, 11600, 9001, 13639, 10009, 11002, 7 ]
 
       # Replica set reconfigurations can be either in the form of an operation
       # error with code 13435, or with an error message stating the server is
@@ -126,7 +126,8 @@ module Moped
       end
 
       def connection_failure?
-        CONNECTION_ERRORS_RECONFIGURATION.include?(details["code"])
+        err = details["err"] || details["errmsg"] || details["$err"] || ""
+        CONNECTION_ERRORS_RECONFIGURATION.include?(details["code"]) || err.include?("could not get last error") || err.include?("connection attempt failed")
       end
 
       # Is the error due to a namespace not being found?

data/lib/moped/failover/retry.rb

@@ -9,7 +9,7 @@ module Moped
     module Retry
       extend self
 
-      # Executes the failover strategy. In the case of retyr, we disconnect and
+      # Executes the failover strategy. In the case of retry, we disconnect and
       # reconnect, then try the operation one more time.
       #
       # @example Execute the retry strategy.
@@ -24,11 +24,13 @@ module Moped
       #
       # @since 2.0.0
       def execute(exception, node)
-        node.disconnect
+        node.disconnect unless exception.is_a?(Errors::PoolTimeout)
         begin
           node.connection do |conn|
             yield(conn) if block_given?
           end
+        rescue Errors::PoolTimeout => e
+          raise Errors::ConnectionFailure.new e
         rescue Exception => e
           node.down!
           raise(e)

data/lib/moped/failover.rb

@@ -21,7 +21,8 @@ module Moped
       Errors::ConnectionFailure => Retry,
       Errors::CursorNotFound => Ignore,
       Errors::OperationFailure => Reconfigure,
-      Errors::QueryFailure => Reconfigure
+      Errors::QueryFailure => Reconfigure,
+      Errors::PoolTimeout => Retry
     }.freeze
 
     # Get the appropriate failover handler given the provided exception.

data/lib/moped/loggable.rb

@@ -42,6 +42,10 @@ module Moped
       Moped.logger.debug([ prefix, payload, "runtime: #{runtime}" ].join(' '))
     end
 
+    def self.info(prefix, payload, runtime)
+      Moped.logger.info([ prefix, payload, "runtime: #{runtime}" ].join(' '))
+    end
+
     # Log the payload to warn.
     #
     # @example Log to warn.

data/lib/moped/node.rb

@@ -111,8 +111,19 @@ module Moped
     #
     # @since 2.0.0
     def connection
-      pool.with do |conn|
-        yield(conn)
+      connection_acquired = false
+      begin
+        pool.with do |conn|
+          connection_acquired = true
+          yield(conn)
+        end
+      rescue Timeout::Error, ConnectionPool::PoolShuttingDownError => e
+        if e.kind_of?(ConnectionPool::PoolShuttingDownError)
+          @pool = nil
+          Connection::Manager.delete_pool(self)
+          raise Errors::PoolTimeout.new(e)
+        end
+        raise connection_acquired ? e : Errors::PoolTimeout.new(e)
       end
     end
 
@@ -150,10 +161,10 @@ module Moped
     #
     # @since 2.0.0
     def down!
-      @pool = Connection::Manager.shutdown(self)
       @down_at = Time.new
+      @pool = nil
       @latency = nil
-      true
+      Connection::Manager.shutdown(self)
     end
 
     # Yields the block if a connection can be established, retrying when a
@@ -182,6 +193,10 @@ module Moped
           yield(conn)
         end
       rescue Exception => e
+        if e.kind_of?(ConnectionPool::PoolShuttingDownError)
+          @pool = nil
+          Connection::Manager.delete_pool(self)
+        end
         Failover.get(e).execute(e, self, &block)
       ensure
         end_execution(:connection)
@@ -198,7 +213,7 @@ module Moped
     #
     # @return [ nil ] nil.
     #
-    # @since 1.0.0
+    # @since 1.0.0s
     def ensure_primary
       execute(:ensure_primary) do
         yield(self)
@@ -585,14 +600,14 @@ module Moped
     def flush(ops = queue)
       operations, callbacks = ops.transpose
       logging(operations) do
-        replies = nil
         ensure_connected do |conn|
           conn.write(operations)
           replies = conn.receive_replies(operations)
+
+          replies.zip(callbacks).map do |reply, callback|
+            callback ? callback[reply] : reply
+          end.last
         end
-        replies.zip(callbacks).map do |reply, callback|
-          callback ? callback[reply] : reply
-        end.last
       end
     ensure
       ops.clear

data/lib/moped/operation/read.rb

@@ -46,10 +46,27 @@ module Moped
       # @since 2.0.0
       def execute(node)
         node.process(operation) do |reply|
-          if operation.failure?(reply)
-            raise operation.failure_exception(reply)
+          # Avoid LocalJumpError
+          ret = nil
+          if reply.unauthorized? && node.credentials.key?(@database)
+            node.connection do |conn|
+              username, password = node.credentials[@database]
+              if username && password
+                conn.login(operation.database, username, password)
+                ret = execute(node)
+              end
+            end
           end
-          operation.results(reply)
+
+          if ret.nil?
+            if operation.failure?(reply)
+              raise operation.failure_exception(reply)
+            end
+
+            ret = operation.results(reply)
+          end
+
+          ret
         end
       end
     end

data/lib/moped/protocol/message.rb

@@ -1,6 +1,5 @@
 module Moped
   module Protocol
-
     # The base class for building all messages needed to implement the Mongo
     # Wire Protocol. It provides a minimal DSL for defining typed fields for
     # serialization and deserialization over the wire.
@@ -34,6 +33,9 @@ module Moped
     #   int32 :op_code
     #
     module Message
+      INT32_DECODE_STR       = 'l<'
+      INT64_DECODE_ARRAY_STR = 'q<*'
+      INT64_DECODE_STR       = 'q<'
 
       # Default implementation for a message is to do nothing when receiving
       # replies.
@@ -214,11 +216,11 @@ module Moped
             end
 
             def serialize_#{name}(buffer)
-              buffer << [#{name}_as_int].pack('l<')
+              buffer << [#{name}_as_int].pack(INT32_DECODE_STR)
             end
 
             def deserialize_#{name}(buffer)
-              bits, = buffer.read(4).unpack('l<')
+              bits, = buffer.read(4).unpack(INT32_DECODE_STR)
 
               self.#{name} = bits
             end
@@ -244,11 +246,11 @@ module Moped
             end
 
             def serialize_#{name}(buffer)
-              buffer << [#{name}].pack('l<')
+              buffer << [#{name}].pack(INT32_DECODE_STR)
             end
 
             def deserialize_#{name}(buffer)
-              self.#{name}, = buffer.read(4).unpack('l<')
+              self.#{name}, = buffer.read(4).unpack(INT32_DECODE_STR)
             end
           RUBY
 
@@ -280,7 +282,7 @@ module Moped
               end
 
               def serialize_#{name}(buffer)
-                buffer << #{name}.pack('q<*')
+                buffer << #{name}.pack(INT64_DECODE_ARRAY_STR)
               end
 
               def deserialize_#{name}(buffer)
@@ -294,11 +296,11 @@ module Moped
               end
 
               def serialize_#{name}(buffer)
-                buffer << [#{name}].pack('q<')
+                buffer << [#{name}].pack(INT64_DECODE_STR)
               end
 
               def deserialize_#{name}(buffer)
-                self.#{name}, = buffer.read(8).unpack('q<')
+                self.#{name}, = buffer.read(8).unpack(INT64_DECODE_STR)
               end
             RUBY
           end

data/lib/moped/query.rb

@@ -21,6 +21,7 @@ module Moped
   #   people.find.count # => 1
   class Query
     include Enumerable
+    include Retryable
 
     # @attribute [r] collection The collection to execute the query on.
     # @attribute [r] operation The query operation.
@@ -321,14 +322,16 @@ module Moped
     #
     # @since 1.0.0
     def remove
-      cluster.with_primary do |node|
-        node.remove(
-          operation.database,
-          operation.collection,
-          operation.basic_selector,
-          write_concern,
-          flags: [ :remove_first ]
-        )
+      with_retry(cluster) do
+        cluster.with_primary do |node|
+          node.remove(
+            operation.database,
+            operation.collection,
+            operation.basic_selector,
+            write_concern,
+            flags: [ :remove_first ]
+          )
+        end
       end
     end
 
@@ -341,13 +344,15 @@ module Moped
     #
     # @since 1.0.0
     def remove_all
-      cluster.with_primary do |node|
-        node.remove(
-          operation.database,
-          operation.collection,
-          operation.basic_selector,
-          write_concern
-        )
+      with_retry(cluster) do
+        cluster.with_primary do |node|
+          node.remove(
+            operation.database,
+            operation.collection,
+            operation.basic_selector,
+            write_concern
+          )
+        end
       end
     end
 
@@ -423,15 +428,17 @@ module Moped
     #
     # @since 1.0.0
     def update(change, flags = nil)
-      cluster.with_primary do |node|
-        node.update(
-          operation.database,
-          operation.collection,
-          operation.selector["$query"] || operation.selector,
-          change,
-          write_concern,
-          flags: flags
-        )
+      with_retry(cluster) do
+        cluster.with_primary do |node|
+          node.update(
+            operation.database,
+            operation.collection,
+            operation.selector["$query"] || operation.selector,
+            change,
+            write_concern,
+            flags: flags
+          )
+        end
       end
     end
 

data/lib/moped/read_preference/selectable.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "moped/retryable"
 module Moped
   module ReadPreference
 
@@ -7,6 +8,7 @@ module Moped
     #
     # @since 2.0.0
     module Selectable
+      include Retryable
 
       # @!attribute tags
       #   @return [ Array<Hash> ] The tag sets.
@@ -39,41 +41,6 @@ module Moped
         options[:flags] |= [ :slave_ok ]
         options
       end
-
-      private
-
-      # Execute the provided block on the cluster and retry if the execution
-      # fails.
-      #
-      # @api private
-      #
-      # @example Execute with retry.
-      #   preference.with_retry(cluster) do
-      #     cluster.with_primary do |node|
-      #       node.refresh
-      #     end
-      #   end
-      #
-      # @param [ Cluster ] cluster The cluster.
-      # @param [ Integer ] retries The number of times to retry.
-      #
-      # @return [ Object ] The result of the block.
-      #
-      # @since 2.0.0
-      def with_retry(cluster, retries = cluster.max_retries, &block)
-        begin
-          block.call
-        rescue Errors::ConnectionFailure => e
-          if retries > 0
-            Loggable.warn("  MOPED:", "Retrying connection attempt #{retries} more time(s).", "n/a")
-            sleep(cluster.retry_interval)
-            cluster.refresh
-            with_retry(cluster, retries - 1, &block)
-          else
-            raise e
-          end
-        end
-      end
     end
   end
 end

data/lib/moped/retryable.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+module Moped
+  # Provides the shared behaviour for retry failed operations.
+  #
+  # @since 2.0.0
+  module Retryable
+
+    private
+
+    # Execute the provided block on the cluster and retry if the execution
+    # fails.
+    #
+    # @api private
+    #
+    # @example Execute with retry.
+    #   preference.with_retry(cluster) do
+    #     cluster.with_primary do |node|
+    #       node.refresh
+    #     end
+    #   end
+    #
+    # @param [ Cluster ] cluster The cluster.
+    # @param [ Integer ] retries The number of times to retry.
+    #
+    # @return [ Object ] The result of the block.
+    #
+    # @since 2.0.0
+    def with_retry(cluster, retries = cluster.max_retries, &block)
+      begin
+        block.call
+      rescue Errors::ConnectionFailure, Errors::PotentialReconfiguration => e
+        raise e if e.is_a?(Errors::PotentialReconfiguration) &&
+          ! (e.message.include?("not master") || e.message.include?("Not primary"))
+
+        if retries > 0
+          Loggable.warn("  MOPED:", "Retrying connection attempt #{retries} more time(s), nodes is #{cluster.nodes.inspect}, seeds are #{cluster.seeds.inspect}, cluster is #{cluster.inspect}. Error backtrace is #{e.backtrace}.", "n/a")
+          sleep(cluster.retry_interval)
+          cluster.refresh
+          with_retry(cluster, retries - 1, &block)
+        else
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/moped/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Moped
-  VERSION = "2.0.3"
+  VERSION = "2.0.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: moped
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.7
 platform: ruby
 authors:
 - Durran Jordan
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-02 00:00:00.000000000 Z
+date: 2015-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: connection_pool
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +114,7 @@ files:
 - lib/moped/read_preference/secondary_preferred.rb
 - lib/moped/read_preference/selectable.rb
 - lib/moped/readable.rb
+- lib/moped/retryable.rb
 - lib/moped/session.rb
 - lib/moped/uri.rb
 - lib/moped/version.rb