right_support 2.8.8 → 2.8.9

data/VERSION

@@ -1 +1 @@
-2.8.8
+2.8.9

data/lib/right_support/db/cassandra_model.rb

@@ -405,27 +405,30 @@ module RightSupport::DB
         rows
       end
 
-      # This method is an attempt to circumvent the Cassandra gem limitation of returning only 100 columns for wide rows
-      # This method returns only columns that are within the result set specified by a secondary index equality query
-      # This method will iterate through chunks of rows of the resultset and it will yield to the caller all of the
-      # columns in chunks of 1,000 until all of the columns have been retrieved
+      # This method is an attempt to circumvent the Cassandra gem limitation of returning only 100 columns for wide rows,
+      # and also to help reliably iterate through a column family when the node is busy and experiencing many timeouts.
+      #
+      # Internally, it uses Cassandra#get_indexed_slices to find rows that match your index constraint; when it finds
+      # a wide row (with more than 1,000 columns), it continues to iterate through the columns of that row
       #
       # == Parameters:
       # @param [String] index column name
       # @param [String] index column value
       #
       # == Yields:
-      # @yield [Array<String, Array<CassandraThrift::ColumnOrSuperColumn>>] irray containing ndex column value passed in and an array of columns matching the index query
+      # @yield [Array<String, Array<CassandraThrift::ColumnOrSuperColumn>>] array containing index column value passed in and an array of columns matching the index query
       def stream_all_indexed_slices(index, key)
         expr = do_op(:create_idx_expr, index, key, "EQ")
-
         start_row		    = ''
-        max_row_count		    = 100
-        max_initial_column_count    = 1000 # number of columns to retrieve in the initial 2ndary index search
-        max_additional_column_count = 10000 # Number of columns to retrieve in a batch once we're targetting a single (long) row
 
         # Loop over all CF rows, with batches of X
         while (start_row != nil)
+          # Reset these to their initieral values on every iteration thru the loop, in case
+          # we backed off due to timeouts (see rescue clauses below)
+          max_row_count               = 100  # how many rows to grab at once
+          max_initial_column_count    = 1000 # how much to grab at first for each row
+          max_additional_column_count = 1000 # how much to grab in each chunk of a long row
+
           clause = do_op(:create_idx_clause, [expr], start_row, max_row_count)
 
           # Now, for each batch of rows, make sure don't ask for "ALL" columns of each row, to avoid hitting rows with a huge amount of columns, 
@@ -433,16 +436,14 @@ module RightSupport::DB
           begin
             rows = self.conn.get_indexed_slices(column_family, clause, :count => max_initial_column_count)
           rescue Exception => e
-            wrapped_timeout      = e.is_a?(CassandraThrift::TimedOutException)
-            unwrapped_timeout    = e.is_a?(Thrift::TransportException) && (e.type == Thrift::TransportException::TIMED_OUT)
-            unwrapped_disconnect = e.is_a?(Thrift::TransportException) && (e.type == Thrift::TransportException::NOT_OPEN)
-
-            if (wrapped_timeout || unwrapped_timeout || unwrapped_disconnect) && (timeout_retries < retry_timeout)
-              timeout_retries += 1
+            if retryable_read_timeout?(e)
+              logger.error "CassandraModel#stream_all_indexed_slices retrying get_indexed_slices with fewer rows/cols due to a %s: %s @ %s (cf='%s' start_row='%s' count=%d)" %
+              [e.class.name, e.message, e.backtrace.first, column_family, start_row, max_row_count]
+              max_row_count /= 10 if max_row_count > 1
+              max_initial_column_count /= 10 if max_initial_column_count > 1
               retry
             else
-              timeout_retries = 0
-              raise e
+              raise
             end
           end
 
@@ -467,16 +468,13 @@ module RightSupport::DB
                     :start => last_column_name,
                     :slices_not_hash => true ).first.columns[1..-1]
                 rescue Exception => e
-                  wrapped_timeout      = e.is_a?(CassandraThrift::TimedOutException)
-                  unwrapped_timeout    = e.is_a?(Thrift::TransportException) && (e.type == Thrift::TransportException::TIMED_OUT)
-                  unwrapped_disconnect = e.is_a?(Thrift::TransportException) && (e.type == Thrift::TransportException::NOT_OPEN)
-
-                  if (wrapped_timeout || unwrapped_timeout || unwrapped_disconnect) && (timeout_retries < retry_timeout)
-                    timeout_retries += 1
+                  if retryable_read_timeout?(e)
+                    logger.error "CassandraModel#stream_all_indexed_slices retrying get_range with fewer rows/cols due to a %s: %s @ %s (cf='%s' row='%s' start='%s' count=%d)" %
+                                   [e.class.name, e.message, e.backtrace.first, column_family, row_key, last_column_name, max_additional_column_count]
+                    max_additional_column_count /= 10 if max_additional_column_count > 1
                     retry
                   else
-                    timeout_retries = 0
-                    raise e
+                    raise
                   end
                 end
 
@@ -693,6 +691,29 @@ module RightSupport::DB
         end
       end
 
+      # Determine whether an exception can safely be retried, if the operation being performed
+      # was a read (i.e. is idempotent).
+      # @param [Exception] e the exception in question
+      # @return [Boolean] true if yes, false otherwise
+      def retryable_read_timeout?(e)
+        # Thrift RPC timeout that has been "improved" by cassandra gem's moronic exception-wrapping
+        wrapped_timeout      = e.is_a?(CassandraThrift::TimedOutException)
+
+        # I have no idea what these are, but they show up when we have a local socket-read timeout after 10s
+        # Probably due to this line from the thrift gem (in lib/thrift/transport/socket.rb)
+        #   raise TransportException.new(TransportException::TIMED_OUT, "Socket: Timed out reading #{sz} bytes from #{@desc}")
+        # The TransportException gets wrapped into a CassandraThrift::Cassandra::Client::TransportException and in the process
+        # the type is lost and the message is promoted to "type"
+        bogus_timeout        = e.is_a?(Thrift::TransportException) && e.type.is_a?(String) && e.type =~ /Timed out/i
+
+        # Thrift RPC timeout
+        unwrapped_timeout    = e.is_a?(Thrift::TransportException) && (e.type == Thrift::TransportException::TIMED_OUT)
+
+        # Thrift socket disconnect
+        unwrapped_disconnect = e.is_a?(Thrift::TransportException) && (e.type == Thrift::TransportException::NOT_OPEN)
+
+        wrapped_timeout || bogus_timeout || unwrapped_timeout || unwrapped_disconnect
+      end
     end # self
 
     attr_accessor :key, :attributes

data/right_support.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "right_support"
-  s.version = "2.8.8"
+  s.version = "2.8.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tony Spataro", "Sergey Sergyenko", "Ryan Williamson", "Lee Kirchhoff", "Alexey Karpik", "Scott Messier"]
-  s.date = "2014-01-09"
+  s.date = "2014-01-13"
   s.description = "A toolkit of useful, reusable foundation code created by RightScale."
   s.email = "support@rightscale.com"
   s.extra_rdoc_files = [

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: right_support
 version: !ruby/object:Gem::Version 
-  hash: 63
+  hash: 61
   prerelease: 
   segments: 
   - 2
   - 8
-  - 8
-  version: 2.8.8
+  - 9
+  version: 2.8.9
 platform: ruby
 authors: 
 - Tony Spataro
@@ -20,7 +20,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-01-09 00:00:00 Z
+date: 2014-01-13 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement