right_support 2.8.15 → 2.8.16

data/VERSION

@@ -1 +1 @@
-2.8.15
+2.8.16

data/lib/right_support/db/cassandra_model.rb

@@ -142,13 +142,20 @@ module RightSupport::DB
   class CassandraModel
     include RightSupport::Log::Mixin
 
+    # For long-running operations, generate period log output after this many columns to help
+    # observers figure out how much progress has been made
+    LONG_OPERATION_LOG_PERIOD = 1_000_000
+
     # Default timeout for client connection to Cassandra server
     DEFAULT_TIMEOUT = 20
 
-    # Default maximum number of rows to retrieve in one chunk
+    # Default maximum number of columns to retrieve in one chunk
     DEFAULT_COUNT   = 100
 
-        # Wrappers for Cassandra client
+    # Default maximum number of rows to retrieve in one chunk
+    DEFAULT_ROW_COUNT = 100
+
+            # Wrappers for Cassandra client
     class << self
 
       attr_reader :default_keyspace
@@ -420,11 +427,11 @@ module RightSupport::DB
       # A row key may be yielded more than once as each "chunk" of columns from that row is
       # read from the ring, but each column will be yielded exactly once.
       #
-      # == Parameters:
+      # @return [Integer] total number of columns read
+      #
       # @param [String] index column name
       # @param [String] index column value
       #
-      # == Yields:
       # @yield [row_key, columns] yields one or more times for every row that contains a matching index column, ultimately yielding EVERY column in that row
       # @yieldparam [String] row_key the row key currently being processes
       # @yieldparam [Array] columns an array of Cassandra CassandraThrift::ColumnOrSuperColumn objects
@@ -432,12 +439,18 @@ module RightSupport::DB
         expr      = do_op(:create_idx_expr, index, key, "EQ")
         start_row = ''
 
+        last_report   = 0
+        total_rows    = 0
+        total_columns = 0
+
         # Loop over all CF rows, with batches of X
         while (start_row != nil)
           # Reset these to their initial values on every iteration thru the loop, in case
           # we backed off due to timeouts (see rescue clauses below)
-          row_count    = 100  # how many rows to grab at once
-          column_count = 1000 # how much to grab in each chunk of a long row
+          #
+          # Since this method is made for wide rows, also ask for 10x the default column count.
+          row_count    = DEFAULT_ROW_COUNT
+          column_count = DEFAULT_COUNT * 10
 
           clause = do_op(:create_idx_clause, [expr], start_row, row_count)
 
@@ -460,6 +473,7 @@ module RightSupport::DB
             # We already processed this row the previous iteration; skip it
             next if row_key == start_row
 
+            total_rows += 1
             start_column = ''
 
             while start_column != nil
@@ -475,11 +489,20 @@ module RightSupport::DB
                 columns = self.conn.get_range(column_family, options).first.columns
 
                 if columns[0].column.name == start_column
+                  total_columns += columns.size - 1
                   yield(row_key, columns[1..-1])
                 else
+                  total_columns += columns.size
                   yield(row_key, columns)
                 end
 
+                # Help Ops figure out where we are
+                if (total_columns - last_report > LONG_OPERATION_LOG_PERIOD)
+                  logger.info "CassandraModel#stream_all_indexed_slices got RangeSlice total_rows=%d total_columns=%d (cf='%s' row='%s' start='%s' count=%d)" %
+                                [total_rows, total_columns, column_family, row_key, start_column, column_count]
+                  last_report = total_columns
+                end
+
                 if columns.size >= column_count
                   start_column = columns.last.column.name
                 else
@@ -488,7 +511,7 @@ module RightSupport::DB
               rescue Exception => e
                 if retryable_read_timeout?(e)
                   logger.error "CassandraModel#stream_all_indexed_slices retrying get_range with fewer cols due to a %s: %s @ %s (cf='%s' row='%s' start='%s' count=%d)" %
-                                 [e.class.name, e.message, e.backtrace.first, column_family, row_key, last_column_name, column_count]
+                                 [e.class.name, e.message, e.backtrace.first, column_family, row_key, start_column, column_count]
                   column_count /= 10 if column_count > 1
                   retry
                 else
@@ -504,6 +527,8 @@ module RightSupport::DB
             start_row = nil
           end
         end
+
+        total_columns
       end
 
       # Get all rows for specified secondary key

data/lib/right_support/net/http_client.rb

@@ -124,11 +124,14 @@ module RightSupport::Net
   class HTTPClient
     # The default options for every request; can be overridden by options
     # passed to #initialize or to the individual request methods (#get,
-    # #post, and so forth).
+    # #post, and so forth). 
+    # NOTE: rest-client 1.6.7 doesn't pass down the ssl_version to Net::HTTP.  
+    # silently discarded.  1.7.0.alpha or later does.
     DEFAULT_OPTIONS = {
       :timeout      => 5,
       :open_timeout => 2,
-      :headers      => {}
+      :headers      => {},
+      :ssl_version  => 'TLSv1'
     }
 
     def initialize(defaults = {})

data/right_support.gemspec

@@ -4,14 +4,14 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = "right_support"
-  s.version = "2.8.15"
+  s.name = %q{right_support}
+  s.version = "2.8.16"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tony Spataro", "Sergey Sergyenko", "Ryan Williamson", "Lee Kirchhoff", "Alexey Karpik", "Scott Messier"]
-  s.date = "2014-02-26"
-  s.description = "A toolkit of useful, reusable foundation code created by RightScale."
-  s.email = "support@rightscale.com"
+  s.date = %q{2014-02-27}
+  s.description = %q{A toolkit of useful, reusable foundation code created by RightScale.}
+  s.email = %q{support@rightscale.com}
   s.extra_rdoc_files = [
     "LICENSE",
     "README.rdoc"
@@ -134,11 +134,11 @@ Gem::Specification.new do |s|
     "spec/validation/openssl_spec.rb",
     "spec/validation/ssh_spec.rb"
   ]
-  s.homepage = "https://github.com/rightscale/right_support"
+  s.homepage = %q{https://github.com/rightscale/right_support}
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.28"
-  s.summary = "Reusable foundation code."
+  s.rubygems_version = %q{1.6.2}
+  s.summary = %q{Reusable foundation code.}
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/spec/db/cassandra_model_spec.rb

@@ -270,27 +270,6 @@ describe RightSupport::DB::CassandraModel do
         end
       end
 
-      def real_get_indexed(index, key, columns = nil, opt = {})
-        rows = {}
-        start = ""
-        count = DEFAULT_COUNT
-        expr = do_op(:create_idx_expr, index, key, "EQ")
-        opt = opt[:consistency] ? {:consistency => opt[:consistency]} : {}
-        while true
-          clause = do_op(:create_idx_clause, [expr], start, count)
-          chunk = do_op(:get_indexed_slices, column_family, clause, columns, opt)
-          rows.merge!(chunk)
-          if chunk.size == count
-            # Assume there are more chunks, use last key as start of next get
-            start = chunk.keys.last
-          else
-            # This must be the last chunk
-            break
-          end
-        end
-        rows
-      end
-
       context :get_indexed do
 
         before(:each) do
@@ -347,6 +326,12 @@ describe RightSupport::DB::CassandraModel do
           RightSupport::DB::CassandraModel.get_indexed(@index, @index_key).should == []
         end
       end
+
+      context :stream_all_indexed_slices do
+        it 'streams rows in chunks'
+        it 'streams columns in chunks'
+        it 'handles timeouts'
+      end
     end
   end
 

data/spec/net/http_client_spec.rb

@@ -97,6 +97,7 @@ describe RightSupport::Net::HTTPClient do
       it 'uses default options on every request' do
         p = {:method=>:get,
              :timeout=>101010,
+             :ssl_version=>'TLSv1',
              :open_timeout=>999,
              :url=>'/moo', :headers=>{:moo=>:bah}}
         flexmock(RestClient::Request).should_receive(:execute).with(p)
@@ -106,10 +107,11 @@ describe RightSupport::Net::HTTPClient do
       it 'allows defaults to be overridden' do
         p = {:method=>:get,
              :timeout=>101010,
+             :ssl_version=>'SSLv3',
              :open_timeout=>3,
              :url=>'/moo', :headers=>{:joe=>:blow}}
         flexmock(RestClient::Request).should_receive(:execute).with(p)
-        @http_client.get('/moo', :open_timeout=>3, :headers=>{:joe=>:blow})
+        @http_client.get('/moo', :ssl_version => 'SSLv3', :open_timeout=>3, :headers=>{:joe=>:blow})
       end
     end
   end
@@ -125,6 +127,7 @@ describe RightSupport::Net::HTTPClient do
       it 'succeeds' do
         p = {:method=>:get,
              :timeout=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:timeout],
+             :ssl_version=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:ssl_version],
              :open_timeout=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:open_timeout],
              :url=>'/moo', :headers=>{}}
         flexmock(RestClient::Request).should_receive(:execute).with(p)
@@ -137,6 +140,7 @@ describe RightSupport::Net::HTTPClient do
       it 'succeeds' do
         p = {:method=>:get,
              :timeout=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:timeout],
+             :ssl_version=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:ssl_version],
              :open_timeout=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:open_timeout],
              :url=>'/moo', :headers=>{:mrm=>1, :blah=>:foo}}
         flexmock(RestClient::Request).should_receive(:execute).with(p)
@@ -150,6 +154,7 @@ describe RightSupport::Net::HTTPClient do
       it 'succeeds' do
         p = {:method=>:get,
              :timeout=>42,
+             :ssl_version=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:ssl_version],
              :open_timeout => RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:open_timeout],
              :url=>'/moo', :headers=>{}}
         flexmock(RestClient::Request).should_receive(:execute).with(p)
@@ -160,8 +165,13 @@ describe RightSupport::Net::HTTPClient do
     
     context 'given a URL and any other parameters' do
       it 'succeeds' do
-        p = { :method=>:get, :timeout=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:timeout],
-              :url=>'/moo', :headers=>{},:open_timeout => 1, :payload=>{:foo => :bar} }
+        p = { :method=>:get,
+              :timeout=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:timeout],
+              :ssl_version=>RightSupport::Net::HTTPClient::DEFAULT_OPTIONS[:ssl_version],
+              :url=>'/moo',
+              :headers=>{},
+              :open_timeout => 1,
+              :payload=>{:foo => :bar} }
         flexmock(RestClient::Request).should_receive(:execute).with(p)
 
         @http_client.get('/moo', :open_timeout => 1, :payload=>{:foo => :bar})

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: right_support
 version: !ruby/object:Gem::Version 
-  hash: 49
+  hash: 15
   prerelease: 
   segments: 
   - 2
   - 8
-  - 15
-  version: 2.8.15
+  - 16
+  version: 2.8.16
 platform: ruby
 authors: 
 - Tony Spataro