analytics-ruby 2.2.3.pre → 2.2.4.pre

data/lib/segment/analytics/defaults.rb

@@ -6,15 +6,31 @@ module Segment
         PORT = 443
         PATH = '/v1/import'
         SSL = true
-        HEADERS = { :accept => 'application/json' }
-        RETRIES = 4
-        BACKOFF = 30.0
+        HEADERS = { 'Accept' => 'application/json',
+                    'Content-Type' => 'application/json',
+                    'User-Agent' => "analytics-ruby/#{Analytics::VERSION}" }
+        RETRIES = 10
       end
 
       module Queue
-        BATCH_SIZE = 100
         MAX_SIZE = 10000
       end
+
+      module Message
+        MAX_BYTES = 32768 # 32Kb
+      end
+
+      module MessageBatch
+        MAX_BYTES = 512_000 # 500Kb
+        MAX_SIZE = 100
+      end
+
+      module BackoffPolicy
+        MIN_TIMEOUT_MS = 100
+        MAX_TIMEOUT_MS = 10000
+        MULTIPLIER = 1.5
+        RANDOMIZATION_FACTOR = 0.5
+      end
     end
   end
 end

data/lib/segment/analytics/logging.rb

@@ -14,12 +14,10 @@ module Segment
                       end
         end
 
-        def logger= logger
-          @logger = logger
-        end
+        attr_writer :logger
       end
 
-      def self.included base
+      def self.included(base)
         class << base
           def logger
             Logging.logger

data/lib/segment/analytics/message.rb

@@ -0,0 +1,26 @@
+require 'segment/analytics/defaults'
+
+module Segment
+  class Analytics
+    # Represents a message to be sent to the API
+    class Message
+      def initialize(hash)
+        @hash = hash
+      end
+
+      def too_big?
+        json_size > Defaults::Message::MAX_BYTES
+      end
+
+      def json_size
+        to_json.bytesize
+      end
+
+      # Since the hash is expected to not be modified (set at initialization),
+      # the JSON version can be cached after the first computation.
+      def to_json(*args)
+        @json ||= @hash.to_json(*args)
+      end
+    end
+  end
+end

data/lib/segment/analytics/message_batch.rb

@@ -0,0 +1,58 @@
+require 'segment/analytics/logging'
+
+module Segment
+  class Analytics
+    # A batch of `Message`s to be sent to the API
+    class MessageBatch
+      extend Forwardable
+      include Segment::Analytics::Logging
+      include Segment::Analytics::Defaults::MessageBatch
+
+      def initialize(max_message_count)
+        @messages = []
+        @max_message_count = max_message_count
+        @json_size = 0
+      end
+
+      def <<(message)
+        if message.too_big?
+          logger.error('a message exceeded the maximum allowed size')
+        else
+          @messages << message
+          @json_size += message.json_size + 1 # One byte for the comma
+        end
+      end
+
+      def full?
+        item_count_exhausted? || size_exhausted?
+      end
+
+      def clear
+        @messages.clear
+        @json_size = 0
+      end
+
+      def_delegators :@messages, :to_json
+      def_delegators :@messages, :empty?
+      def_delegators :@messages, :length
+
+      private
+
+      def item_count_exhausted?
+        @messages.length >= @max_message_count
+      end
+
+      # We consider the max size here as just enough to leave room for one more
+      # message of the largest size possible. This is a shortcut that allows us
+      # to use a native Ruby `Queue` that doesn't allow peeking. The tradeoff
+      # here is that we might fit in less messages than possible into a batch.
+      #
+      # The alternative is to use our own `Queue` implementation that allows
+      # peeking, and to consider the next message size when calculating whether
+      # the message can be accomodated in this batch.
+      def size_exhausted?
+        @json_size >= (MAX_BYTES - Defaults::Message::MAX_BYTES)
+      end
+    end
+  end
+end

data/lib/segment/analytics/request.rb

@@ -2,6 +2,7 @@ require 'segment/analytics/defaults'
 require 'segment/analytics/utils'
 require 'segment/analytics/response'
 require 'segment/analytics/logging'
+require 'segment/analytics/backoff_policy'
 require 'net/http'
 require 'net/https'
 require 'json'
@@ -19,10 +20,11 @@ module Segment
         options[:host] ||= HOST
         options[:port] ||= PORT
         options[:ssl] ||= SSL
-        options[:headers] ||= HEADERS
+        @headers = options[:headers] || HEADERS
         @path = options[:path] || PATH
         @retries = options[:retries] || RETRIES
-        @backoff = options[:backoff] || BACKOFF
+        @backoff_policy =
+          options[:backoff_policy] || Segment::Analytics::BackoffPolicy.new
 
         http = Net::HTTP.new(options[:host], options[:port])
         http.use_ssl = options[:ssl]
@@ -36,42 +38,92 @@ module Segment
       #
       # returns - Response of the status and error if it exists
       def post(write_key, batch)
-        status, error = nil, nil
-        remaining_retries = @retries
-        backoff = @backoff
-        headers = { 'Content-Type' => 'application/json', 'accept' => 'application/json' }
+        last_response, exception = retry_with_backoff(@retries) do
+          status_code, body = send_request(write_key, batch)
+          error = JSON.parse(body)['error']
+          should_retry = should_retry_request?(status_code, body)
+
+          [Response.new(status_code, error), should_retry]
+        end
+
+        if exception
+          logger.error(exception.message)
+          exception.backtrace.each { |line| logger.error(line) }
+          Response.new(-1, "Connection error: #{exception}")
+        else
+          last_response
+        end
+      end
+
+      private
+
+      def should_retry_request?(status_code, body)
+        if status_code >= 500
+          true # Server error
+        elsif status_code == 429
+          true # Rate limited
+        elsif status_code >= 400
+          logger.error(body)
+          false # Client error. Do not retry, but log
+        else
+          false
+        end
+      end
+
+      # Takes a block that returns [result, should_retry].
+      #
+      # Retries upto `retries_remaining` times, if `should_retry` is false or
+      # an exception is raised. `@backoff_policy` is used to determine the
+      # duration to sleep between attempts
+      #
+      # Returns [last_result, raised_exception]
+      def retry_with_backoff(retries_remaining, &block)
+        result, caught_exception = nil
+        should_retry = false
+
         begin
-          payload = JSON.generate :sentAt => datetime_in_iso8601(Time.new), :batch => batch
-          request = Net::HTTP::Post.new(@path, headers)
-          request.basic_auth write_key, nil
-
-          if self.class.stub
-            status = 200
-            error = nil
-            logger.debug "stubbed request to #{@path}: write key = #{write_key}, payload = #{payload}"
-          else
-            res = @http.request(request, payload)
-            status = res.code.to_i
-            body = JSON.parse(res.body)
-            error = body["error"]
-          end
-        rescue Exception => e
-          unless (remaining_retries -=1).zero?
-            sleep(backoff)
-            retry
-          end
-
-          logger.error e.message
-          e.backtrace.each { |line| logger.error line }
-          status = -1
-          error = "Connection error: #{e}"
+          result, should_retry = yield
+          return [result, nil] unless should_retry
+        rescue StandardError => e
+          should_retry = true
+          caught_exception = e
+        end
+
+        if should_retry && (retries_remaining > 1)
+          sleep(@backoff_policy.next_interval.to_f / 1000)
+          retry_with_backoff(retries_remaining - 1, &block)
+        else
+          [result, caught_exception]
         end
+      end
+
+      # Sends a request for the batch, returns [status_code, body]
+      def send_request(write_key, batch)
+        payload = JSON.generate(
+          :sentAt => datetime_in_iso8601(Time.now),
+          :batch => batch
+        )
+        request = Net::HTTP::Post.new(@path, @headers)
+        request.basic_auth(write_key, nil)
+
+        if self.class.stub
+          logger.debug "stubbed request to #{@path}: " \
+            "write key = #{write_key}, batch = JSON.generate(#{batch})"
 
-        Response.new status, error
+          [200, '{}']
+        else
+          # If `start` is not called, Ruby adds a 'Connection: close' header to
+          # all requests, preventing us from reusing a connection for multiple
+          # HTTP requests
+          @http.start unless @http.started?
+
+          response = @http.request(request, payload)
+          [response.code.to_i, response.body]
+        end
       end
 
       class << self
-        attr_accessor :stub
+        attr_writer :stub
 
         def stub
           @stub || ENV['STUB']

data/lib/segment/analytics/response.rb

@@ -13,4 +13,3 @@ module Segment
     end
   end
 end
-

data/lib/segment/analytics/utils.rb

@@ -8,7 +8,9 @@ module Segment
       # public: Return a new hash with keys converted from strings to symbols
       #
       def symbolize_keys(hash)
-        hash.inject({}) { |memo, (k,v)| memo[k.to_sym] = v; memo }
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_sym] = v
+        end
       end
 
       # public: Convert hash keys from strings to symbols in place
@@ -20,17 +22,18 @@ module Segment
       # public: Return a new hash with keys as strings
       #
       def stringify_keys(hash)
-        hash.inject({}) { |memo, (k,v)| memo[k.to_s] = v; memo }
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_s] = v
+        end
       end
 
       # public: Returns a new hash with all the date values in the into iso8601
       #         strings
       #
       def isoify_dates(hash)
-        hash.inject({}) { |memo, (k, v)|
+        hash.each_with_object({}) do |(k, v), memo|
           memo[k] = datetime_in_iso8601(v)
-          memo
-        }
+        end
       end
 
       # public: Converts all the date values in the into iso8601 strings in place
@@ -42,18 +45,18 @@ module Segment
       # public: Returns a uid string
       #
       def uid
-        arr = SecureRandom.random_bytes(16).unpack("NnnnnN")
+        arr = SecureRandom.random_bytes(16).unpack('NnnnnN')
         arr[2] = (arr[2] & 0x0fff) | 0x4000
         arr[3] = (arr[3] & 0x3fff) | 0x8000
-        "%08x-%04x-%04x-%04x-%04x%08x" % arr
+        '%08x-%04x-%04x-%04x-%04x%08x' % arr
       end
 
-      def datetime_in_iso8601 datetime
+      def datetime_in_iso8601(datetime)
         case datetime
         when Time
-            time_in_iso8601 datetime
+          time_in_iso8601 datetime
         when DateTime
-            time_in_iso8601 datetime.to_time
+          time_in_iso8601 datetime.to_time
         when Date
           date_in_iso8601 datetime
         else
@@ -61,19 +64,19 @@ module Segment
         end
       end
 
-      def time_in_iso8601 time, fraction_digits = 3
+      def time_in_iso8601(time, fraction_digits = 3)
         fraction = if fraction_digits > 0
-                     (".%06i" % time.usec)[0, fraction_digits + 1]
+                     ('.%06i' % time.usec)[0, fraction_digits + 1]
                    end
 
-        "#{time.strftime("%Y-%m-%dT%H:%M:%S")}#{fraction}#{formatted_offset(time, true, 'Z')}"
+        "#{time.strftime('%Y-%m-%dT%H:%M:%S')}#{fraction}#{formatted_offset(time, true, 'Z')}"
       end
 
-      def date_in_iso8601 date
-        date.strftime("%F")
+      def date_in_iso8601(date)
+        date.strftime('%F')
       end
 
-      def formatted_offset time, colon = true, alternate_utc_string = nil
+      def formatted_offset(time, colon = true, alternate_utc_string = nil)
         time.utc? && alternate_utc_string || seconds_to_utc_offset(time.utc_offset, colon)
       end
 

data/lib/segment/analytics/version.rb

@@ -1,5 +1,5 @@
 module Segment
   class Analytics
-    VERSION = '2.2.3.pre'
+    VERSION = '2.2.4.pre'
   end
 end

data/lib/segment/analytics/worker.rb

@@ -1,7 +1,8 @@
 require 'segment/analytics/defaults'
-require 'segment/analytics/utils'
-require 'segment/analytics/defaults'
+require 'segment/analytics/message'
+require 'segment/analytics/message_batch'
 require 'segment/analytics/request'
+require 'segment/analytics/utils'
 
 module Segment
   class Analytics
@@ -24,10 +25,11 @@ module Segment
         symbolize_keys! options
         @queue = queue
         @write_key = write_key
-        @batch_size = options[:batch_size] || Queue::BATCH_SIZE
-        @on_error = options[:on_error] || Proc.new { |status, error| }
-        @batch = []
+        @on_error = options[:on_error] || proc { |status, error| }
+        batch_size = options[:batch_size] || Defaults::MessageBatch::MAX_SIZE
+        @batch = MessageBatch.new(batch_size)
         @lock = Mutex.new
+        @request = Request.new
       end
 
       # public: Continuously runs the loop to check for new events
@@ -37,14 +39,13 @@ module Segment
           return if @queue.empty?
 
           @lock.synchronize do
-            until @batch.length >= @batch_size || @queue.empty?
-              @batch << @queue.pop
+            until @batch.full? || @queue.empty?
+              @batch << Message.new(@queue.pop)
             end
           end
 
-          res = Request.new.post @write_key, @batch
-
-          @on_error.call res.status, res.error unless res.status == 200
+          res = @request.post(@write_key, @batch)
+          @on_error.call(res.status, res.error) unless res.status == 200
 
           @lock.synchronize { @batch.clear }
         end

data/spec/helpers/runscope_client.rb

@@ -0,0 +1,38 @@
+require 'faraday'
+require 'pmap'
+
+class RunscopeClient
+  def initialize(api_token)
+    headers = { 'Authorization' => "Bearer #{api_token}" }
+    @conn = Faraday.new('https://api.runscope.com', headers: headers)
+  end
+
+  def requests(bucket_key)
+    with_retries(3) do
+      response = @conn.get("/buckets/#{bucket_key}/messages", count: 20)
+
+      raise "Runscope error. #{response.body}" unless response.status == 200
+
+      message_uuids = JSON.parse(response.body)['data'].map { |message|
+        message.fetch('uuid')
+      }
+
+      message_uuids.pmap { |uuid|
+        response = @conn.get("/buckets/#{bucket_key}/messages/#{uuid}")
+        raise "Runscope error. #{response.body}" unless response.status == 200
+        JSON.parse(response.body).fetch('data').fetch('request')
+      }
+    end
+  end
+
+  private
+
+  def with_retries(max_retries)
+    retries ||= 0
+    yield
+  rescue StandardError => e
+    retries += 1
+    retry if retries < max_retries
+    raise e
+  end
+end