segment 2.2.5

data/lib/segment/analytics/defaults.rb

@@ -0,0 +1,36 @@
+module Segment
+  class Analytics
+    module Defaults
+      module Request
+        HOST = 'api.segment.io'
+        PORT = 443
+        PATH = '/v1/import'
+        SSL = true
+        HEADERS = { 'Accept' => 'application/json',
+                    'Content-Type' => 'application/json',
+                    'User-Agent' => "analytics-ruby/#{Analytics::VERSION}" }
+        RETRIES = 10
+      end
+
+      module Queue
+        MAX_SIZE = 10000
+      end
+
+      module Message
+        MAX_BYTES = 32768 # 32Kb
+      end
+
+      module MessageBatch
+        MAX_BYTES = 512_000 # 500Kb
+        MAX_SIZE = 100
+      end
+
+      module BackoffPolicy
+        MIN_TIMEOUT_MS = 100
+        MAX_TIMEOUT_MS = 10000
+        MULTIPLIER = 1.5
+        RANDOMIZATION_FACTOR = 0.5
+      end
+    end
+  end
+end

data/lib/segment/analytics/logging.rb

@@ -0,0 +1,33 @@
+require 'logger'
+
+module Segment
+  class Analytics
+    module Logging
+      class << self
+        def logger
+          @logger ||= if defined?(Rails)
+                        Rails.logger
+                      else
+                        logger = Logger.new STDOUT
+                        logger.progname = 'Segment::Analytics'
+                        logger
+                      end
+        end
+
+        attr_writer :logger
+      end
+
+      def self.included(base)
+        class << base
+          def logger
+            Logging.logger
+          end
+        end
+      end
+
+      def logger
+        Logging.logger
+      end
+    end
+  end
+end

data/lib/segment/analytics/message.rb

@@ -0,0 +1,26 @@
+require 'segment/analytics/defaults'
+
+module Segment
+  class Analytics
+    # Represents a message to be sent to the API
+    class Message
+      def initialize(hash)
+        @hash = hash
+      end
+
+      def too_big?
+        json_size > Defaults::Message::MAX_BYTES
+      end
+
+      def json_size
+        to_json.bytesize
+      end
+
+      # Since the hash is expected to not be modified (set at initialization),
+      # the JSON version can be cached after the first computation.
+      def to_json(*args)
+        @json ||= @hash.to_json(*args)
+      end
+    end
+  end
+end

data/lib/segment/analytics/message_batch.rb

@@ -0,0 +1,59 @@
+require 'forwardable'
+require 'segment/analytics/logging'
+
+module Segment
+  class Analytics
+    # A batch of `Message`s to be sent to the API
+    class MessageBatch
+      extend Forwardable
+      include Segment::Analytics::Logging
+      include Segment::Analytics::Defaults::MessageBatch
+
+      def initialize(max_message_count)
+        @messages = []
+        @max_message_count = max_message_count
+        @json_size = 0
+      end
+
+      def <<(message)
+        if message.too_big?
+          logger.error('a message exceeded the maximum allowed size')
+        else
+          @messages << message
+          @json_size += message.json_size + 1 # One byte for the comma
+        end
+      end
+
+      def full?
+        item_count_exhausted? || size_exhausted?
+      end
+
+      def clear
+        @messages.clear
+        @json_size = 0
+      end
+
+      def_delegators :@messages, :to_json
+      def_delegators :@messages, :empty?
+      def_delegators :@messages, :length
+
+      private
+
+      def item_count_exhausted?
+        @messages.length >= @max_message_count
+      end
+
+      # We consider the max size here as just enough to leave room for one more
+      # message of the largest size possible. This is a shortcut that allows us
+      # to use a native Ruby `Queue` that doesn't allow peeking. The tradeoff
+      # here is that we might fit in less messages than possible into a batch.
+      #
+      # The alternative is to use our own `Queue` implementation that allows
+      # peeking, and to consider the next message size when calculating whether
+      # the message can be accomodated in this batch.
+      def size_exhausted?
+        @json_size >= (MAX_BYTES - Defaults::Message::MAX_BYTES)
+      end
+    end
+  end
+end

data/lib/segment/analytics/request.rb

@@ -0,0 +1,134 @@
+require 'segment/analytics/defaults'
+require 'segment/analytics/utils'
+require 'segment/analytics/response'
+require 'segment/analytics/logging'
+require 'segment/analytics/backoff_policy'
+require 'net/http'
+require 'net/https'
+require 'json'
+
+module Segment
+  class Analytics
+    class Request
+      include Segment::Analytics::Defaults::Request
+      include Segment::Analytics::Utils
+      include Segment::Analytics::Logging
+
+      # public: Creates a new request object to send analytics batch
+      #
+      def initialize(options = {})
+        options[:host] ||= HOST
+        options[:port] ||= PORT
+        options[:ssl] ||= SSL
+        @headers = options[:headers] || HEADERS
+        @path = options[:path] || PATH
+        @retries = options[:retries] || RETRIES
+        @backoff_policy =
+          options[:backoff_policy] || Segment::Analytics::BackoffPolicy.new
+
+        http = Net::HTTP.new(options[:host], options[:port])
+        http.use_ssl = options[:ssl]
+        http.read_timeout = 8
+        http.open_timeout = 4
+
+        @http = http
+      end
+
+      # public: Posts the write key and batch of messages to the API.
+      #
+      # returns - Response of the status and error if it exists
+      def post(write_key, batch)
+        last_response, exception = retry_with_backoff(@retries) do
+          status_code, body = send_request(write_key, batch)
+          error = JSON.parse(body)['error']
+          should_retry = should_retry_request?(status_code, body)
+
+          [Response.new(status_code, error), should_retry]
+        end
+
+        if exception
+          logger.error(exception.message)
+          exception.backtrace.each { |line| logger.error(line) }
+          Response.new(-1, "Connection error: #{exception}")
+        else
+          last_response
+        end
+      end
+
+      private
+
+      def should_retry_request?(status_code, body)
+        if status_code >= 500
+          true # Server error
+        elsif status_code == 429
+          true # Rate limited
+        elsif status_code >= 400
+          logger.error(body)
+          false # Client error. Do not retry, but log
+        else
+          false
+        end
+      end
+
+      # Takes a block that returns [result, should_retry].
+      #
+      # Retries upto `retries_remaining` times, if `should_retry` is false or
+      # an exception is raised. `@backoff_policy` is used to determine the
+      # duration to sleep between attempts
+      #
+      # Returns [last_result, raised_exception]
+      def retry_with_backoff(retries_remaining, &block)
+        result, caught_exception = nil
+        should_retry = false
+
+        begin
+          result, should_retry = yield
+          return [result, nil] unless should_retry
+        rescue StandardError => e
+          should_retry = true
+          caught_exception = e
+        end
+
+        if should_retry && (retries_remaining > 1)
+          sleep(@backoff_policy.next_interval.to_f / 1000)
+          retry_with_backoff(retries_remaining - 1, &block)
+        else
+          [result, caught_exception]
+        end
+      end
+
+      # Sends a request for the batch, returns [status_code, body]
+      def send_request(write_key, batch)
+        payload = JSON.generate(
+          :sentAt => datetime_in_iso8601(Time.now),
+          :batch => batch
+        )
+        request = Net::HTTP::Post.new(@path, @headers)
+        request.basic_auth(write_key, nil)
+
+        if self.class.stub
+          logger.debug "stubbed request to #{@path}: " \
+            "write key = #{write_key}, batch = JSON.generate(#{batch})"
+
+          [200, '{}']
+        else
+          # If `start` is not called, Ruby adds a 'Connection: close' header to
+          # all requests, preventing us from reusing a connection for multiple
+          # HTTP requests
+          @http.start unless @http.started?
+
+          response = @http.request(request, payload)
+          [response.code.to_i, response.body]
+        end
+      end
+
+      class << self
+        attr_writer :stub
+
+        def stub
+          @stub || ENV['STUB']
+        end
+      end
+    end
+  end
+end

data/lib/segment/analytics/response.rb

@@ -0,0 +1,15 @@
+module Segment
+  class Analytics
+    class Response
+      attr_reader :status, :error
+
+      # public: Simple class to wrap responses from the API
+      #
+      #
+      def initialize(status = 200, error = nil)
+        @status = status
+        @error  = error
+      end
+    end
+  end
+end

data/lib/segment/analytics/utils.rb

@@ -0,0 +1,91 @@
+require 'securerandom'
+
+module Segment
+  class Analytics
+    module Utils
+      extend self
+
+      # public: Return a new hash with keys converted from strings to symbols
+      #
+      def symbolize_keys(hash)
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_sym] = v
+        end
+      end
+
+      # public: Convert hash keys from strings to symbols in place
+      #
+      def symbolize_keys!(hash)
+        hash.replace symbolize_keys hash
+      end
+
+      # public: Return a new hash with keys as strings
+      #
+      def stringify_keys(hash)
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_s] = v
+        end
+      end
+
+      # public: Returns a new hash with all the date values in the into iso8601
+      #         strings
+      #
+      def isoify_dates(hash)
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k] = datetime_in_iso8601(v)
+        end
+      end
+
+      # public: Converts all the date values in the into iso8601 strings in place
+      #
+      def isoify_dates!(hash)
+        hash.replace isoify_dates hash
+      end
+
+      # public: Returns a uid string
+      #
+      def uid
+        arr = SecureRandom.random_bytes(16).unpack('NnnnnN')
+        arr[2] = (arr[2] & 0x0fff) | 0x4000
+        arr[3] = (arr[3] & 0x3fff) | 0x8000
+        '%08x-%04x-%04x-%04x-%04x%08x' % arr
+      end
+
+      def datetime_in_iso8601(datetime)
+        case datetime
+        when Time
+          time_in_iso8601 datetime
+        when DateTime
+          time_in_iso8601 datetime.to_time
+        when Date
+          date_in_iso8601 datetime
+        else
+          datetime
+        end
+      end
+
+      def time_in_iso8601(time, fraction_digits = 3)
+        fraction = if fraction_digits > 0
+                     ('.%06i' % time.usec)[0, fraction_digits + 1]
+                   end
+
+        "#{time.strftime('%Y-%m-%dT%H:%M:%S')}#{fraction}#{formatted_offset(time, true, 'Z')}"
+      end
+
+      def date_in_iso8601(date)
+        date.strftime('%F')
+      end
+
+      def formatted_offset(time, colon = true, alternate_utc_string = nil)
+        time.utc? && alternate_utc_string || seconds_to_utc_offset(time.utc_offset, colon)
+      end
+
+      def seconds_to_utc_offset(seconds, colon = true)
+        (colon ? UTC_OFFSET_WITH_COLON : UTC_OFFSET_WITHOUT_COLON) % [(seconds < 0 ? '-' : '+'), (seconds.abs / 3600), ((seconds.abs % 3600) / 60)]
+      end
+
+      UTC_OFFSET_WITH_COLON = '%s%02d:%02d'
+      UTC_OFFSET_WITHOUT_COLON = UTC_OFFSET_WITH_COLON.sub(':', '')
+    end
+  end
+end

data/lib/segment/analytics/version.rb

@@ -0,0 +1,5 @@
+module Segment
+  class Analytics
+    VERSION = '2.2.5'
+  end
+end

data/lib/segment/analytics/worker.rb

@@ -0,0 +1,61 @@
+require 'segment/analytics/defaults'
+require 'segment/analytics/message'
+require 'segment/analytics/message_batch'
+require 'segment/analytics/request'
+require 'segment/analytics/utils'
+
+module Segment
+  class Analytics
+    class Worker
+      include Segment::Analytics::Utils
+      include Segment::Analytics::Defaults
+
+      # public: Creates a new worker
+      #
+      # The worker continuously takes messages off the queue
+      # and makes requests to the segment.io api
+      #
+      # queue   - Queue synchronized between client and worker
+      # write_key  - String of the project's Write key
+      # options - Hash of worker options
+      #           batch_size - Fixnum of how many items to send in a batch
+      #           on_error   - Proc of what to do on an error
+      #
+      def initialize(queue, write_key, options = {})
+        symbolize_keys! options
+        @queue = queue
+        @write_key = write_key
+        @on_error = options[:on_error] || proc { |status, error| }
+        batch_size = options[:batch_size] || Defaults::MessageBatch::MAX_SIZE
+        @batch = MessageBatch.new(batch_size)
+        @lock = Mutex.new
+        @request = Request.new
+      end
+
+      # public: Continuously runs the loop to check for new events
+      #
+      def run
+        until Thread.current[:should_exit]
+          return if @queue.empty?
+
+          @lock.synchronize do
+            until @batch.full? || @queue.empty?
+              @batch << Message.new(@queue.pop)
+            end
+          end
+
+          res = @request.post(@write_key, @batch)
+          @on_error.call(res.status, res.error) unless res.status == 200
+
+          @lock.synchronize { @batch.clear }
+        end
+      end
+
+      # public: Check whether we have outstanding requests.
+      #
+      def is_requesting?
+        @lock.synchronize { !@batch.empty? }
+      end
+    end
+  end
+end