analytics-ruby 2.0.13 → 2.4.0

data/lib/segment/analytics/field_parser.rb

@@ -0,0 +1,192 @@
+module Segment
+  class Analytics
+    # Handles parsing fields according to the Segment Spec
+    #
+    # @see https://segment.com/docs/spec/
+    class FieldParser
+      class << self
+        include Segment::Analytics::Utils
+
+        # In addition to the common fields, track accepts:
+        #
+        # - "event"
+        # - "properties"
+        def parse_for_track(fields)
+          common = parse_common_fields(fields)
+
+          event = fields[:event]
+          properties = fields[:properties] || {}
+
+          check_presence!(event, 'event')
+          check_is_hash!(properties, 'properties')
+
+          isoify_dates! properties
+
+          common.merge({
+            :type => 'track',
+            :event => event.to_s,
+            :properties => properties
+          })
+        end
+
+        # In addition to the common fields, identify accepts:
+        #
+        # - "traits"
+        def parse_for_identify(fields)
+          common = parse_common_fields(fields)
+
+          traits = fields[:traits] || {}
+          check_is_hash!(traits, 'traits')
+          isoify_dates! traits
+
+          common.merge({
+            :type => 'identify',
+            :traits => traits
+          })
+        end
+
+        # In addition to the common fields, alias accepts:
+        #
+        # - "previous_id"
+        def parse_for_alias(fields)
+          common = parse_common_fields(fields)
+
+          previous_id = fields[:previous_id]
+          check_presence!(previous_id, 'previous_id')
+
+          common.merge({
+            :type => 'alias',
+            :previousId => previous_id
+          })
+        end
+
+        # In addition to the common fields, group accepts:
+        #
+        # - "group_id"
+        # - "traits"
+        def parse_for_group(fields)
+          common = parse_common_fields(fields)
+
+          group_id = fields[:group_id]
+          traits = fields[:traits] || {}
+
+          check_presence!(group_id, 'group_id')
+          check_is_hash!(traits, 'traits')
+
+          isoify_dates! traits
+
+          common.merge({
+            :type => 'group',
+            :groupId => group_id,
+            :traits => traits
+          })
+        end
+
+        # In addition to the common fields, page accepts:
+        #
+        # - "name"
+        # - "properties"
+        def parse_for_page(fields)
+          common = parse_common_fields(fields)
+
+          name = fields[:name] || ''
+          properties = fields[:properties] || {}
+
+          check_is_hash!(properties, 'properties')
+
+          isoify_dates! properties
+
+          common.merge({
+            :type => 'page',
+            :name => name.to_s,
+            :properties => properties
+          })
+        end
+
+        # In addition to the common fields, screen accepts:
+        #
+        # - "name"
+        # - "properties"
+        # - "category" (Not in spec, retained for backward compatibility"
+        def parse_for_screen(fields)
+          common = parse_common_fields(fields)
+
+          name = fields[:name]
+          properties = fields[:properties] || {}
+          category = fields[:category]
+
+          check_presence!(name, 'name')
+          check_is_hash!(properties, 'properties')
+
+          isoify_dates! properties
+
+          parsed = common.merge({
+            :type => 'screen',
+            :name => name,
+            :properties => properties
+          })
+
+          parsed[:category] = category if category
+
+          parsed
+        end
+
+        private
+
+        def parse_common_fields(fields)
+          timestamp = fields[:timestamp] || Time.new
+          message_id = fields[:message_id].to_s if fields[:message_id]
+          context = fields[:context] || {}
+
+          check_user_id! fields
+          check_timestamp! timestamp
+
+          add_context! context
+
+          parsed = {
+            :context => context,
+            :messageId => message_id,
+            :timestamp => datetime_in_iso8601(timestamp)
+          }
+
+          parsed[:userId] = fields[:user_id] if fields[:user_id]
+          parsed[:anonymousId] = fields[:anonymous_id] if fields[:anonymous_id]
+          parsed[:integrations] = fields[:integrations] if fields[:integrations]
+
+          # Not in spec, retained for backward compatibility
+          parsed[:options] = fields[:options] if fields[:options]
+
+          parsed
+        end
+
+        def check_user_id!(fields)
+          unless fields[:user_id] || fields[:anonymous_id]
+            raise ArgumentError, 'Must supply either user_id or anonymous_id'
+          end
+        end
+
+        def check_timestamp!(timestamp)
+          raise ArgumentError, 'Timestamp must be a Time' unless timestamp.is_a? Time
+        end
+
+        def add_context!(context)
+          context[:library] = { :name => 'analytics-ruby', :version => Segment::Analytics::VERSION.to_s }
+        end
+
+        # private: Ensures that a string is non-empty
+        #
+        # obj    - String|Number that must be non-blank
+        # name   - Name of the validated value
+        def check_presence!(obj, name)
+          if obj.nil? || (obj.is_a?(String) && obj.empty?)
+            raise ArgumentError, "#{name} must be given"
+          end
+        end
+
+        def check_is_hash!(obj, name)
+          raise ArgumentError, "#{name} must be a Hash" unless obj.is_a? Hash
+        end
+      end
+    end
+  end
+end

data/lib/segment/analytics/logging.rb

@@ -2,24 +2,49 @@ require 'logger'
 
 module Segment
   class Analytics
+    # Wraps an existing logger and adds a prefix to all messages
+    class PrefixedLogger
+      def initialize(logger, prefix)
+        @logger = logger
+        @prefix = prefix
+      end
+
+      def debug(msg)
+        @logger.debug("#{@prefix} #{msg}")
+      end
+
+      def info(msg)
+        @logger.info("#{@prefix} #{msg}")
+      end
+
+      def warn(msg)
+        @logger.warn("#{@prefix} #{msg}")
+      end
+
+      def error(msg)
+        @logger.error("#{@prefix} #{msg}")
+      end
+    end
+
     module Logging
       class << self
         def logger
-          @logger ||= if defined?(Rails)
-                        Rails.logger
-                      else
-                        logger = Logger.new STDOUT
-                        logger.progname = 'Segment::Analytics'
-                        logger
-                      end
-        end
+          return @logger if @logger
 
-        def logger= logger
-          @logger = logger
+          base_logger = if defined?(Rails)
+                          Rails.logger
+                        else
+                          logger = Logger.new STDOUT
+                          logger.progname = 'Segment::Analytics'
+                          logger
+                        end
+          @logger = PrefixedLogger.new(base_logger, '[analytics-ruby]')
         end
+
+        attr_writer :logger
       end
 
-      def self.included base
+      def self.included(base)
         class << base
           def logger
             Logging.logger

data/lib/segment/analytics/message_batch.rb

@@ -0,0 +1,72 @@
+require 'forwardable'
+require 'segment/analytics/logging'
+
+module Segment
+  class Analytics
+    # A batch of `Message`s to be sent to the API
+    class MessageBatch
+      class JSONGenerationError < StandardError; end
+
+      extend Forwardable
+      include Segment::Analytics::Logging
+      include Segment::Analytics::Defaults::MessageBatch
+
+      def initialize(max_message_count)
+        @messages = []
+        @max_message_count = max_message_count
+        @json_size = 0
+      end
+
+      def <<(message)
+        begin
+          message_json = message.to_json
+        rescue StandardError => e
+          raise JSONGenerationError, "Serialization error: #{e}"
+        end
+
+        message_json_size = message_json.bytesize
+        if message_too_big?(message_json_size)
+          logger.error('a message exceeded the maximum allowed size')
+        else
+          @messages << message
+          @json_size += message_json_size + 1 # One byte for the comma
+        end
+      end
+
+      def full?
+        item_count_exhausted? || size_exhausted?
+      end
+
+      def clear
+        @messages.clear
+        @json_size = 0
+      end
+
+      def_delegators :@messages, :to_json
+      def_delegators :@messages, :empty?
+      def_delegators :@messages, :length
+
+      private
+
+      def item_count_exhausted?
+        @messages.length >= @max_message_count
+      end
+
+      def message_too_big?(message_json_size)
+        message_json_size > Defaults::Message::MAX_BYTES
+      end
+
+      # We consider the max size here as just enough to leave room for one more
+      # message of the largest size possible. This is a shortcut that allows us
+      # to use a native Ruby `Queue` that doesn't allow peeking. The tradeoff
+      # here is that we might fit in less messages than possible into a batch.
+      #
+      # The alternative is to use our own `Queue` implementation that allows
+      # peeking, and to consider the next message size when calculating whether
+      # the message can be accomodated in this batch.
+      def size_exhausted?
+        @json_size >= (MAX_BYTES - Defaults::Message::MAX_BYTES)
+      end
+    end
+  end
+end

data/lib/segment/analytics/response.rb

@@ -13,4 +13,3 @@ module Segment
     end
   end
 end
-

data/lib/segment/analytics/test_queue.rb

@@ -0,0 +1,56 @@
+module Segment
+  class Analytics
+    class TestQueue
+      attr_reader :messages
+
+      def initialize
+        reset!
+      end
+
+      def [](key)
+        all[key]
+      end
+
+      def count
+        all.count
+      end
+
+      def <<(message)
+        all << message
+        send(message[:type]) << message
+      end
+
+      def alias
+        messages[:alias] ||= []
+      end
+
+      def all
+        messages[:all] ||= []
+      end
+
+      def group
+        messages[:group] ||= []
+      end
+
+      def identify
+        messages[:identify] ||= []
+      end
+
+      def page
+        messages[:page] ||= []
+      end
+
+      def screen
+        messages[:screen] ||= []
+      end
+
+      def track
+        messages[:track] ||= []
+      end
+
+      def reset!
+        @messages = {}
+      end
+    end
+  end
+end

data/lib/segment/analytics/transport.rb

@@ -0,0 +1,138 @@
+require 'segment/analytics/defaults'
+require 'segment/analytics/utils'
+require 'segment/analytics/response'
+require 'segment/analytics/logging'
+require 'segment/analytics/backoff_policy'
+require 'net/http'
+require 'net/https'
+require 'json'
+
+module Segment
+  class Analytics
+    class Transport
+      include Segment::Analytics::Defaults::Request
+      include Segment::Analytics::Utils
+      include Segment::Analytics::Logging
+
+      def initialize(options = {})
+        options[:host] ||= HOST
+        options[:port] ||= PORT
+        options[:ssl] ||= SSL
+        @headers = options[:headers] || HEADERS
+        @path = options[:path] || PATH
+        @retries = options[:retries] || RETRIES
+        @backoff_policy =
+          options[:backoff_policy] || Segment::Analytics::BackoffPolicy.new
+
+        http = Net::HTTP.new(options[:host], options[:port])
+        http.use_ssl = options[:ssl]
+        http.read_timeout = 8
+        http.open_timeout = 4
+
+        @http = http
+      end
+
+      # Sends a batch of messages to the API
+      #
+      # @return [Response] API response
+      def send(write_key, batch)
+        logger.debug("Sending request for #{batch.length} items")
+
+        last_response, exception = retry_with_backoff(@retries) do
+          status_code, body = send_request(write_key, batch)
+          error = JSON.parse(body)['error']
+          should_retry = should_retry_request?(status_code, body)
+          logger.debug("Response status code: #{status_code}")
+          logger.debug("Response error: #{error}") if error
+
+          [Response.new(status_code, error), should_retry]
+        end
+
+        if exception
+          logger.error(exception.message)
+          exception.backtrace.each { |line| logger.error(line) }
+          Response.new(-1, exception.to_s)
+        else
+          last_response
+        end
+      end
+
+      # Closes a persistent connection if it exists
+      def shutdown
+        @http.finish if @http.started?
+      end
+
+      private
+
+      def should_retry_request?(status_code, body)
+        if status_code >= 500
+          true # Server error
+        elsif status_code == 429
+          true # Rate limited
+        elsif status_code >= 400
+          logger.error(body)
+          false # Client error. Do not retry, but log
+        else
+          false
+        end
+      end
+
+      # Takes a block that returns [result, should_retry].
+      #
+      # Retries upto `retries_remaining` times, if `should_retry` is false or
+      # an exception is raised. `@backoff_policy` is used to determine the
+      # duration to sleep between attempts
+      #
+      # Returns [last_result, raised_exception]
+      def retry_with_backoff(retries_remaining, &block)
+        result, caught_exception = nil
+        should_retry = false
+
+        begin
+          result, should_retry = yield
+          return [result, nil] unless should_retry
+        rescue StandardError => e
+          should_retry = true
+          caught_exception = e
+        end
+
+        if should_retry && (retries_remaining > 1)
+          logger.debug("Retrying request, #{retries_remaining} retries left")
+          sleep(@backoff_policy.next_interval.to_f / 1000)
+          retry_with_backoff(retries_remaining - 1, &block)
+        else
+          [result, caught_exception]
+        end
+      end
+
+      # Sends a request for the batch, returns [status_code, body]
+      def send_request(write_key, batch)
+        payload = JSON.generate(
+          :sentAt => datetime_in_iso8601(Time.now),
+          :batch => batch
+        )
+        request = Net::HTTP::Post.new(@path, @headers)
+        request.basic_auth(write_key, nil)
+
+        if self.class.stub
+          logger.debug "stubbed request to #{@path}: " \
+            "write key = #{write_key}, batch = #{JSON.generate(batch)}"
+
+          [200, '{}']
+        else
+          @http.start unless @http.started? # Maintain a persistent connection
+          response = @http.request(request, payload)
+          [response.code.to_i, response.body]
+        end
+      end
+
+      class << self
+        attr_writer :stub
+
+        def stub
+          @stub || ENV['STUB']
+        end
+      end
+    end
+  end
+end

data/lib/segment/analytics/utils.rb

@@ -8,7 +8,9 @@ module Segment
       # public: Return a new hash with keys converted from strings to symbols
       #
       def symbolize_keys(hash)
-        hash.inject({}) { |memo, (k,v)| memo[k.to_sym] = v; memo }
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_sym] = v
+        end
       end
 
       # public: Convert hash keys from strings to symbols in place
@@ -20,17 +22,18 @@ module Segment
       # public: Return a new hash with keys as strings
       #
       def stringify_keys(hash)
-        hash.inject({}) { |memo, (k,v)| memo[k.to_s] = v; memo }
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_s] = v
+        end
       end
 
       # public: Returns a new hash with all the date values in the into iso8601
       #         strings
       #
       def isoify_dates(hash)
-        hash.inject({}) { |memo, (k, v)|
+        hash.each_with_object({}) do |(k, v), memo|
           memo[k] = datetime_in_iso8601(v)
-          memo
-        }
+        end
       end
 
       # public: Converts all the date values in the into iso8601 strings in place
@@ -42,18 +45,18 @@ module Segment
       # public: Returns a uid string
       #
       def uid
-        arr = SecureRandom.random_bytes(16).unpack("NnnnnN")
+        arr = SecureRandom.random_bytes(16).unpack('NnnnnN')
         arr[2] = (arr[2] & 0x0fff) | 0x4000
         arr[3] = (arr[3] & 0x3fff) | 0x8000
-        "%08x-%04x-%04x-%04x-%04x%08x" % arr
+        '%08x-%04x-%04x-%04x-%04x%08x' % arr
       end
 
-      def datetime_in_iso8601 datetime
+      def datetime_in_iso8601(datetime)
         case datetime
         when Time
-            time_in_iso8601 datetime
+          time_in_iso8601 datetime
         when DateTime
-            time_in_iso8601 datetime.to_time
+          time_in_iso8601 datetime.to_time
         when Date
           date_in_iso8601 datetime
         else
@@ -61,19 +64,15 @@ module Segment
         end
       end
 
-      def time_in_iso8601 time, fraction_digits = 3
-        fraction = if fraction_digits > 0
-                     (".%06i" % time.usec)[0, fraction_digits + 1]
-                   end
-
-        "#{time.strftime("%Y-%m-%dT%H:%M:%S")}#{fraction}#{formatted_offset(time, true, 'Z')}"
+      def time_in_iso8601(time)
+        "#{time.strftime('%Y-%m-%dT%H:%M:%S.%6N')}#{formatted_offset(time, true, 'Z')}"
       end
 
-      def date_in_iso8601 date
-        date.strftime("%F")
+      def date_in_iso8601(date)
+        date.strftime('%F')
       end
 
-      def formatted_offset time, colon = true, alternate_utc_string = nil
+      def formatted_offset(time, colon = true, alternate_utc_string = nil)
         time.utc? && alternate_utc_string || seconds_to_utc_offset(time.utc_offset, colon)
       end
 

data/lib/segment/analytics/version.rb

@@ -1,5 +1,5 @@
 module Segment
   class Analytics
-    VERSION = '2.0.13'
+    VERSION = '2.4.0'
   end
 end

data/lib/segment/analytics/worker.rb

@@ -1,13 +1,14 @@
 require 'segment/analytics/defaults'
+require 'segment/analytics/message_batch'
+require 'segment/analytics/transport'
 require 'segment/analytics/utils'
-require 'segment/analytics/defaults'
-require 'segment/analytics/request'
 
 module Segment
   class Analytics
     class Worker
       include Segment::Analytics::Utils
       include Segment::Analytics::Defaults
+      include Segment::Analytics::Logging
 
       # public: Creates a new worker
       #
@@ -24,10 +25,11 @@ module Segment
         symbolize_keys! options
         @queue = queue
         @write_key = write_key
-        @batch_size = options[:batch_size] || Queue::BATCH_SIZE
-        @on_error = options[:on_error] || Proc.new { |status, error| }
-        @batch = []
+        @on_error = options[:on_error] || proc { |status, error| }
+        batch_size = options[:batch_size] || Defaults::MessageBatch::MAX_SIZE
+        @batch = MessageBatch.new(batch_size)
         @lock = Mutex.new
+        @transport = Transport.new(options)
       end
 
       # public: Continuously runs the loop to check for new events
@@ -37,17 +39,16 @@ module Segment
           return if @queue.empty?
 
           @lock.synchronize do
-            until @batch.length >= @batch_size || @queue.empty?
-              @batch << @queue.pop
-            end
+            consume_message_from_queue! until @batch.full? || @queue.empty?
           end
 
-          res = Request.new.post @write_key, @batch
+          res = @transport.send @write_key, @batch
+          @on_error.call(res.status, res.error) unless res.status == 200
 
           @lock.synchronize { @batch.clear }
-
-          @on_error.call res.status, res.error unless res.status == 200
         end
+      ensure
+        @transport.shutdown
       end
 
       # public: Check whether we have outstanding requests.
@@ -55,6 +56,14 @@ module Segment
       def is_requesting?
         @lock.synchronize { !@batch.empty? }
       end
+
+      private
+
+      def consume_message_from_queue!
+        @batch << @queue.pop
+      rescue MessageBatch::JSONGenerationError => e
+        @on_error.call(-1, e.to_s)
+      end
     end
   end
 end