clearbit 0.2.7 → 0.3.3

data/lib/clearbit/analytics/logging.rb

@@ -0,0 +1,60 @@
+require 'logger'
+
+module Clearbit
+  class Analytics
+    # Wraps an existing logger and adds a prefix to all messages
+    class PrefixedLogger
+      def initialize(logger, prefix)
+        @logger = logger
+        @prefix = prefix
+      end
+
+      def debug(msg)
+        @logger.debug("#{@prefix} #{msg}")
+      end
+
+      def info(msg)
+        @logger.info("#{@prefix} #{msg}")
+      end
+
+      def warn(msg)
+        @logger.warn("#{@prefix} #{msg}")
+      end
+
+      def error(msg)
+        @logger.error("#{@prefix} #{msg}")
+      end
+    end
+
+    module Logging
+      class << self
+        def logger
+          return @logger if @logger
+
+          base_logger = if defined?(Rails)
+                          Rails.logger
+                        else
+                          logger = Logger.new STDOUT
+                          logger.progname = 'Clearbit::Analytics'
+                          logger
+                        end
+          @logger = PrefixedLogger.new(base_logger, '[clearbit-ruby]')
+        end
+
+        attr_writer :logger
+      end
+
+      def self.included(base)
+        class << base
+          def logger
+            Logging.logger
+          end
+        end
+      end
+
+      def logger
+        Logging.logger
+      end
+    end
+  end
+end

data/lib/clearbit/analytics/message_batch.rb

@@ -0,0 +1,72 @@
+require 'forwardable'
+require 'clearbit/analytics/logging'
+
+module Clearbit
+  class Analytics
+    # A batch of `Message`s to be sent to the API
+    class MessageBatch
+      class JSONGenerationError < StandardError; end
+
+      extend Forwardable
+      include Clearbit::Analytics::Logging
+      include Clearbit::Analytics::Defaults::MessageBatch
+
+      def initialize(max_message_count)
+        @messages = []
+        @max_message_count = max_message_count
+        @json_size = 0
+      end
+
+      def <<(message)
+        begin
+          message_json = message.to_json
+        rescue StandardError => e
+          raise JSONGenerationError, "Serialization error: #{e}"
+        end
+
+        message_json_size = message_json.bytesize
+        if message_too_big?(message_json_size)
+          logger.error('a message exceeded the maximum allowed size')
+        else
+          @messages << message
+          @json_size += message_json_size + 1 # One byte for the comma
+        end
+      end
+
+      def full?
+        item_count_exhausted? || size_exhausted?
+      end
+
+      def clear
+        @messages.clear
+        @json_size = 0
+      end
+
+      def_delegators :@messages, :to_json
+      def_delegators :@messages, :empty?
+      def_delegators :@messages, :length
+
+      private
+
+      def item_count_exhausted?
+        @messages.length >= @max_message_count
+      end
+
+      def message_too_big?(message_json_size)
+        message_json_size > Defaults::Message::MAX_BYTES
+      end
+
+      # We consider the max size here as just enough to leave room for one more
+      # message of the largest size possible. This is a shortcut that allows us
+      # to use a native Ruby `Queue` that doesn't allow peeking. The tradeoff
+      # here is that we might fit in less messages than possible into a batch.
+      #
+      # The alternative is to use our own `Queue` implementation that allows
+      # peeking, and to consider the next message size when calculating whether
+      # the message can be accomodated in this batch.
+      def size_exhausted?
+        @json_size >= (MAX_BYTES - Defaults::Message::MAX_BYTES)
+      end
+    end
+  end
+end

data/lib/clearbit/analytics/request.rb

@@ -0,0 +1,134 @@
+require 'clearbit/analytics/defaults'
+require 'clearbit/analytics/utils'
+require 'clearbit/analytics/response'
+require 'clearbit/analytics/logging'
+require 'clearbit/analytics/backoff_policy'
+require 'net/http'
+require 'net/https'
+require 'json'
+
+module Clearbit
+  class Analytics
+    class Request
+      include Clearbit::Analytics::Defaults::Request
+      include Clearbit::Analytics::Utils
+      include Clearbit::Analytics::Logging
+
+      # public: Creates a new request object to send analytics batch
+      #
+      def initialize(options = {})
+        options[:host] ||= HOST
+        options[:port] ||= PORT
+        options[:ssl] ||= SSL
+        @headers = options[:headers] || HEADERS
+        @path = options[:path] || PATH
+        @retries = options[:retries] || RETRIES
+        @backoff_policy =
+          options[:backoff_policy] || Clearbit::Analytics::BackoffPolicy.new
+
+        http = Net::HTTP.new(options[:host], options[:port])
+        http.use_ssl = options[:ssl]
+        http.read_timeout = 8
+        http.open_timeout = 4
+
+        @http = http
+      end
+
+      # public: Posts the write key and batch of messages to the API.
+      #
+      # returns - Response of the status and error if it exists
+      def post(write_key, batch)
+        logger.debug("Sending request for #{batch.length} items")
+
+        last_response, exception = retry_with_backoff(@retries) do
+          status_code, body = send_request(write_key, batch)
+          error = JSON.parse(body)['error']
+          should_retry = should_retry_request?(status_code, body)
+          logger.debug("Response status code: #{status_code}")
+          logger.debug("Response error: #{error}") if error
+
+          [Response.new(status_code, error), should_retry]
+        end
+
+        if exception
+          logger.error(exception.message)
+          exception.backtrace.each { |line| logger.error(line) }
+          Response.new(-1, exception.to_s)
+        else
+          last_response
+        end
+      end
+
+      private
+
+      def should_retry_request?(status_code, body)
+        if status_code >= 500
+          true # Server error
+        elsif status_code == 429
+          true # Rate limited
+        elsif status_code >= 400
+          logger.error(body)
+          false # Client error. Do not retry, but log
+        else
+          false
+        end
+      end
+
+      # Takes a block that returns [result, should_retry].
+      #
+      # Retries upto `retries_remaining` times, if `should_retry` is false or
+      # an exception is raised. `@backoff_policy` is used to determine the
+      # duration to sleep between attempts
+      #
+      # Returns [last_result, raised_exception]
+      def retry_with_backoff(retries_remaining, &block)
+        result, caught_exception = nil
+        should_retry = false
+
+        begin
+          result, should_retry = yield
+          return [result, nil] unless should_retry
+        rescue StandardError => e
+          should_retry = true
+          caught_exception = e
+        end
+
+        if should_retry && (retries_remaining > 1)
+          logger.debug("Retrying request, #{retries_remaining} retries left")
+          sleep(@backoff_policy.next_interval.to_f / 1000)
+          retry_with_backoff(retries_remaining - 1, &block)
+        else
+          [result, caught_exception]
+        end
+      end
+
+      # Sends a request for the batch, returns [status_code, body]
+      def send_request(write_key, batch)
+        payload = JSON.generate(
+          :sentAt => datetime_in_iso8601(Time.now),
+          :batch => batch
+        )
+        request = Net::HTTP::Post.new(@path, @headers)
+        request.basic_auth(write_key, nil)
+
+        if self.class.stub
+          logger.debug "stubbed request to #{@path}: " \
+            "write key = #{write_key}, batch = JSON.generate(#{batch})"
+
+          [200, '{}']
+        else
+          response = @http.request(request, payload)
+          [response.code.to_i, response.body]
+        end
+      end
+
+      class << self
+        attr_writer :stub
+
+        def stub
+          @stub || ENV['STUB']
+        end
+      end
+    end
+  end
+end

data/lib/clearbit/analytics/response.rb

@@ -0,0 +1,15 @@
+module Clearbit
+  class Analytics
+    class Response
+      attr_reader :status, :error
+
+      # public: Simple class to wrap responses from the API
+      #
+      #
+      def initialize(status = 200, error = nil)
+        @status = status
+        @error  = error
+      end
+    end
+  end
+end

data/lib/clearbit/analytics/utils.rb

@@ -0,0 +1,91 @@
+require 'securerandom'
+
+module Clearbit
+  class Analytics
+    module Utils
+      extend self
+
+      # public: Return a new hash with keys converted from strings to symbols
+      #
+      def symbolize_keys(hash)
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_sym] = v
+        end
+      end
+
+      # public: Convert hash keys from strings to symbols in place
+      #
+      def symbolize_keys!(hash)
+        hash.replace symbolize_keys hash
+      end
+
+      # public: Return a new hash with keys as strings
+      #
+      def stringify_keys(hash)
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k.to_s] = v
+        end
+      end
+
+      # public: Returns a new hash with all the date values in the into iso8601
+      #         strings
+      #
+      def isoify_dates(hash)
+        hash.each_with_object({}) do |(k, v), memo|
+          memo[k] = datetime_in_iso8601(v)
+        end
+      end
+
+      # public: Converts all the date values in the into iso8601 strings in place
+      #
+      def isoify_dates!(hash)
+        hash.replace isoify_dates hash
+      end
+
+      # public: Returns a uid string
+      #
+      def uid
+        arr = SecureRandom.random_bytes(16).unpack('NnnnnN')
+        arr[2] = (arr[2] & 0x0fff) | 0x4000
+        arr[3] = (arr[3] & 0x3fff) | 0x8000
+        '%08x-%04x-%04x-%04x-%04x%08x' % arr
+      end
+
+      def datetime_in_iso8601(datetime)
+        case datetime
+        when Time
+          time_in_iso8601 datetime
+        when DateTime
+          time_in_iso8601 datetime.to_time
+        when Date
+          date_in_iso8601 datetime
+        else
+          datetime
+        end
+      end
+
+      def time_in_iso8601(time, fraction_digits = 3)
+        fraction = if fraction_digits > 0
+                     ('.%06i' % time.usec)[0, fraction_digits + 1]
+                   end
+
+        "#{time.strftime('%Y-%m-%dT%H:%M:%S')}#{fraction}#{formatted_offset(time, true, 'Z')}"
+      end
+
+      def date_in_iso8601(date)
+        date.strftime('%F')
+      end
+
+      def formatted_offset(time, colon = true, alternate_utc_string = nil)
+        time.utc? && alternate_utc_string || seconds_to_utc_offset(time.utc_offset, colon)
+      end
+
+      def seconds_to_utc_offset(seconds, colon = true)
+        (colon ? UTC_OFFSET_WITH_COLON : UTC_OFFSET_WITHOUT_COLON) % [(seconds < 0 ? '-' : '+'), (seconds.abs / 3600), ((seconds.abs % 3600) / 60)]
+      end
+
+      UTC_OFFSET_WITH_COLON = '%s%02d:%02d'
+      UTC_OFFSET_WITHOUT_COLON = UTC_OFFSET_WITH_COLON.sub(':', '')
+    end
+  end
+end

data/lib/clearbit/analytics/worker.rb

@@ -0,0 +1,66 @@
+require 'clearbit/analytics/defaults'
+require 'clearbit/analytics/message_batch'
+require 'clearbit/analytics/request'
+require 'clearbit/analytics/utils'
+
+module Clearbit
+  class Analytics
+    class Worker
+      include Clearbit::Analytics::Utils
+      include Clearbit::Analytics::Defaults
+      include Clearbit::Analytics::Logging
+
+      # public: Creates a new worker
+      #
+      # The worker continuously takes messages off the queue
+      # and makes requests to the segment.io api
+      #
+      # queue   - Queue synchronized between client and worker
+      # write_key  - String of the project's Write key
+      # options - Hash of worker options
+      #           batch_size - Fixnum of how many items to send in a batch
+      #           on_error   - Proc of what to do on an error
+      #
+      def initialize(queue, write_key, options = {})
+        symbolize_keys! options
+        @queue = queue
+        @write_key = write_key
+        @on_error = options[:on_error] || proc { |status, error| }
+        batch_size = options[:batch_size] || Defaults::MessageBatch::MAX_SIZE
+        @batch = MessageBatch.new(batch_size)
+        @lock = Mutex.new
+      end
+
+      # public: Continuously runs the loop to check for new events
+      #
+      def run
+        until Thread.current[:should_exit]
+          return if @queue.empty?
+
+          @lock.synchronize do
+            consume_message_from_queue! until @batch.full? || @queue.empty?
+          end
+
+          res = Request.new.post @write_key, @batch
+          @on_error.call(res.status, res.error) unless res.status == 200
+
+          @lock.synchronize { @batch.clear }
+        end
+      end
+
+      # public: Check whether we have outstanding requests.
+      #
+      def is_requesting?
+        @lock.synchronize { !@batch.empty? }
+      end
+
+      private
+
+      def consume_message_from_queue!
+        @batch << @queue.pop
+      rescue MessageBatch::JSONGenerationError => e
+        @on_error.call(-1, e.to_s)
+      end
+    end
+  end
+end

data/lib/clearbit/audiences.rb

@@ -0,0 +1,14 @@
+module Clearbit
+  class Audiences < Base
+    endpoint 'https://audiences.clearbit.com'
+    path '/v1/audiences'
+
+    def self.add_email(values = {})
+      post('email', values)
+    end
+
+    def self.add_domain(values = {})
+      post('domain', values)
+    end
+  end
+end