hawkei 1.0.0

data/lib/hawkei/hawkei_object.rb

@@ -0,0 +1,179 @@
+module Hawkei
+  ##
+  # == Hawkei \Object
+  #
+  # Define the API objects
+  class HawkeiObject
+    ##
+    # @return [Hash] JSON parsed response
+    attr_reader :raw
+
+    ##
+    # @return [Hash] JSON parsed response
+    attr_reader :data
+
+    attr_reader :errors
+
+    attr_reader :successful
+    alias successful? successful
+
+    class << self
+
+      ##
+      # Initialize from the API response
+      #
+      # @return [Hawkei::HawkeiObject]
+      def initialize_from(response, object = new)
+        object.load_response_api(response.is_a?(Hash) ? response : Util.safe_json_parse(response))
+        object.update_attributes(Hawkei::Util.except_keys(object.raw, :data))
+        if object.raw[:object] == 'list'
+          object.raw[:data].each do |response_object|
+            data = object.type_from_string_object(response_object[:object]).initialize_from(response_object)
+
+            object.add_data(data)
+          end
+        end
+
+        object
+      end
+
+    end
+
+    ##
+    # Update the object based on the response from the API
+    # Remove and new accessor
+    #
+    # @param [Hash] response
+    #
+    # @return [Hawkei::HawkeiObject]
+    def update_from(response)
+      self.class.initialize_from(response, self)
+
+      (@values.keys - raw.keys).each { |key| remove_accessor(key) }
+
+      self
+    end
+
+    ##
+    # Initialize and create accessor for values
+    #
+    # @params [Hash] values
+    def initialize(values = {})
+      @data   = []
+      @values = {}
+
+      update_attributes(values)
+    end
+
+    ##
+    # get attribute value
+    def [](key)
+      @values[key.to_sym]
+    end
+
+    ##
+    # set attribute value
+    def []=(key, value)
+      send(:"#{key}=", value)
+    end
+
+    ##
+    # @return [Array] all the keys
+    def keys
+      @values.keys
+    end
+
+    ##
+    # @return [Hash] values to hash
+    def to_hash
+      @values
+    end
+
+    ##
+    # @return [JSON] values to JSON
+    def to_json(_object = nil)
+      JSON.generate(@values)
+    end
+
+    ##
+    # Update the attribute and add accessor for new attributes
+    #
+    # @param [Hash] values
+    def update_attributes(attributes)
+      attributes.each do |(key, value)|
+        add_accessor(key, value)
+      end
+    end
+
+    ##
+    # Load the root components for the API response
+    #
+    # @param [Hash] values
+    def load_response_api(response)
+      @raw        = Hawkei::Util.deep_underscore_key(response)
+      @successful = true
+    end
+
+    ##
+    # Add data for sub-object
+    #
+    # @param [Object] data
+    def add_data(data)
+      @data << data
+    end
+
+    ##
+    # Create an object from a string
+    #
+    # @param [String] object to be build
+    #
+    # @return [Object]
+    def type_from_string_object(string_object)
+      klass_name = Hawkei::Util.camelize(string_object.to_s)
+
+      Object.const_get("Hawkei::#{klass_name}")
+    end
+
+    def inspect
+      id_string = respond_to?(:id) && !id.nil? ? " id=#{id}" : ''
+      "#<#{self.class}:0x#{object_id.to_s(16)}#{id_string}> JSON: " + JSON.pretty_generate(@values)
+    end
+
+    def method_missing(name, *args)
+      super unless name.to_s.end_with?('=')
+
+      attribute = name.to_s[0...-1].to_sym
+      value     = args.first
+
+      add_accessor(attribute, value)
+    end
+
+    private
+
+    def add_accessors(keys, payload = raw)
+      keys.each do |key|
+        add_accessor(key, payload[key])
+      end
+    end
+
+    def add_accessor(name, value)
+      @values[name] = value
+
+      define_singleton_method(name) { @values[name] }
+      define_singleton_method(:"#{name}=") do |v|
+        @values[name] = v
+      end
+
+      define_singleton_method(:"#{name}?") { value } if [FalseClass, TrueClass].include?(value.class)
+    end
+
+    def remove_accessor(name)
+      @values.delete(name)
+
+      singleton_class.class_eval { remove_method name.to_sym } if singleton_methods.include?(name.to_sym)
+      singleton_class.class_eval { remove_method "#{name}=".to_sym } if singleton_methods.include?("#{name}=".to_sym)
+      singleton_class.class_eval { remove_method "#{name}?".to_sym } if singleton_methods.include?("#{name}?".to_sym)
+    end
+
+  end
+end

data/lib/hawkei/library_name.rb

@@ -0,0 +1,3 @@
+module Hawkei # :nodoc:
+  LIBRARY_NAME = 'hawkei'.freeze
+end

data/lib/hawkei/message.rb

@@ -0,0 +1,79 @@
+module Hawkei
+  ##
+  # == Hawkei \Message
+  #
+  # Return the base message for an event
+  class Message
+    class << self
+
+      def base
+        Util.deep_compact(
+          message_id: SecureRandom.uuid,
+          timestamp: Time.now.utc.iso8601(3),
+          session_tracker_id: session_tracker_id,
+        )
+      end
+
+      def extended
+        Util.deep_compact(
+          library: library,
+          server: server,
+          request: Hawkei::Store.get(:request),
+          worker: Hawkei::Store.get(:worker),
+          environment: Hawkei::Store.get(:environment) || Hawkei.configurations.environment_name,
+          metadata: metadata
+        ).merge(base)
+      end
+
+      private
+
+      def session_tracker_id
+        return Hawkei::Store.get(:session_tracker_id) if Hawkei::Store.exist?(:session_tracker_id)
+
+        Hawkei::Store.set(:session_tracker_id, SecureRandom.uuid)
+      end
+
+      def library
+        {
+          name: Hawkei::LIBRARY_NAME,
+          language: 'ruby',
+          version: Hawkei::VERSION,
+        }
+      end
+
+      def server
+        {
+          host: Socket.gethostname,
+          pid: Process.pid,
+          software: Hawkei::Store.get(:server_software) || program_name,
+        }
+      end
+
+      def program_name
+        name = $PROGRAM_NAME
+
+        name.split('/').last
+      rescue StandardError => _e
+        name
+      end
+
+      def metadata
+        return nil if Hawkei.configurations.metadata.nil?
+
+        Hawkei.configurations.metadata.each_with_object({}) do |(key, value), hash|
+          if value.is_a?(Proc)
+            begin
+              hash[key] = value.call
+            rescue StandardError => _e
+              nil
+            end
+            next
+          end
+
+          hash[key] = value
+        end
+      end
+
+    end
+  end
+end

data/lib/hawkei/plugins/rack/middleware.rb

@@ -0,0 +1,139 @@
+require 'uri'
+
+module Hawkei
+  module Plugins
+    module Rack
+      ##
+      # Hawkei \Plugins \Rack \Middleware
+      #
+      # Middleware for Rack
+      #
+      class Middleware
+
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          request = ::Rack::Request.new(env)
+
+          store_tracker(request)
+          store_request_data(request, env)
+
+          Hawkei::Plugins::Rails::MiddlewareData.store_data(request) if defined?(::Rails)
+          status, headers, body = @app.call(env)
+
+          write_cookie_session_tracker_id!(headers)
+          [status, headers, body]
+        ensure
+          Hawkei::Store.clear! unless env['hawkei_test']
+        end
+
+        private
+
+        def session_tracker_id_name
+          '_hawkei_stid'.freeze
+        end
+
+        def store_tracker(request)
+          Hawkei::Store.set(
+            :session_tracker_id,
+            request.cookies[session_tracker_id_name] || SecureRandom.uuid
+          )
+        end
+
+        def store_request_data(request, env)
+          Hawkei::Store.set(
+            :request,
+            url: obfuscate_uri(request.url),
+            ssl: request.ssl?,
+            host: request.host,
+            port: request.port,
+            path: request.path,
+            referrer: obfuscate_uri(request.referrer),
+            method: request.request_method,
+            xhr: request.xhr?,
+            user_agent: request.user_agent,
+            ip: request.ip,
+            get_params: obfuscation_get_params(request, 'GET'),
+            post_params: obfuscation_get_params(request, 'POST'),
+            headers: obfuscate_headers(env)
+          )
+
+          Hawkei::Store.bulk_set(
+            server_software: request.env['SERVER_SOFTWARE']
+          )
+        end
+
+        def write_cookie_session_tracker_id!(headers)
+          ::Rack::Utils.set_cookie_header!(
+            headers || {},
+            session_tracker_id_name,
+            Util.deep_compact(
+              value: Hawkei::Store.get(:session_tracker_id),
+              path: '/',
+              domain: Hawkei.configurations.domain,
+              http_only: false,
+              max_age: (10 * 365 * 24 * 60 * 60),
+            )
+          )
+        end
+
+        def obfuscate_uri(url)
+          uri = URI.parse(url)
+
+          params =
+            Util.deep_obfuscate_value(
+              ::Rack::Utils.parse_query(uri.query),
+              Hawkei.configurations.obfuscated_fields,
+              'HIDDEN'
+            )
+
+          return url if params.empty?
+
+          uri.merge(
+            "?#{::Rack::Utils.build_query(params)}"
+          ).to_s
+        rescue StandardError => _e
+          ''
+        end
+
+        def obfuscation_get_params(request, type)
+          Util.deep_obfuscate_value(
+            request.send(type),
+            Hawkei.configurations.obfuscated_fields
+          )
+        rescue StandardError => _e
+          {}
+        end
+
+        def https_request?(env)
+          env['HTTPS'] == 'on' ||
+            env['HTTP_X_FORWARDED_SSL'] == 'on' ||
+            env['HTTP_X_FORWARDED_PROTO'].to_s.split(',').first == 'https' ||
+            env['rack.url_scheme'] == 'https'
+        end
+
+        def obfuscate_headers(env)
+          skip_headers = %w[HTTP_COOKIE]
+
+          headers = env.keys.grep(/^HTTP_|^CONTENT_/).each_with_object({}) do |key, hash|
+            next if skip_headers.include?(key)
+
+            name = key.gsub(/^HTTP_/, '').split('_').map(&:capitalize).join('-')
+
+            hash[name] = env[key]
+          end
+
+          Util.deep_obfuscate_value(
+            headers,
+            Hawkei.configurations.obfuscated_fields
+          )
+        rescue StandardError => _e
+          {}
+        end
+
+      end
+    end
+  end
+end

data/lib/hawkei/plugins/rails/data.rb

@@ -0,0 +1,19 @@
+module Hawkei
+  module Plugins
+    module Rails
+      class Data
+
+        class << self
+
+          def store_data
+            Hawkei::Store.bulk_set(
+              environment: ::Rails.env
+            )
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/hawkei/plugins/rails/middleware_data.rb

@@ -0,0 +1,28 @@
+module Hawkei
+  module Plugins
+    module Rails
+      class MiddlewareData
+
+        class << self
+
+          def store_data(request = nil)
+            store_request_data(request) if request
+          end
+
+          private
+
+          def store_request_data(request)
+            object = Hawkei::Store.get(:request) || {}
+
+            Hawkei::Store.set(
+              :request,
+              object.merge(id: request.env['action_dispatch.request_id'])
+            )
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/hawkei/plugins/rails/railtie.rb

@@ -0,0 +1,23 @@
+module Hawkei
+  module Plugins
+    module Rails
+      class Railtie < ::Rails::Railtie
+
+        initializer 'hawkei.middleware' do |app|
+          app.config.middleware.use(Hawkei::Plugins::Rack::Middleware)
+
+          if ActiveSupport.const_defined?(:Reloader) && ActiveSupport::Reloader.respond_to?(:to_complete)
+            ActiveSupport::Reloader.to_complete do
+              Hawkei::Store.clear!
+            end
+          elsif ActionDispatch.const_defined?(:Reloader) && ActionDispatch::Reloader.respond_to?(:to_cleanup)
+            ActionDispatch::Reloader.to_cleanup do
+              Hawkei::Store.clear!
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/hawkei/plugins/sidekiq/client_middleware.rb

@@ -0,0 +1,19 @@
+module Hawkei
+  module Plugins
+    module Sidekiq
+      ##
+      # Hawkei \Plugins \Sidekiq \Client \Middleware
+      #
+      # Client middleware for sidekiq
+      #
+      class ClientMiddleware
+
+        def call(_worker_class, job, _queue, _redis_pool)
+          job['_hawkei_stid'] = Hawkei::Store.store[:session_tracker_id]
+          yield
+        end
+
+      end
+    end
+  end
+end

data/lib/hawkei/plugins/sidekiq/load.rb

@@ -0,0 +1,14 @@
+require 'hawkei/plugins/sidekiq/client_middleware'
+require 'hawkei/plugins/sidekiq/server_middleware'
+
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Hawkei::Plugins::Sidekiq::ClientMiddleware
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Hawkei::Plugins::Sidekiq::ServerMiddleware
+  end
+end

data/lib/hawkei/plugins/sidekiq/server_middleware.rb

@@ -0,0 +1,48 @@
+module Hawkei
+  module Plugins
+    module Sidekiq
+      ##
+      # Hawkei \Plugins \Sidekiq \Server \Middleware
+      #
+      # Server middleware for sidekiq
+      #
+      class ServerMiddleware
+
+        def call(_worker, job, _queue)
+          Hawkei::Store.set(:session_tracker_id, job['_hawkei_stid'] || SecureRandom.uuid)
+          Hawkei::Store.set(:server_software, "Sidekiq #{::Sidekiq::VERSION}")
+          store_worker_data(job)
+          yield
+        ensure
+          Hawkei::Store.clear!
+        end
+
+        private
+
+        def store_worker_data(job)
+          Hawkei::Store.set(
+            :worker,
+            name: 'sidekiq',
+            version: ::Sidekiq::VERSION,
+            queue: job['queue'],
+            class: job['class'],
+            id: job['jid'],
+            created_at: convert_to_iso8601(job['created_at']),
+            process_at: convert_to_iso8601(job['enqueued_at']),
+            args: job['args'],
+            retried: job['retry_count'].is_a?(Integer),
+            retry_number: job['retry_count'],
+            failed_at: convert_to_iso8601(job['failed_at'])
+          )
+        end
+
+        def convert_to_iso8601(int_time)
+          Time.at(int_time).utc.iso8601(3)
+        rescue StandardError => _e
+          nil
+        end
+
+      end
+    end
+  end
+end

data/lib/hawkei/plugins.rb

@@ -0,0 +1,17 @@
+##
+# Rack & Rails
+#
+if defined?(Rack)
+  require 'hawkei/plugins/rack/middleware'
+
+  if defined?(Rails)
+    require 'hawkei/plugins/rails/railtie'
+    require 'hawkei/plugins/rails/middleware_data'
+    require 'hawkei/plugins/rails/data'
+  end
+end
+
+##
+# Sidekiq
+#
+require 'hawkei/plugins/sidekiq/load' if defined?(Sidekiq)

data/lib/hawkei/processor/async.rb

@@ -0,0 +1,50 @@
+module Hawkei
+  module Processor
+    class Async
+
+      def initialize
+        @queue = Queue.new
+        @state_worker = Concurrent::AtomicBoolean.new(true)
+        @worker = Worker.new(@queue, @state_worker)
+
+        at_exit do
+          shutdown_worker if worker_running?
+        end
+      end
+
+      def enqueue(attributes)
+        ensure_worker_running
+
+        @queue << attributes
+
+        true
+      end
+
+      private
+
+      def shutdown_worker
+        @state_worker.make_false
+        @queue << Hawkei::Processor::Worker::SHUTDOWN_MESSAGE
+        @worker_thread.wait
+
+        executor.shutdown
+        executor.wait_for_termination
+      end
+
+      def executor
+        @executor ||= Concurrent.global_io_executor
+      end
+
+      def ensure_worker_running
+        return if worker_running?
+
+        @worker_thread = Concurrent::Future.execute { @worker.run }
+      end
+
+      def worker_running?
+        @worker_thread && @worker_thread.incomplete?
+      end
+
+    end
+  end
+end

data/lib/hawkei/processor/batch.rb

@@ -0,0 +1,84 @@
+module Hawkei
+  module Processor
+    class Batch
+
+      attr_reader :messages
+
+      MAX_BYTES = 204_800 # 200Kb
+      MAX_MESSAGES = 100
+      MAX_MESSAGE_BYTES = 32_768 # 32Kb
+
+      DEFAULT_RETRY_TIME = 60
+
+      MAX_RETRY = 10
+      RETRY_MAP = {
+        1 => 2,
+        2 => 5,
+        3 => 10,
+        4 => 20,
+        5 => 30,
+        6 => 30,
+        7 => 30,
+        8 => 30,
+        9 => 30,
+        10 => 30,
+      }.freeze
+
+      def initialize
+        @retry_count = 0
+        @total_bytes = 0
+        @messages = []
+      end
+
+      def <<(message)
+        message_json_size = message.to_json.bytesize
+
+        if max_message_reached?(message_json_size)
+          Hawkei.configurations.logger.error('Message is too big to be send')
+          return false
+        end
+
+        @total_bytes += message_json_size
+
+        @messages << message
+
+        true
+      end
+
+      def empty?
+        @messages.size.zero?
+      end
+
+      def full?
+        max_messages_reached? || max_size_reached?
+      end
+
+      def next_retry
+        RETRY_MAP[@retry_count] || DEFAULT_RETRY_TIME
+      end
+
+      def update_retry
+        @retry_count += 1
+      end
+
+      def can_retry?
+        @retry_count < MAX_RETRY
+      end
+
+      private
+
+      def max_messages_reached?
+        @messages.length >= MAX_MESSAGES
+      end
+
+      def max_size_reached?
+        @total_bytes >= MAX_BYTES
+      end
+
+      def max_message_reached?(message_json_size)
+        message_json_size > MAX_MESSAGE_BYTES
+      end
+
+    end
+  end
+end