tcell_agent 1.1.3 → 1.1.4

data/lib/tcell_agent/servers/webrick.rb

@@ -1,12 +1,10 @@
 Rack::Handler::WEBrick.class_eval do
-
   class << self
     alias_method :original_run, :run
-    def run(app, options={})
-      TCellAgent.run_instrumentation("WEBrick")
+    def run(app, options = {})
+      TCellAgent.run_instrumentation('WEBrick')
 
       original_run(app, options)
     end
   end
-
 end

data/lib/tcell_agent/settings_reporter.rb

@@ -0,0 +1,126 @@
+require 'tcell_agent/agent'
+require 'tcell_agent/configuration'
+require 'tcell_agent/sensor_events/app_config'
+require 'tcell_agent/sensor_events/server_agent'
+require 'thread'
+
+module TCellAgent
+  def self.report_settings(send_startup_events)
+    return unless send_startup_events && TCellAgent.configuration.should_instrument?
+
+    Thread.new do
+      TCellAgent::Instrumentation.safe_block('Instrumenting Agent Details') do
+        event = TCellAgent::SensorEvents::ServerAgentDetailsSensorEvent.new
+        TCellAgent.send_event(event)
+      end
+
+      TCellAgent::Instrumentation.safe_block('Instrumenting Server Packages') do
+        event = TCellAgent::SensorEvents::ServerAgentPackagesSensorEvent.new
+        TCellAgent.send_event(event)
+      end
+
+      TCellAgent::Instrumentation.safe_block('Instrumenting Native Lib Status') do
+        require 'tcell_agent/rust/whisperer'
+
+        TCellAgent.send_event(
+          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            'native_lib_loaded',
+            TCellAgent::Rust::Wrapper.common_lib_available?.to_s
+          )
+        )
+      end
+
+      TCellAgent::Instrumentation.safe_block('Instrumenting Initial Config') do
+        TCellAgent.send_event(
+          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            'allow_payloads',
+            (!!TCellAgent.configuration.allow_payloads).to_s # rubocop:disable Style/DoubleNegation
+          )
+        )
+
+        TCellAgent.send_event(
+          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            'reverse_proxy',
+            (!!TCellAgent.configuration.reverse_proxy).to_s # rubocop:disable Style/DoubleNegation
+          )
+        )
+
+        # Because of all the diff ways to initialize the agent
+        # some some of the following vars might not be set until
+        # we call this method, so call this method to set all
+        # the variables
+        TCellAgent.configuration.log_filename
+
+        TCellAgent.send_event(
+          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            'config_filename',
+            TCellAgent.configuration.config_filename
+          )
+        )
+        TCellAgent.send_event(
+          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            'logging_directory',
+            TCellAgent.configuration.agent_log_dir
+          )
+        )
+
+        TCellAgent.send_event(
+          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            'agent_home_directory',
+            TCellAgent.configuration.agent_home_dir
+          )
+        )
+
+        TCellAgent.send_event(
+          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            'agent_home_owner',
+            TCellAgent.configuration.agent_home_owner
+          )
+        )
+
+        logging_options = TCellAgent.configuration.logging_options || {}
+        use_default_setting = !logging_options.key?(:enabled) && !logging_options.key?('enabled')
+        if use_default_setting || logging_options[:enabled] || logging_options['enabled']
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::TCellAgentSettingEvent.new('logging_enabled', 'true')
+          )
+
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+              'logging_level',
+              logging_options[:level] || logging_options['level'] || 'INFO'
+            )
+          )
+        else
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::TCellAgentSettingEvent.new('logging_enabled', 'false')
+          )
+        end
+
+        if TCellAgent.configuration.hmac_key
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+              'hmac_key_present',
+              (!!TCellAgent.configuration.hmac_key).to_s # rubocop:disable Style/DoubleNegation
+            )
+          )
+        end
+
+        if TCellAgent.configuration.reverse_proxy
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+              'reverse_proxy_ip_address_header',
+              TCellAgent.configuration.reverse_proxy_ip_address_header
+            )
+          )
+        end
+      end
+
+      if defined?(::Rails)
+        TCellAgent::Instrumentation.safe_block('Instrumenting routes') do
+          TCellAgent::Instrumentation::Rails.instrument_routes
+        end
+      end
+    end
+  end
+end

data/lib/tcell_agent/sinatra.rb

@@ -6,36 +6,34 @@ require 'tcell_agent/agent'
 require 'tcell_agent/instrumentation'
 
 module TCellAgent
-    class Sinatra::Response
-      include Sinatra
+  class Sinatra::Response # rubocop:disable Style/ClassAndModuleChildren
+    include Sinatra
 
-      alias_method :original_finish, :finish
-      def finish
-        status, headers, response = original_finish
+    alias_method :original_finish, :finish
+    def finish
+      status, headers, response = original_finish
 
-        TCellAgent::Instrumentation.safe_block("Setting Headers") do
-          rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
-          if rust_policies
-            policy_headers = rust_policies.get_headers(
-              request.env[TCellAgent::Instrumentation::TCELL_ID]
-            )
-            policy_headers.each do |header_info|
-              header_name = header_info['name']
-              header_value = header_info['value']
-              existing_header_value = headers[header_name]
-              if existing_header_value
-                headers[header_name] = "#{existing_header_value}, #{header_value}"
-              else
-                headers[header_name] = header_value
-              end
-            end
-            response = [status, headers, active_response]
+      TCellAgent::Instrumentation.safe_block('Setting Headers') do
+        rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
+        if rust_policies
+          policy_headers = rust_policies.get_headers(
+            request.env[TCellAgent::Instrumentation::TCELL_ID]
+          )
+          policy_headers.each do |header_info|
+            header_name = header_info['name']
+            header_value = header_info['value']
+            existing_header_value = headers[header_name]
+            headers[header_name] = if existing_header_value
+                                     "#{existing_header_value}, #{header_value}"
+                                   else
+                                     header_value
+                                   end
           end
+          response = [status, headers, active_response]
         end
-
-        [status, headers, response]
       end
+
+      [status, headers, response]
     end
+  end
 end
-
-

data/lib/tcell_agent/start_background_thread.rb

@@ -1,176 +1,55 @@
 # See the file "LICENSE" for the full license governing this code.
 require 'tcell_agent/configuration'
 
-if (TCellAgent.configuration.disable_all == false)
+unless TCellAgent.configuration.disable_all
   require 'tcell_agent/logger'
   require 'tcell_agent/agent'
   require 'thread'
+
   module TCellAgent
-    #require 'tcell_agent/sinatra' if defined?(Sinatra)
+    # require 'tcell_agent/sinatra' if defined?(Sinatra)
     require 'tcell_agent/rails' if defined?(Rails)
     require 'tcell_agent/cmdi'
 
-    def self.run_instrumentation(server_name, send_startup_events=true)
-
+    def self.run_instrumentation(server_name, send_startup_events = true)
       require 'tcell_agent/hooks/login_fraud'
       require 'tcell_agent/rails/on_start' if defined?(Rails)
+      require 'tcell_agent/settings_reporter'
 
-      TCellAgent::Instrumentation.safe_block("Starting thread agent") do
+      TCellAgent::Instrumentation.safe_block('Starting thread agent') do
         TCellAgent.logger.debug("Instrumenting: #{server_name}")
         TCellAgent.thread_agent.start
       end
 
-      if send_startup_events && TCellAgent.configuration.should_instrument?
-        Thread.new do
-
-          TCellAgent::Instrumentation.safe_block("Instrumenting Agent Details") do
-            event = TCellAgent::SensorEvents::ServerAgentDetailsSensorEvent.new
-            TCellAgent.send_event(event)
-          end
-
-          TCellAgent::Instrumentation.safe_block("Instrumenting Server Packages") do
-            event = TCellAgent::SensorEvents::ServerAgentPackagesSensorEvent.new
-            TCellAgent.send_event(event)
-          end
-
-          TCellAgent::Instrumentation.safe_block("Instrumenting Initial Config") do
-            require 'tcell_agent/rust/whisperer'
-
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-              "native_lib_loaded",
-              TCellAgent::Rust::Wrapper.common_lib_available?.to_s)
-            )
-
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-              "allow_payloads",
-              (!!TCellAgent.configuration.allow_payloads).to_s)
-            )
-
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                "reverse_proxy",
-                (!!TCellAgent.configuration.reverse_proxy).to_s)
-            )
-
-            # Because of all the diff ways to initialize the agent
-            # some some of the following vars might not be set until
-            # we call this method, so call this method to set all
-            # the variables
-            TCellAgent.configuration.log_filename
-
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                "config_filename",
-                TCellAgent.configuration.config_filename)
-            )
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                "logging_directory",
-                TCellAgent.configuration.agent_log_dir)
-            )
-
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                "agent_home_directory",
-                TCellAgent.configuration.agent_home_dir)
-            )
-
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                "agent_home_owner",
-                TCellAgent.configuration.agent_home_owner)
-            )
-
-            logging_options = TCellAgent.configuration.logging_options || {}
-            use_default_setting = !logging_options.has_key?(:enabled) && !logging_options.has_key?("enabled")
-            if use_default_setting || logging_options[:enabled] || logging_options["enabled"]
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::TCellAgentSettingEvent.new("logging_enabled", "true")
-              )
-
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                  "logging_level",
-                  logging_options[:level] || logging_options["level"] || "INFO"
-                )
-              )
-
-              # Deprecated: this is so we can ensure no one is using this setting so we can remove
-              # the code altogether in the future
-              if logging_options.has_key?(:log_dir) || logging_options.has_key?("log_dir")
-                TCellAgent.send_event(
-                  TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                    "deprecated_log_dir",
-                    logging_options[:log_dir] || logging_options["log_dir"]
-                  )
-                )
-              end
-            else
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::TCellAgentSettingEvent.new("logging_enabled", "false")
-              )
-            end
-
-            if TCellAgent.configuration.hmac_key
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                  "hmac_key_present",
-                  (!!TCellAgent.configuration.hmac_key).to_s
-                )
-              )
-            end
-
-            if TCellAgent.configuration.reverse_proxy
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-                  "reverse_proxy_ip_address_header",
-                  TCellAgent.configuration.reverse_proxy_ip_address_header)
-              )
-            end
-          end
-
-          if defined?(Rails)
-            TCellAgent::Instrumentation.safe_block("Instrumenting routes") do
-              TCellAgent::Instrumentation::Rails.instrument_routes
-            end
-          end
-        end
-      end
-
+      report_settings(send_startup_events)
     end
-
   end
 
-  tcell_server = ENV["TCELL_AGENT_SERVER"]
+  tcell_server = ENV['TCELL_AGENT_SERVER']
 
   if TCellAgent.configuration.should_instrument?
-    if (!(tcell_server && tcell_server == "mock"))
+    unless tcell_server && tcell_server == 'mock'
 
-      if (tcell_server && tcell_server == "webrick") || defined?(Rails::Server)
-        require("tcell_agent/servers/rails_server")
+      if (tcell_server && tcell_server == 'webrick') || defined?(Rails::Server)
+        require('tcell_agent/servers/rails_server')
 
-      elsif (tcell_server && tcell_server == "thin") || defined?(Thin)
-        require("tcell_agent/servers/thin")
+      elsif (tcell_server && tcell_server == 'thin') || defined?(Thin)
+        require('tcell_agent/servers/thin')
 
-      elsif (tcell_server && tcell_server == "puma") || defined?(Puma)
-        require("tcell_agent/servers/puma")
+      elsif (tcell_server && tcell_server == 'puma') || defined?(Puma)
+        require('tcell_agent/servers/puma')
 
-      elsif (tcell_server && tcell_server == "unicorn") || defined?(Unicorn)
-        require("tcell_agent/servers/unicorn")
+      elsif (tcell_server && tcell_server == 'unicorn') || defined?(Unicorn)
+        require('tcell_agent/servers/unicorn')
 
-      elsif (tcell_server && tcell_server == "passenger") || defined?(PhusionPassenger)
-        require("tcell_agent/servers/passenger")
+      elsif (tcell_server && tcell_server == 'passenger') || defined?(PhusionPassenger)
+        require('tcell_agent/servers/passenger')
 
       end
     end
 
-  else
-
+  elsif (tcell_server && tcell_server == 'unicorn') || defined?(Unicorn)
     # unicorn is always instrumented to support rolling restarts
-    if (tcell_server && tcell_server == "unicorn") || defined?(Unicorn)
-      require("tcell_agent/servers/unicorn")
-    end
+    require('tcell_agent/servers/unicorn')
   end
 end

data/lib/tcell_agent/system_info.rb

@@ -1,10 +1,11 @@
 module TCellAgent
   module SystemInfo
     def self.get_language_version
-      return RUBY_VERSION
+      RUBY_VERSION
     end
+
     def self.get_language
-      return "Ruby"
+      'Ruby'
     end
   end
-end
+end

data/lib/tcell_agent/tcell_context.rb

@@ -0,0 +1,150 @@
+require 'tcell_agent/logger'
+require 'tcell_agent/utils/params'
+
+# TODO(ralba): move TCellData from instrumentation.rb here
+# and merge both models into one and drop usage of MetaData.
+# TCellData needs to be serializable in order to do that
+module TCellAgent
+  class MetaData
+    class << self
+      def from_request(request)
+        tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
+        # use uri stored in tcell_context because
+        # rails modifies original request.url
+        # to always return /404 (or whatever error code
+        # it encountered)
+        meta_event = MetaData.new(
+          tcell_context.request_method,
+          tcell_context.ip_address,
+          tcell_context.route_id,
+          tcell_context.hmac_session_id,
+          tcell_context.user_id,
+          tcell_context.transaction_id,
+          tcell_context.uri
+        )
+        meta_event.path = tcell_context.path
+
+        meta_event.set_parameter_dicts(request)
+
+        meta_event
+      end
+    end
+
+    attr_accessor :remote_address,
+                  :method,
+                  :route_id,
+                  :session_id,
+                  :user_id,
+                  :transaction_id,
+                  :location,
+                  :path,
+                  :request_content_bytes_len,
+                  :response_content_bytes_len,
+                  :response_code,
+                  :user_agent,
+                  :response_headers,
+                  :csrf_exception_name,
+                  :sql_exceptions,
+                  :database_result_sizes
+
+    attr_reader :flattened_get_dict,
+                :flattened_cookie_dict,
+                :flattened_headers_dict,
+                :flattened_post_dict,
+                :flattened_body_dict,
+                :flattened_path_parameters
+
+    def initialize(method,
+                   remote_address,
+                   route_id,
+                   session_id,
+                   user_id,
+                   transaction_id,
+                   location)
+      @method = method
+      @remote_address = remote_address
+      @route_id = route_id
+      @session_id = session_id
+      @user_id = user_id
+      @transaction_id = transaction_id
+      @location = location
+      @path = path
+
+      @flattened_get_dict = {}
+      @flattened_cookie_dict = {}
+      @flattened_path_parameters = {}
+
+      # POST/body dict can contain files, so always flatten it
+      # to get rid of those and to make this model serializable
+      @flattened_post_dict = {}
+      @flattened_body_dict = {}
+
+      @request_content_bytes_len = 0
+      @response_content_bytes_len = 0
+      @user_agent = nil
+    end
+
+    def get_dict=(value)
+      @flattened_get_dict = TCellAgent::Utils::Params.flatten(value)
+    end
+
+    def cookie_dict=(value)
+      @flattened_cookie_dict = TCellAgent::Utils::Params.flatten(value)
+    end
+
+    def headers_dict=(value)
+      headers_dict = value.select do |k, _v|
+        header_downcased = k.downcase
+        (header_downcased != 'http_cookie' && header_downcased.start_with?('http_')) ||
+          %w[content_type content_length].include?(header_downcased)
+      end
+
+      headers_dict = headers_dict.each_with_object({}) do |(k, v), memo|
+        memo[k.downcase.sub(/^http_/, '').tr('_', '-')] = v
+      end
+      @flattened_headers_dict = TCellAgent::Utils::Params.flatten(headers_dict)
+    end
+
+    def post_dict=(value)
+      @flattened_post_dict = TCellAgent::Utils::Params.flatten(value)
+    end
+
+    def path_parameters=(value)
+      @flattened_path_parameters = TCellAgent::Utils::Params.flatten(value)
+    end
+
+    def set_parameter_dicts(request)
+      self.get_dict = request.GET
+      self.cookie_dict = request.cookies
+      self.post_dict = request.POST
+      self.headers_dict = request.env
+
+      # Positions strio to the beginning of input, resetting lineno to zero.
+      # rails 4.1 seems to read the stringIO directly and so body.gets is empty
+      # this is called
+      request.body.rewind
+
+      @request_content_bytes_len = (request.content_length || 0).to_i
+      set_body_dict(
+        @request_content_bytes_len,
+        request.content_type,
+        request.body.gets
+      )
+    end
+
+    def set_body_dict(request_content_bytes_len, request_content_type, request_body)
+      @flattened_body_dict = {}
+
+      return if request_content_bytes_len > 2_000_000
+
+      return unless request_body && (request_content_type =~ %r{application/json}i)
+
+      begin
+        # don't enqueue parameter values of unknown type to avoid any serialization issues
+        @flattened_body_dict = TCellAgent::Utils::Params.flatten(JSON.parse(request_body))
+      rescue JSON::ParserError
+        TCellAgent.logger.debug('JSON body parameter parsing failed')
+      end
+    end
+  end
+end