tcell_agent 1.1.12 → 2.0.0

data/lib/tcell_agent/sensor_events/util/utils.rb

@@ -1,8 +1,6 @@
 # See the file "LICENSE" for the full license governing this code.
 
-require 'logger'
 require 'cgi'
-require 'uri'
 require 'openssl'
 
 module TCellAgent

data/lib/tcell_agent/servers/passenger.rb

@@ -1,37 +1,10 @@
 PhusionPassenger::LoaderSharedHelpers.class_eval do
-  alias_method :tcell_after_loading_app_code, :after_loading_app_code
-  def after_loading_app_code(options)
-    tcell_after_loading_app_code(options)
-
-    TCellAgent::Instrumentation.safe_block('Initial Passenger Instrumentation') do
-      # This runs in Passenger's AppPreloader (a process which is killed at some point)
-      # but it's still a good place to do the initial instrumentation so it's only sent once.
-      # Since this process doesn't receive any requests there is no need to run policy polling
-      # but we still need the event processor to send the startup events
-      original_policy_polling = TCellAgent.configuration.enable_policy_polling
-      TCellAgent.configuration.enable_policy_polling = false
-      TCellAgent.run_instrumentation('Passenger')
-      TCellAgent.configuration.enable_policy_polling = original_policy_polling
-    end
-  end
-
   alias_method :tcell_before_handling_requests, :before_handling_requests
   def before_handling_requests(forked, options)
     result_if_needed = tcell_before_handling_requests(forked, options)
 
-    TCellAgent.run_instrumentation('Passenger', false)
+    TCellAgent.thread_agent.start('Passenger')
 
     result_if_needed
   end
 end
-
-# Passenger's parent process is known as the AppPreloader, the problem is this
-# process seems to disappear at some point taking the event processor with it.
-# This will give every child process its own event manager to avoid the dependency
-# on this disappearing process
-class << TCellAgent::Agent
-  alias_method :tcell_parent_process?, :parent_process?
-  def parent_process?
-    true
-  end
-end

data/lib/tcell_agent/servers/puma.rb

@@ -6,35 +6,17 @@ if defined?(Puma.cli_config)
       Puma::Runner.class_eval do
         alias_method :original_start_server, :start_server
         def start_server
-          TCellAgent.run_instrumentation('Puma Single Mode')
+          TCellAgent.thread_agent.start('Puma Single Mode')
 
           original_start_server
         end
       end
 
     else
-
-      # Runs initial instrumentation only once on the master process
-      puma_server_starting = proc { TCellAgent.run_instrumentation('Puma Cluster Mode') }
-
-      # before_fork was added in Puma v2.13.0
-      if Puma.cli_config.options[:before_fork]
-        Puma.cli_config.options[:before_fork].push(puma_server_starting)
-      else
-        Puma.cli_config.options[:before_fork] = [puma_server_starting]
-      end
-
-      # Each puma worker still needs the agent started but no need to run
-      # initial instrumentation again
       Puma::Server.class_eval do
         alias_method :original_run, :run
         def run(background = true)
-          begin
-            TCellAgent.logger.debug('Instrumenting: Puma Cluster Mode (Worker)')
-            TCellAgent.thread_agent.start
-          rescue StandardError => e
-            TCellAgent.logger.error("Could not start thread agent. #{e.message}")
-          end
+          TCellAgent.thread_agent.start('Puma Cluster Mode (Worker)')
 
           original_run(background)
         end
@@ -48,7 +30,7 @@ if defined?(Puma.cli_config)
     Puma::Server.class_eval do
       alias_method :original_run, :run
       def run(background = true)
-        TCellAgent.run_instrumentation('Puma Cluster Mode (Worker)')
+        TCellAgent.thread_agent.start('Puma Cluster Mode (Worker)')
 
         original_run(background)
       end

data/lib/tcell_agent/servers/rails_server.rb

@@ -13,7 +13,7 @@ Rails::Server.class_eval do
       Puma::Server.class_eval do
         alias_method :original_run, :run
         def run(background = true)
-          TCellAgent.run_instrumentation('Puma Single Mode')
+          TCellAgent.thread_agent.start('Puma Single Mode')
 
           original_run(background)
         end

data/lib/tcell_agent/servers/thin.rb

@@ -1,9 +1,9 @@
 require('tcell_agent/servers/unicorn') if defined?(Unicorn::HttpServer)
-
 Thin::Server.class_eval do
   alias_method :original_start, :start
   def start
-    TCellAgent.run_instrumentation('Thin Server')
+    TCellAgent.thread_agent.start('Thin Server')
+
     original_start
   end
 end

data/lib/tcell_agent/servers/unicorn.rb

@@ -3,17 +3,22 @@ Unicorn::HttpServer.class_eval do
   # - This check also ensures that a server is running as opposed to a different command such
   # as `bundle exec rails runner User.count`.
   unless Unicorn::HttpServer::START_CTX && Unicorn::HttpServer::START_CTX[0]
-    TCellAgent.run_instrumentation('Unicorn')
+    require 'tcell_agent/rails/start_agent_after_initializers'
   end
 
+  # This only gets instrumented when preload_app is true
   # this only runs when preload_app=true because when preload_app=false
   # the gems aren't loaded early enough for tcell to override
   # the class definitions
-  alias_method :tcell_bind_new_listeners!, :bind_new_listeners!
-  def bind_new_listeners!
-    TCellAgent.run_instrumentation('Unicorn')
+  alias_method :tcell_init_worker_process, :init_worker_process
+  def init_worker_process(work)
+    start_process = tcell_init_worker_process(work)
+
+    TCellAgent.logger.debug('Instrumenting Unicorn server', 'TCellAgent::Unicorn')
 
-    tcell_bind_new_listeners!
+    TCellAgent.thread_agent.start('Unicorn Worker')
+
+    start_process
   end
 
   # This gets called when unicorn receives the HUP signal to reload its config.
@@ -21,86 +26,20 @@ Unicorn::HttpServer.class_eval do
   # or stopped accordingly
   alias_method :tcell_load_config!, :load_config!
   def load_config!
-    tcell_load_config!
-
-    TCellAgent::Instrumentation.safe_block('Reloading Tcell Config') do
-      new_config = TCellAgent::Configuration.new
-      TCellAgent.logger.debug('Reloading config')
-      TCellAgent.logger.debug(
-        "ENABLED:#{new_config.enabled}" \
-        "|ENABLE_EVENT_MANAGER:#{new_config.enable_event_manager}" \
-        "|ENABLE_EVENT_CONSUMER:#{new_config.enable_event_consumer}" \
-        "|ENABLE_POLICY_POLLING:#{new_config.enable_policy_polling}" \
-        "|ENABLE_INSTRUMENTATION:#{new_config.enable_instrumentation}" \
-        "|ENABLE_INTERCEPT_REQUESTS:#{new_config.enable_intercept_requests}"
-      )
-      old_config = TCellAgent.configuration
-
-      TCellAgent.configuration = new_config
-
-      if new_config.enabled ^ old_config.enabled
-        if new_config.enabled
-          TCellAgent.run_instrumentation('Unicorn')
-
-        else
-          TCellAgent.thread_agent.stop_event_processor
-          TCellAgent.thread_agent.stop_metrics_event_thread
-          TCellAgent.thread_agent.stop_policy_polling
-        end
-      end
-
-      if new_config.enable_event_manager ^ old_config.enable_event_manager
-        if new_config.enable_event_manager
-          TCellAgent.run_instrumentation('Unicorn Restart')
-        else
-          TCellAgent.thread_agent.stop_event_processor
-        end
-      elsif new_config.enable_event_manager
-        # Just in case
-        TCellAgent.thread_agent.ensure_event_processor_running
-      end
+    result_if_exists = tcell_load_config!
 
-      if new_config.enable_event_consumer ^ old_config.enable_event_consumer
-        if new_config.enable_event_consumer
-          TCellAgent.thread_agent.ensure_metrics_event_thread_running
-        else
-          TCellAgent.thread_agent.stop_metrics_event_thread
-        end
-      elsif new_config.enable_event_consumer
-        # Just in case
-        TCellAgent.thread_agent.ensure_metrics_event_thread_running
-      end
+    hot_reload_tcell_agent
 
-      if new_config.enable_policy_polling ^ old_config.enable_policy_polling
-        if new_config.enable_policy_polling
-          TCellAgent.thread_agent.ensure_policy_polling_running
-        else
-          TCellAgent.thread_agent.stop_policy_polling
-        end
-      elsif new_config.enable_policy_polling
-        # Just in case
-        TCellAgent.thread_agent.ensure_policy_polling_running
-      end
-    end
+    result_if_exists
   end
 
-  # this only runs when preload_app=true because when preload_app=false
-  # the gems aren't loaded early enough for tcell to override
-  # the class definitions
-  alias_method :tcell_init_worker_process, :init_worker_process
-  def init_worker_process(work)
-    start_process = tcell_init_worker_process(work)
+  def hot_reload_tcell_agent
+    TCellAgent::Instrumentation.safe_block('Reloading Tcell Config') do
+      TCellAgent.logger.debug('[TCellAgent] Reloading configuration', 'TCellAgent::Unicorn')
 
-    if TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument?
-      begin
-        TCellAgent.thread_agent.policy_polling_worker_mutex = Mutex.new
-        TCellAgent.thread_agent.policy_polling_thread = nil
-        TCellAgent.thread_agent.start
-      rescue StandardError => e
-        TCellAgent.logger.error("Could not start thread agent. #{e.message}")
-      end
-    end
+      new_config = TCellAgent::Configuration.new
 
-    start_process
+      TCellAgent.configuration = new_config
+    end
   end
 end

data/lib/tcell_agent/servers/webrick.rb

@@ -2,7 +2,7 @@ Rack::Handler::WEBrick.class_eval do
   class << self
     alias_method :original_run, :run
     def run(app, options = {})
-      TCellAgent.run_instrumentation('WEBrick')
+      TCellAgent.thread_agent.start('WEBrick')
 
       original_run(app, options)
     end

data/lib/tcell_agent/settings_reporter.rb

@@ -1,13 +1,11 @@
 require 'tcell_agent/agent'
 require 'tcell_agent/configuration'
-require 'tcell_agent/sensor_events/app_config'
+require 'tcell_agent/sensor_events/agent_setting_event'
 require 'tcell_agent/sensor_events/server_agent'
 require 'thread'
 
 module TCellAgent
-  def self.report_settings(send_startup_events)
-    return unless send_startup_events && TCellAgent.configuration.should_instrument?
-
+  def self.report_settings
     Thread.new do
       TCellAgent::Instrumentation.safe_block('Instrumenting Agent Details') do
         event = TCellAgent::SensorEvents::ServerAgentDetailsSensorEvent.new
@@ -19,27 +17,36 @@ module TCellAgent
         TCellAgent.send_event(event)
       end
 
+      TCellAgent::Instrumentation.safe_block('Instrumenting Language Info') do
+        TCellAgent.send_event(
+          TCellAgent::SensorEvents::ServerAgentDetailsLanguageEvent.new(
+            'Ruby',
+            RUBY_VERSION
+          )
+        )
+      end
+
       TCellAgent::Instrumentation.safe_block('Instrumenting Native Lib Status') do
-        require 'tcell_agent/rust/whisperer'
+        require 'tcell_agent/rust/native_agent'
 
         TCellAgent.send_event(
-          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+          TCellAgent::SensorEvents::AgentSettingEvent.new(
             'native_lib_loaded',
-            TCellAgent::Rust::Wrapper.common_lib_available?.to_s
+            TCellAgent::Rust::NativeLibrary.common_lib_available?.to_s
           )
         )
       end
 
       TCellAgent::Instrumentation.safe_block('Instrumenting Initial Config') do
         TCellAgent.send_event(
-          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+          TCellAgent::SensorEvents::AgentSettingEvent.new(
             'allow_payloads',
             (!!TCellAgent.configuration.allow_payloads).to_s # rubocop:disable Style/DoubleNegation
           )
         )
 
         TCellAgent.send_event(
-          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+          TCellAgent::SensorEvents::AgentSettingEvent.new(
             'reverse_proxy',
             (!!TCellAgent.configuration.reverse_proxy).to_s # rubocop:disable Style/DoubleNegation
           )
@@ -52,54 +59,47 @@ module TCellAgent
         TCellAgent.configuration.log_filename
 
         TCellAgent.send_event(
-          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+          TCellAgent::SensorEvents::AgentSettingEvent.new(
             'config_filename',
             TCellAgent.configuration.config_filename
           )
         )
         TCellAgent.send_event(
-          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+          TCellAgent::SensorEvents::AgentSettingEvent.new(
             'logging_directory',
             TCellAgent.configuration.agent_log_dir
           )
         )
 
         TCellAgent.send_event(
-          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+          TCellAgent::SensorEvents::AgentSettingEvent.new(
             'agent_home_directory',
             TCellAgent.configuration.agent_home_dir
           )
         )
 
-        TCellAgent.send_event(
-          TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-            'agent_home_owner',
-            TCellAgent.configuration.agent_home_owner
-          )
-        )
-
         logging_options = TCellAgent.configuration.logging_options || {}
         use_default_setting = !logging_options.key?(:enabled) && !logging_options.key?('enabled')
         if use_default_setting || logging_options[:enabled] || logging_options['enabled']
           TCellAgent.send_event(
-            TCellAgent::SensorEvents::TCellAgentSettingEvent.new('logging_enabled', 'true')
+            TCellAgent::SensorEvents::AgentSettingEvent.new('logging_enabled', 'true')
           )
 
           TCellAgent.send_event(
-            TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            TCellAgent::SensorEvents::AgentSettingEvent.new(
               'logging_level',
               logging_options[:level] || logging_options['level'] || 'INFO'
             )
           )
         else
           TCellAgent.send_event(
-            TCellAgent::SensorEvents::TCellAgentSettingEvent.new('logging_enabled', 'false')
+            TCellAgent::SensorEvents::AgentSettingEvent.new('logging_enabled', 'false')
           )
         end
 
         if TCellAgent.configuration.hmac_key
           TCellAgent.send_event(
-            TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            TCellAgent::SensorEvents::AgentSettingEvent.new(
               'hmac_key_present',
               (!!TCellAgent.configuration.hmac_key).to_s # rubocop:disable Style/DoubleNegation
             )
@@ -108,7 +108,7 @@ module TCellAgent
 
         if TCellAgent.configuration.reverse_proxy
           TCellAgent.send_event(
-            TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+            TCellAgent::SensorEvents::AgentSettingEvent.new(
               'reverse_proxy_ip_address_header',
               TCellAgent.configuration.reverse_proxy_ip_address_header
             )

data/lib/tcell_agent/sinatra.rb

@@ -14,23 +14,21 @@ module TCellAgent
       status, headers, response = original_finish
 
       TCellAgent::Instrumentation.safe_block('Setting Headers') do
-        rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
-        if rust_policies
-          policy_headers = rust_policies.get_headers(
-            request.env[TCellAgent::Instrumentation::TCELL_ID]
-          )
-          policy_headers.each do |header_info|
-            header_name = header_info['name']
-            header_value = header_info['value']
-            existing_header_value = headers[header_name]
-            headers[header_name] = if existing_header_value
-                                     "#{existing_header_value}, #{header_value}"
-                                   else
-                                     header_value
-                                   end
-          end
-          response = [status, headers, active_response]
+        headers_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HEADERS)
+        policy_headers = headers_policy.get_headers(
+          request.env[TCellAgent::Instrumentation::TCELL_ID]
+        )
+        policy_headers.each do |header_info|
+          header_name = header_info['name']
+          header_value = header_info['value']
+          existing_header_value = headers[header_name]
+          headers[header_name] = if existing_header_value
+                                   "#{existing_header_value}, #{header_value}"
+                                 else
+                                   header_value
+                                 end
         end
+        response = [status, headers, active_response]
       end
 
       [status, headers, response]

data/lib/tcell_agent/tcell_context.rb

@@ -12,26 +12,44 @@ TCELL_MAX_BODY_LENGTH = 50_000
 module TCellAgent
   class MetaData
     class << self
-      def from_request(request)
+      def for_appfirewall(request, response_content_length, response_code, response_headers)
+        meta_data = TCellAgent::MetaData.for_patches(request)
+
+        tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
+        meta_data.csrf_exception_name = tcell_context.csrf_exception_name
+        meta_data.user_agent = tcell_context.user_agent
+        meta_data.path_parameters = tcell_context.path_parameters
+        meta_data.sql_exceptions = tcell_context.sql_exceptions
+        meta_data.database_result_sizes = tcell_context.database_result_sizes
+
+        meta_data.response_content_bytes_len = response_content_length
+
+        meta_data.response_code = response_code
+        meta_data.response_headers = response_headers
+
+        meta_data
+      end
+
+      def for_patches(request)
         tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
         # use uri stored in tcell_context because
         # rails modifies original request.url
         # to always return /404 (or whatever error code
         # it encountered)
-        meta_event = MetaData.new(
+        meta_data = MetaData.new(
           tcell_context.request_method,
-          tcell_context.ip_address,
+          tcell_context.remote_address,
           tcell_context.route_id,
           tcell_context.hmac_session_id,
           tcell_context.user_id,
           tcell_context.transaction_id,
           tcell_context.uri
         )
-        meta_event.path = tcell_context.path
+        meta_data.path = tcell_context.path
 
-        meta_event.set_parameter_dicts(request)
+        meta_data.set_parameter_dicts(request)
 
-        meta_event
+        meta_data
       end
     end
 
@@ -119,6 +137,12 @@ module TCellAgent
       @flattened_path_parameters = TCellAgent::Utils::Params.flatten(value)
     end
 
+    def charset
+      Rack::MediaType.params(@content_type)['charset'] || Encoding.default_external
+    rescue StandardError
+      Encoding.default_external
+    end
+
     def get_raw_post_data(request)
       content_length = request.content_length.to_i if request.content_length
       if !content_length.nil? && content_length > TCELL_MAX_BODY_LENGTH || request.content_type.nil?
@@ -128,30 +152,32 @@ module TCellAgent
       # Positions strio to the beginning of input, resetting lineno to zero.
       # rails 4.1 seems to read the stringIO directly and so body.gets is empty
       # this is called
+
       body = request.body
       body.rewind if body.respond_to?(:rewind)
       raw_post_data = body.read(request.content_length.to_i) if request.content_length
       body.rewind if body.respond_to?(:rewind)
-      raw_post_data
+
+      raw_post_data.force_encoding(charset) unless raw_post_data.nil?
     end
 
     def set_parameter_dicts(request)
+      @flattened_body_dict = {} # deprecated
+      @content_type = request.content_type || ''
+      @raw_request_body = get_raw_post_data(request) unless @content_type.start_with?('application/octet-stream',
+                                                                                      'multipart/form-data')
+      @request_content_bytes_len = (request.content_length || 0).to_i
+
       self.get_dict = request.GET
       self.cookie_dict = request.cookies
 
-      self.post_dict = if !(request.content_type =~ %r{application/json}i).nil? ||
-                          !(request.content_type =~ %r{application/xml}i).nil?
+      self.post_dict = if @content_type.start_with?('application/json', 'application/xml')
                          {}
                        else
                          request.POST
                        end
 
       self.headers_dict = request.env
-
-      @flattened_body_dict = {} # deprecated
-      @content_type = request.content_type
-      @raw_request_body = get_raw_post_data(request)
-      @request_content_bytes_len = (request.content_length || 0).to_i
     end
   end
 end