tcell_agent 2.2.1 → 2.5.0

data/lib/tcell_agent/rails/dlp/process_request.rb

@@ -37,6 +37,7 @@ module TCellAgent
       dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
       tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
       return unless tcell_context && dataex_policy && dataex_policy.actions_for_form_parameter?
+
       for_params(request) do |_method, param_name, param_value|
         actions = dataex_policy.get_actions_for_form_parameter(param_name, tcell_context.route_id)
         if actions
@@ -51,11 +52,13 @@ module TCellAgent
       dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
       tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
       return unless tcell_context && dataex_policy && dataex_policy.actions_for_headers?
+
       headers = request.env.select { |k, _v| k.start_with? 'HTTP_' }
       headers.each do |header_name, header_value|
         header_name = header_name.sub(/^HTTP_/, '').tr('_', '-')
         actions = dataex_policy.get_actions_for_header(header_name)
         next unless actions
+
         actions.each do |action|
           tcell_context.add_filter_for_header_value(header_value, action, header_name)
         end
@@ -66,9 +69,11 @@ module TCellAgent
       dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
       tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
       return unless tcell_context && dataex_policy && dataex_policy.actions_for_cookie?
+
       request.cookies.each do |cookie_name, cookie_value|
         actions = dataex_policy.get_actions_for_cookie(cookie_name)
         next unless actions
+
         actions.each do |action|
           tcell_context.add_filter_for_cookie_value(cookie_value, action, cookie_name)
         end

data/lib/tcell_agent/rails/dlp.rb

@@ -80,6 +80,7 @@ module TCellAgent
               normalized_column_names[namespaced_column_name] = column_name
 
               next unless column_name && (!namespace || namespace == table_name)
+
               rules = dlp_policy.get_actions_for_table(
                 database_name,
                 '*',
@@ -190,6 +191,7 @@ module TCellAgent
             results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |record|
               column_name_to_rules.each do |column_name, rules|
                 next unless rules
+
                 rules.each do |rule|
                   tcell_context.add_response_db_filter(
                     record[column_name.to_sym],
@@ -301,31 +303,29 @@ module TCellAgent
       def log_enforce(tcell_context, sanitize_string)
         if TCellAgent.configuration.should_instrument? &&
            TCellAgent.configuration.should_intercept_requests?
-          if tcell_context && tcell_context.session_id
-            session_id_actions = get_actions_for_session_id
-            if session_id_actions
-              send_event = false
-              sanitize_string.gsub!(tcell_context.session_id) do |m|
-                if session_id_actions.log_redact
-                  send_event = true
-                  m = '[session_id]'
-                elsif session_id_actions.log_hash
-                  send_event = true
-                  m = '[hash]'
-                elsif session_id_actions.log_event
-                  send_event = true
-                end
-                m
-              end
-              if send_event
-                TCellAgent.send_event(
-                  TCellAgent::SensorEvents::DlpEvent.new(
-                    tcell_context.route_id,
-                    tcell_context.uri,
-                    TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG
-                  ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
-                )
+          session_id_actions = get_actions_for_session_id
+          if tcell_context && tcell_context.session_id && session_id_actions
+            send_event = false
+            sanitize_string.gsub!(tcell_context.session_id) do |m|
+              if session_id_actions.log_redact
+                send_event = true
+                m = '[session_id]'
+              elsif session_id_actions.log_hash
+                send_event = true
+                m = '[hash]'
+              elsif session_id_actions.log_event
+                send_event = true
               end
+              m
+            end
+            if send_event
+              TCellAgent.send_event(
+                TCellAgent::SensorEvents::DlpEvent.new(
+                  tcell_context.route_id,
+                  tcell_context.uri,
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG
+                ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
+              )
             end
           end
         end
@@ -336,32 +336,32 @@ module TCellAgent
       def response_body_enforce(tcell_context, sanitize_string)
         if TCellAgent.configuration.should_instrument? &&
            TCellAgent.configuration.should_intercept_requests?
-          if tcell_context && tcell_context.session_id
-            session_id_actions = get_actions_for_session_id
-            if session_id_actions
-              send_event = false
-              sanitize_string.gsub!(tcell_context.session_id) do |m|
-                if session_id_actions.body_redact
-                  # m = "[session_id]"
-                  send_event = true
-                elsif session_id_actions.body_hash
-                  # m = "[hash]"
-                  send_event = true
-                elsif session_id_actions.body_event
-                  send_event = true
-                end
-                m
+          session_id_actions = get_actions_for_session_id
+          if tcell_context && tcell_context.session_id && session_id_actions
+            send_event = false
+            sanitize_string.gsub!(tcell_context.session_id) do |m|
+              # rubocop:disable Lint/DuplicateBranch
+              if session_id_actions.body_redact
+                # m = "[session_id]"
+                send_event = true
+              elsif session_id_actions.body_hash
+                # m = "[hash]"
+                send_event = true
+              elsif session_id_actions.body_event
+                send_event = true
               end
+              # rubocop:enable Lint/DuplicateBranch
+              m
             end
-            if send_event
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::DlpEvent.new(
-                  tcell_context.route_id,
-                  tcell_context.uri,
-                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY
-                ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
-              )
-            end
+          end
+          if send_event
+            TCellAgent.send_event(
+              TCellAgent::SensorEvents::DlpEvent.new(
+                tcell_context.route_id,
+                tcell_context.uri,
+                TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY
+              ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
+            )
           end
         end
 

data/lib/tcell_agent/rails/dlp_handler.rb

@@ -39,19 +39,18 @@ module TCellAgent
 
           TCellAgent::Instrumentation.safe_block('DLP Handler get handler and context') do
             if TCellAgent.configuration.should_instrument? &&
-               TCellAgent.configuration.should_intercept_requests?
+               TCellAgent.configuration.should_intercept_requests? &&
+               TCellAgent::Utils::Rails.processable_response?(response_headers)
 
               # do all this work so that dlp doesn't run at all unless it's on and there
               # are rules to run
-              if TCellAgent::Utils::Rails.processable_response?(response_headers)
-                dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
-                if dlp_policy && dlp_policy.get_actions_for_session_id
-                  tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
-                  if tcell_context && tcell_context.session_id
-                    dlp_handler = proc { |tc, resp|
-                      handle_dlp!(tc, resp)
-                    }
-                  end
+              dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
+              if dlp_policy && dlp_policy.get_actions_for_session_id
+                tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
+                if tcell_context && tcell_context.session_id
+                  dlp_handler = proc { |tc, resp|
+                    handle_dlp!(tc, resp)
+                  }
                 end
               end
             end

data/lib/tcell_agent/rails/js_agent_insert.rb

@@ -4,8 +4,7 @@ module TCellAgent
   module Instrumentation
     module Rails
       module JSAgent
-        HEAD_SEARCH_REGEX = /<head>/
-
+        HEAD_SEARCH_REGEX = Regexp.new('(<head>|<head( |\n).*?>)', Regexp::IGNORECASE)
         def self.insert_now(js_agent_handler, script_insert, rack_body, content_length)
           TCellAgent::Instrumentation.safe_block('Handling JSAgent Insert Now') do
             if js_agent_handler
@@ -32,7 +31,7 @@ module TCellAgent
           TCellAgent::Instrumentation.safe_block('Handling JSAgent insert') do
             new_response = response.sub(
               TCellAgent::Instrumentation::Rails::JSAgent::HEAD_SEARCH_REGEX,
-              "<head>#{script_insert}"
+              "\\1#{script_insert}"
             )
           end
 

data/lib/tcell_agent/rails/middleware/context_middleware.rb

@@ -26,7 +26,8 @@ module TCellAgent
                 env[TCellAgent::Instrumentation::TCELL_ID].path = request.path
                 env[TCellAgent::Instrumentation::TCELL_ID].user_agent = request.user_agent
                 env[TCellAgent::Instrumentation::TCELL_ID].referrer = request.referrer
-                env[TCellAgent::Instrumentation::TCELL_ID].remote_address = TCellAgent::Utils::Rails.better_ip(request)
+                env[TCellAgent::Instrumentation::TCELL_ID].remote_address = request.ip
+                env[TCellAgent::Instrumentation::TCELL_ID].reverse_proxy_header_value = TCellAgent::Utils::Rails.reverse_proxy_header(request)
                 if request.request_method
                   env[TCellAgent::Instrumentation::TCELL_ID].request_method = request.request_method
                 end

data/lib/tcell_agent/rails/middleware/global_middleware.rb

@@ -25,8 +25,6 @@ module TCellAgent
             if TCellAgent.configuration.should_intercept_requests?
               request = Rack::Request.new(env)
 
-              request['init'] = true
-
               TCellAgent::Instrumentation.safe_block('Setting session_id & user_id') do
                 if request.session
                   env[TCellAgent::Instrumentation::TCELL_ID].session_id =
@@ -44,9 +42,7 @@ module TCellAgent
               end
             end
 
-            response = @app.call(env)
-
-            response
+            @app.call(env)
           end
         end
       end

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -39,6 +39,7 @@ module TCellAgent
             TCellAgent::Instrumentation.safe_block('Handling headers') do
               headers_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HEADERS)
               policy_headers = headers_policy.get_headers(
+                headers['Content-Type'],
                 request.env[TCellAgent::Instrumentation::TCELL_ID]
               )
               policy_headers.each do |header_info|

data/lib/tcell_agent/rails/routes/grape.rb

@@ -7,8 +7,9 @@ module TCellAgent
         begin
           return route.app < Grape::API if ::Rails::VERSION::MAJOR == 4 &&
                                            ::Rails::VERSION::MINOR < 2
+
           return route.app.app < Grape::API
-        rescue StandardError # rubocop:disable Lint/HandleExceptions
+        rescue StandardError
           # do nothing
         end
       end

data/lib/tcell_agent/rails/settings_reporter.rb

@@ -6,12 +6,9 @@ require 'tcell_agent/sensor_events/server_agent'
 module TCellAgent
   module Instrumentation
     module Rails
-      def self.send_framework_info
-        TCellAgent.send_event(
-          TCellAgent::SensorEvents::ServerAgentAppFrameworkEvent.new(
-            'Rails', ::Rails.version
-          )
-        )
+      def self.framework_details
+        { 'app_framework' => 'Rails',
+          'app_framework_version' => ::Rails.version }
       end
 
       def self.send_settings

data/lib/tcell_agent/rails/tcell_body_proxy.rb

@@ -53,18 +53,16 @@ module TCellAgent
           @body.respond_to?(method_name, include_all)
         end
 
-        def method_missing(method_name, *args, &block) # rubocop:disable Style/MethodMissing
+        def method_missing(method_name, *args, &block)
           @body.__send__(method_name, *args, &block)
         end
 
         def process_body(body)
           TCellAgent::Instrumentation.safe_block('Processing tcell body proxy body') do
             chunked_response_match = nil
-            if body.class.name == 'String'
-              if body =~ /^([[:xdigit:]]+)(;.+)?\r\n/
-                chunked_response_match = Regexp.last_match(1)
-                @content_length += chunked_response_match.to_i(16)
-              end
+            if body.class.name == 'String' && body =~ /^([[:xdigit:]]+)(;.+)?\r\n/
+              chunked_response_match = Regexp.last_match(1)
+              @content_length += chunked_response_match.to_i(16)
             end
 
             new_body = body

data/lib/tcell_agent/routes/table.rb

@@ -2,6 +2,7 @@ module TCellAgent
   module Routes
     class FieldEndpoint
       attr_accessor :discovered
+
       def initialize
         super()
         @discovered = false
@@ -11,6 +12,7 @@ module TCellAgent
     class RouteEndpoint
       attr_accessor :database
       attr_accessor :database_queries_discovered
+
       def initialize
         @database_queries_discovered = {}
         @database = Hash.new do |d_h, d_k| # Database
@@ -27,6 +29,7 @@ module TCellAgent
 
     class RouteTable
       attr_accessor :routes
+
       def initialize
         @routes = Hash.new { |h, k| h[k] = RouteEndpoint.new }
       end

data/lib/tcell_agent/rust/agent_config.rb

@@ -19,8 +19,23 @@ module TCellAgent
           self['overrides'] = overrides
         else
           self['overrides'] = { 'applications' => [{ :enable_json_body_inspection => true }],
-                                'config_file_dir' => configuration.get_config_file_dir }
+                                'config_file_path' => configuration.get_config_file_path }
         end
+
+        set_agent_details
+      end
+
+      def set_agent_details
+        framework_details = if defined?(Rails)
+                              TCellAgent::Instrumentation::Rails.framework_details
+                            else
+                              {}
+                            end
+
+        self['agent_details'] = { 'language' => 'Ruby',
+                                  'language_version' => RUBY_VERSION,
+                                  'app_framework' => framework_details['app_framework'],
+                                  'app_framework_version' => framework_details['app_framework_version'] }
       end
     end
 
@@ -40,7 +55,7 @@ module TCellAgent
 
         self['api_url'] = configuration.tcell_api_url
         self['applications'] = [Models.clean_nils(applications)]
-        self['config_file_dir'] = configuration.get_config_file_dir
+        self['config_file_path'] = configuration.get_config_file_path
         self['disabled_instrumentation'] = configuration.disabled_instrumentation
         self['enabled'] = configuration.enabled
         self['host_identifier'] = configuration.host_identifier
@@ -53,6 +68,9 @@ module TCellAgent
         self['log_enabled'] = configuration.logging_options[:enabled]
         self['log_filename'] = configuration.logging_options[:log_filename]
         self['log_level'] = configuration.logging_options[:level]
+        self['proxy_url'] = configuration.proxy_url
+        self['proxy_username'] = configuration.proxy_username
+        self['proxy_password'] = configuration.proxy_password
         self['update_policy'] = configuration.fetch_policies_from_tcell
       end
     end

data/lib/tcell_agent/rust/native_agent.rb

@@ -9,55 +9,6 @@ require 'tcell_agent/utils/headers'
 module TCellAgent
   module Rust
     class NativeAgent # rubocop:disable Metrics/ClassLength
-      def self.test_event_sender(events)
-        config = TCellAgent.configuration
-        event_sender = {
-          :uuid =>            config.uuid,
-          :hostname =>        config.host_identifier,
-          :agent_type =>      'Ruby',
-          :agent_version =>   TCellAgent::VERSION,
-          :app_id =>          config.app_id,
-          :api_key =>         config.api_key,
-          :tcell_input_url => config.tcell_input_url,
-          :events =>          events
-        }
-        event_sender_pointer = FFI::MemoryPointer.from_string(
-          JSON.dump(event_sender)
-        )
-
-        buf = FFI::MemoryPointer.new(:uint8, 1024 * 8)
-        # config_pointer.size - 1: strips null terminator
-        result_size = TCellAgent::Rust::NativeLibrary.test_event_sender(
-          event_sender_pointer, event_sender_pointer.size - 1, buf, buf.size
-        )
-
-        response = NativeAgentResponse.new('test_event_sender', buf, result_size)
-
-        response.errors
-      end
-
-      def self.test_policies
-        config = TCellAgent.configuration
-        policies_info = {
-          :app_id =>        config.app_id,
-          :api_key =>       config.api_key,
-          :tcell_api_url => config.tcell_api_url
-        }
-        policies_info_pointer = FFI::MemoryPointer.from_string(
-          JSON.dump(policies_info)
-        )
-
-        buf = FFI::MemoryPointer.new(:uint8, 1024 * 8)
-        # config_pointer.size - 1: strips null terminator
-        result_size = TCellAgent::Rust::NativeLibrary.test_policies(
-          policies_info_pointer, policies_info_pointer.size - 1, buf, buf.size
-        )
-
-        response = NativeAgentResponse.new('test_event_sender', buf, result_size)
-
-        response.errors
-      end
-
       def self.test_agent(config)
         agent_config = TCellAgent::Rust::AgentConfig.new(config)
 
@@ -133,6 +84,7 @@ module TCellAgent
           :headers => header_params,
           :cookies => cookie_params,
           :path_params => path_params,
+          :reverse_proxy_header_value => appsensor_meta.reverse_proxy_header_value,
           :remote_address => appsensor_meta.remote_address,
           :full_uri => appsensor_meta.location,
           :session_id => appsensor_meta.session_id,
@@ -143,7 +95,6 @@ module TCellAgent
           :content_type => appsensor_meta.content_type,
           :request_body => appsensor_meta.raw_request_body
         }
-
         request_response_json[:sql_exceptions] = appsensor_meta.sql_exceptions if appsensor_meta.sql_exceptions
         request_response_json[:database_result_sizes] = appsensor_meta.database_result_sizes if appsensor_meta.database_result_sizes
 
@@ -184,13 +135,15 @@ module TCellAgent
           :method => appsensor_meta.method,
           :path => appsensor_meta.path,
           :remote_address => appsensor_meta.remote_address,
+          :reverse_proxy_header_value => appsensor_meta.reverse_proxy_header_value,
           :request_bytes_length => appsensor_meta.request_content_bytes_len,
           :query_params => query_params,
-          :post_params =>  post_params,
+          :post_params => post_params,
           :headers => header_params,
           :cookies => cookie_params,
           :content_type => appsensor_meta.content_type,
-          :full_uri => appsensor_meta.location
+          :full_uri => appsensor_meta.location,
+          :request_body => appsensor_meta.raw_request_body
         }
 
         patches_request_pointer = FFI::MemoryPointer.from_string(
@@ -213,6 +166,37 @@ module TCellAgent
         response.response
       end
 
+      def apply_suspicious_quick_check(appsensor_meta)
+        return {} unless appsensor_meta
+
+        sus_quick_check_request_json = {
+          :reverse_proxy_header_value => appsensor_meta.reverse_proxy_header_value,
+          :method => appsensor_meta.method,
+          :path => appsensor_meta.path,
+          :full_uri => appsensor_meta.location,
+          :request_bytes_length => appsensor_meta.request_content_bytes_len
+        }
+
+        if appsensor_meta.reverse_proxy_header_value.nil?
+          sus_quick_check_request_json.merge(
+            {
+              :client_ip_override => appsensor_meta.remote_address
+            }
+          )
+        end
+
+        sus_quick_check_request_ptr = FFI::MemoryPointer.from_string(
+          JSON.dump(sus_quick_check_request_json)
+        )
+
+        # sus_quick_check_request_ptr.size - 1: strips null terminator
+        TCellAgent::Rust::NativeLibrary.suspicious_quick_check_apply(
+          FFI::Pointer.new(@agent_ptr),
+          sus_quick_check_request_ptr,
+          sus_quick_check_request_ptr.size - 1
+        )
+      end
+
       def apply_cmdi(command, tcell_context)
         return unless TCellAgent::Utils::Strings.present?(command)
 
@@ -220,6 +204,7 @@ module TCellAgent
           :command => command,
           :method => tcell_context.request_method,
           :path => tcell_context.path,
+          :reverse_proxy_header_value => tcell_context.reverse_proxy_header_value,
           :remote_address => tcell_context.remote_address,
           :route_id => tcell_context.route_id,
           :session_id => tcell_context.session_id,
@@ -247,10 +232,11 @@ module TCellAgent
         response.response
       end
 
-      def get_headers(tcell_context)
+      def get_headers(content_type, tcell_context)
         return unless tcell_context
 
         headers_request = {
+          :content_type => content_type,
           :method => tcell_context.request_method,
           :path => tcell_context.path,
           :route_id => tcell_context.route_id.to_s,
@@ -288,8 +274,9 @@ module TCellAgent
           :status_code => status_code,
           :method => tcell_context.request_method,
           :path => tcell_context.path,
+          :reverse_proxy_header_value => tcell_context.reverse_proxy_header_value,
           :remote_addr => tcell_context.remote_address,
-          :full_uri => tcell_context.fullpath,
+          :full_uri => tcell_context.uri,
           :route_id => tcell_context.route_id,
           :session_id => tcell_context.session_id,
           :user_id => tcell_context.user_id
@@ -353,11 +340,12 @@ module TCellAgent
           :event_name => event_name,
           :user_id => user_id,
           :user_agent => tcell_context.user_agent,
+          :reverse_proxy_header_value => tcell_context.reverse_proxy_header_value,
           :remote_address => tcell_context.remote_address,
           :header_keys => header_keys,
-          :passsword => password,
+          :password => password,
           :session_id => tcell_context.session_id,
-          :full_uri => tcell_context.fullpath,
+          :full_uri => tcell_context.uri,
           :referrer => tcell_context.referrer,
           :user_valid => user_valid
         }
@@ -395,11 +383,14 @@ module TCellAgent
         if tcell_context
           file_access_info = file_access_info.merge(
             {
-              :full_uri => tcell_context.fullpath,
+              :full_uri => tcell_context.uri,
+              :reverse_proxy_header_value => tcell_context.reverse_proxy_header_value,
               :remote_address => tcell_context.remote_address,
               :route_id => tcell_context.route_id,
               :session_id => tcell_context.session_id,
-              :user_id => tcell_context.user_id
+              :user_id => tcell_context.user_id,
+              :method => tcell_context.request_method,
+              :request_path => tcell_context.path
             }
           )
         end
@@ -471,6 +462,7 @@ module TCellAgent
           :session_id => tcell_context && tcell_context.session_id,
           :user_id => tcell_context && tcell_context.user_id,
           :user_agent => tcell_context && tcell_context.user_agent,
+          :reverse_proxy_header_value => tcell_context.reverse_proxy_header_value,
           :remote_address => tcell_context && tcell_context.remote_address
         }
         message_pointer = FFI::MemoryPointer.from_string(
@@ -522,7 +514,7 @@ module TCellAgent
         end
       end
 
-      # Note: for tests
+      # NOTE: for tests
       def update_policies(policies)
         return {} unless TCellAgent::Utils::Strings.present?(policies)