tcell_agent 0.4.0 → 1.0.0

data/lib/tcell_agent/rails/auth/devise.rb

@@ -1,26 +1,37 @@
 if TCellAgent.configuration.should_instrument_devise? && defined?(Devise)
   module TCellAgent
-
     require 'base64'
     require 'tcell_agent/agent'
     require 'tcell_agent/sensor_events/login_fraud'
-    require 'tcell_agent/policies/appsensor_policy'
 
     module DeviseInstrumentation
       module TCellFailureAppRespond
         def respond
-          TCellAgent::Instrumentation.safe_block("Devise Failure App Respond") do
-            if (TCellAgent.configuration.enabled && TCellAgent.configuration.should_intercept_requests?)
+          TCellAgent::Instrumentation.safe_block('Devise Failure App Respond') do
+            if TCellAgent.configuration.enabled &&
+               TCellAgent.configuration.should_intercept_requests?
               tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
               if tcell_data
-                # in the case of http auth, the user_id is set in
+                # in the case of http auth, user_id is set in
                 # Devise::Strategies::Authenticatable.valid_for_http_auth?
                 user_id = tcell_data.user_id
-                user_id = _get_tcell_username unless user_id
+                user_id ||= _get_tcell_username
+
+                # in the case of http auth, password is set in
+                # Devise::Strategies::Authenticatable.valid_for_http_auth?
+                password = tcell_data.password
+                password ||= _get_tcell_password
 
                 login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
-                if (login_fraud_policy && login_fraud_policy.login_failed_enabled)
-                  TCellAgent.send_event(TCellAgent::SensorEvents::LoginFailure.new(request.env, tcell_data, user_id))
+                if login_fraud_policy && login_fraud_policy.login_failed_enabled
+                  TCellAgent.send_event(
+                    TCellAgent::SensorEvents::LoginFailure.new(
+                      request.env,
+                      tcell_data,
+                      user_id,
+                      password
+                    )
+                  )
                 end
               end
 
@@ -31,19 +42,28 @@ if TCellAgent.configuration.should_instrument_devise? && defined?(Devise)
         end
 
         def _get_tcell_username
-          _tcell_username = nil
-          TCellAgent::Instrumentation.safe_block("Devise Get TCell Username") {
+          tcell_username = nil
+          TCellAgent::Instrumentation.safe_block('Devise Get TCell Username') do
             keys = scope_class.authentication_keys.dup
-            user_params = request.POST.fetch("user",{})
+            user_params = request.POST.fetch('user', {})
             keys.each do |key|
               next_usename = user_params.fetch(key, nil)
               if next_usename
-                _tcell_username ||= ""
-                _tcell_username += next_usename
+                tcell_username ||= ''
+                tcell_username += next_usename
               end
             end
-          }
-          _tcell_username
+          end
+          tcell_username
+        end
+
+        def _get_tcell_password
+          tcell_password = nil
+          TCellAgent::Instrumentation.safe_block('Devise Get TCell Password') do
+            user_params = request.POST.fetch('user', {})
+            tcell_password = user_params['password']
+          end
+          tcell_password
         end
       end
     end
@@ -56,11 +76,13 @@ if TCellAgent.configuration.should_instrument_devise? && defined?(Devise)
       def valid_for_http_auth?
         is_valid = tcell_valid_for_http_auth?
 
-        TCellAgent::Instrumentation.safe_block("Devise set username for http basic auth") do
+        TCellAgent::Instrumentation.safe_block('Devise set username for http basic auth') do
           tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
           if http_auth_hash && tcell_data
             username = http_auth_hash[http_authentication_key]
+            password = http_auth_hash[:password]
             tcell_data.user_id = username if username && !tcell_data.user_id
+            tcell_data.password = password if password && !tcell_data.password
           end
         end
 
@@ -71,22 +93,23 @@ if TCellAgent.configuration.should_instrument_devise? && defined?(Devise)
       def validate(resource, &block)
         is_valid = tcell_validate(resource, &block)
 
-        TCellAgent::Instrumentation.safe_block("Devise Authenticatable Validate") do
-          if (is_valid && TCellAgent.configuration.enabled && TCellAgent.configuration.should_intercept_requests?)
+        TCellAgent::Instrumentation.safe_block('Devise Authenticatable Validate') do
+          if is_valid && TCellAgent.configuration.enabled &&
+             TCellAgent.configuration.should_intercept_requests?
             username = nil
             (authentication_keys || []).each do |auth_key|
               attr = authentication_hash[auth_key]
               if attr
-                username ||= ""
+                username ||= ''
                 username += attr
               end
             end
 
             login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
-            if (login_fraud_policy && login_fraud_policy.login_success_enabled)
+            if login_fraud_policy && login_fraud_policy.login_success_enabled
               tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
               if tcell_data
-                TCellAgent.send_event(TCellAgent::SensorEvents::LoginSuccess.new(request.env, tcell_data, username))
+                TCellAgent.send_event(TCellAgent::SensorEvents::LoginSuccess.new(request.env, tcell_data, username, nil))
               end
             end
           end
@@ -94,8 +117,6 @@ if TCellAgent.configuration.should_instrument_devise? && defined?(Devise)
 
         is_valid
       end
-
     end
-
   end
 end

data/lib/tcell_agent/rails/auth/doorkeeper.rb

@@ -3,30 +3,42 @@ if TCellAgent.configuration.should_instrument_doorkeeper?
   if defined?(Doorkeeper)
     require 'tcell_agent/agent'
     require 'tcell_agent/sensor_events/login_fraud'
-    require 'tcell_agent/policies/appsensor_policy'
-    require 'tcell_agent/sensor_events/login_fraud'
 
     module TCellAgent
       module DoorkeeperInstrumentation
-
         Doorkeeper::TokensController.class_eval do
           alias_method :tcell_authorize_response, :authorize_response
           def authorize_response
             result = tcell_authorize_response
 
-            TCellAgent::Instrumentation.safe_block("Doorkeeper Token Authorize") do
-              if (TCellAgent.configuration.enabled && TCellAgent.configuration.should_intercept_requests?)
+            TCellAgent::Instrumentation.safe_block('Doorkeeper Token Authorize') do
+              if TCellAgent.configuration.enabled &&
+                 TCellAgent.configuration.should_intercept_requests?
                 login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
-                if (login_fraud_policy && login_fraud_policy.enabled && login_fraud_policy.login_failed_enabled)
+                if login_fraud_policy &&
+                   login_fraud_policy.enabled &&
+                   login_fraud_policy.login_failed_enabled
                   tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
+
                   if tcell_data
+                    password = nil
                     if result.is_a?(Doorkeeper::OAuth::TokenResponse)
                       TCellAgent.send_event(
-                        TCellAgent::SensorEvents::LoginSuccess.new(request.env, tcell_data, result.token.resource_owner_id)
+                        TCellAgent::SensorEvents::LoginSuccess.new(
+                          request.env,
+                          tcell_data,
+                          result.token.resource_owner_id,
+                          password
+                        )
                       )
                     elsif result.is_a?(Doorkeeper::OAuth::ErrorResponse)
                       TCellAgent.send_event(
-                        TCellAgent::SensorEvents::LoginFailure.new(request.env, tcell_data, request.POST['client_id'])
+                        TCellAgent::SensorEvents::LoginFailure.new(
+                          request.env,
+                          tcell_data,
+                          request.POST['client_id'],
+                          password
+                        )
                       )
                     end
 
@@ -43,15 +55,24 @@ if TCellAgent.configuration.should_instrument_doorkeeper?
           def new
             super if defined?(super)
 
-            TCellAgent::Instrumentation.safe_block("Doorkeeper Token Authorize") do
-              if (TCellAgent.configuration.enabled && TCellAgent.configuration.should_intercept_requests?)
+            TCellAgent::Instrumentation.safe_block('Doorkeeper Token Authorize') do
+              if TCellAgent.configuration.enabled &&
+                 TCellAgent.configuration.should_intercept_requests?
                 if pre_auth.error
                   login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
-                  if (login_fraud_policy && login_fraud_policy.enabled && login_fraud_policy.login_failed_enabled)
+                  if login_fraud_policy &&
+                     login_fraud_policy.enabled &&
+                     login_fraud_policy.login_failed_enabled
                     tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
                     if tcell_data && pre_auth.error
+                      password = nil
                       TCellAgent.send_event(
-                        TCellAgent::SensorEvents::LoginFailure.new(request.env, tcell_data, current_resource_owner.id)
+                        TCellAgent::SensorEvents::LoginFailure.new(
+                          request.env,
+                          tcell_data,
+                          current_resource_owner.id,
+                          password
+                        )
                       )
                     end
                   end
@@ -64,9 +85,6 @@ if TCellAgent.configuration.should_instrument_doorkeeper?
         # prepend is ruby 2+ feature
         Doorkeeper::AuthorizationsController.send(:prepend, TCellAuthorizationsNew)
       end
-
     end
-
   end
-
 end

data/lib/tcell_agent/rails/better_ip.rb

@@ -11,7 +11,7 @@ module TCellAgent
           TCellAgent::Instrumentation.safe_block("Extracting reverse proxy IP") do
             reverse_proxy_header = TCellAgent.configuration.reverse_proxy_ip_address_header
             if TCellAgent::Utils::Strings.present?(reverse_proxy_header)
-              reverse_proxy_header = "HTTP_" + reverse_proxy_header.upcase().gsub('-','_')
+              reverse_proxy_header = "HTTP_" + reverse_proxy_header.upcase().tr('-','_')
             else
               reverse_proxy_header = "HTTP_X_FORWARDED_FOR"
             end

data/lib/tcell_agent/rails/csrf_exception.rb

@@ -5,8 +5,8 @@ module TCellAgent
   module CsrfExceptionReporter
     def handle_unverified_request
       TCellAgent::Instrumentation.safe_block("AppSensor CSRF Exception processing") do
-        appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
-        if appsensor_policy
+        rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+        if rust_policies && rust_policies.appfirewall_enabled
           tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
           if tcell_data
             tcell_data.csrf_exception_name = ActionController::InvalidAuthenticityToken.name

data/lib/tcell_agent/rails/dlp/process_request.rb

@@ -59,7 +59,7 @@ module TCellAgent
       if tcell_context && dataex_policy && dataex_policy.has_actions_for_headers?
         headers = request.env.select {|k,v| k.start_with? 'HTTP_'}
         headers.each { |header_name, header_value|
-          header_name = header_name.sub(/^HTTP_/, '').gsub('_','-')
+          header_name = header_name.sub(/^HTTP_/, '').tr('_','-')
           actions = dataex_policy.get_actions_for_header(header_name)
           if actions
             actions.each { |action|

data/lib/tcell_agent/rails/dlp.rb

@@ -116,9 +116,9 @@ module TCellAgent
           def translate_exception(e, message)
             result = tcell_translate_exception(e, message)
 
-            TCellAgent::Instrumentation.safe_block("Call AppSensorPolicy.sql_exception_detected") do
-              appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
-              if appsensor_policy
+            TCellAgent::Instrumentation.safe_block("Set sql_exception_detected in meta") do
+              rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+              if rust_policies && rust_policies.appfirewall_enabled
                 request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
                 tcell_data = request_env[TCellAgent::Instrumentation::TCELL_ID]
                 if tcell_data && result.is_a?(ActiveRecord::StatementInvalid)
@@ -189,7 +189,7 @@ module TCellAgent
       def log_enforce(tcell_context, sanitize_string)
         if TCellAgent.configuration.enabled &&
             TCellAgent.configuration.should_instrument? &&
-            TCellAgent.configuration.should_intercept_requests? &&
+            TCellAgent.configuration.should_intercept_requests?
           if (tcell_context && tcell_context.session_id)
             session_id_actions = self.get_actions_for_session_id
             if session_id_actions
@@ -225,7 +225,7 @@ module TCellAgent
       def response_body_enforce(tcell_context, sanitize_string)
         if TCellAgent.configuration.enabled &&
             TCellAgent.configuration.should_instrument? &&
-            TCellAgent.configuration.should_intercept_requests? &&
+            TCellAgent.configuration.should_intercept_requests?
           if (tcell_context && tcell_context.session_id)
             session_id_actions = self.get_actions_for_session_id
             if session_id_actions
@@ -279,7 +279,7 @@ class Logger
         end
       end
 
-      TCellAgent::Instrumentation.safe_block_no_log("Handling JSAgent add") {
+      TCellAgent::Instrumentation.safe_block_no_log("Handling DLP log message filtering") {
         dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
         request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
         if message && dlp_policy && request_env

data/lib/tcell_agent/rails/dlp_handler.rb

@@ -50,7 +50,7 @@ module TCellAgent
                 if dlp_policy && dlp_policy.get_actions_for_session_id
                   tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
                   if tcell_context && tcell_context.session_id
-                    dlp_handler = Proc.new { |tc, resp|
+                    dlp_handler = proc { |tc, resp|
                       self.handle_dlp!(tc, resp)
                     }
                   end

data/lib/tcell_agent/rails/js_agent_insert.rb

@@ -55,7 +55,7 @@ module TCellAgent
                 script_insert = "<script src=\"#{script_tag_policy.js_agent_url}\" "
                 script_insert += "tcellapikey=\"#{script_tag_policy.js_agent_api_key}\" "
                 script_insert += "tcellappid=\"#{script_tag_policy.js_agent_app_id}\"#{base_url_vars}></script>\n"
-                js_agent_handler = Proc.new { |si, resp|
+                js_agent_handler = proc { |si, resp|
                   self.handle_js_agent_insert(si, resp)
                 }
               end

data/lib/tcell_agent/rails/middleware/body_filter_middleware.rb

@@ -45,7 +45,7 @@ module TCellAgent
                       hmac_session_id,
                       user_id,
                       env[TCellAgent::Instrumentation::TCELL_ID].ip_address,
-                      env[TCellAgent::Instrumentation::TCELL_ID].user_agent,
+                      env[TCellAgent::Instrumentation::TCELL_ID].user_agent
                     )
                   end
                 end

data/lib/tcell_agent/rails/middleware/context_middleware.rb

@@ -32,9 +32,10 @@ module TCellAgent
               TCellAgent::Instrumentation.safe_block("Setting transaction_id") {
                 env[TCellAgent::Instrumentation::TCELL_ID].transaction_id = SecureRandom.uuid
                 request = Rack::Request.new(env)
-                env[TCellAgent::Instrumentation::TCELL_ID].uri = request.fullpath
+                env[TCellAgent::Instrumentation::TCELL_ID].uri = request.url
+                env[TCellAgent::Instrumentation::TCELL_ID].fullpath = request.fullpath
                 if request.request_method
-                  env[TCellAgent::Instrumentation::TCELL_ID].request_method = request.request_method.downcase
+                  env[TCellAgent::Instrumentation::TCELL_ID].request_method = request.request_method
                 end
               }
               env["filter_body_set"] = Set.new

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -35,7 +35,7 @@ module TCellAgent
             if TCellAgent.configuration.should_intercept_requests?
               TCellAgent::Instrumentation.safe_block("Handling Request") {
                 tcell_response = response
-                unless request.env[TCellAgent::Instrumentation::TCELL_ID].ip_blocking_triggered
+                unless request.env[TCellAgent::Instrumentation::TCELL_ID].patches_blocking_triggered
                   tcell_response = self._handle_appsensor_js_agent_and_dlp(request, tcell_response)
                 end
                 tcell_response = self._handle_redirect(request, tcell_response)
@@ -117,16 +117,17 @@ module TCellAgent
               status, headers, active_response = response
               http_redirect_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HttpRedirect)
               if http_redirect_policy && headers.has_key?("Location")
-                route_id = request.env[TCellAgent::Instrumentation::TCELL_ID].route_id
-                hmac_session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
+                tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
+                route_id = tcell_context.route_id
+                hmac_session_id = tcell_context.hmac_session_id
                 new_location = http_redirect_policy.enforce(
                   headers["Location"],
-                  request.url,
-                  request.fullpath,
-                  request.request_method,
+                  tcell_context.uri,
+                  tcell_context.fullpath,
+                  tcell_context.request_method,
                   route_id,
                   status,
-                  TCellAgent::Utils::Rails.better_ip(request),
+                  tcell_context.ip_address,
                   hmac_session_id)
                 # Enforcement
                 if (new_location)
@@ -185,8 +186,8 @@ module TCellAgent
                 defer_appfw_due_to_streaming = true
               end
 
-              appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
-              if appsensor_policy
+              rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+              if rust_policies && rust_policies.appfirewall_enabled
                 event = TCellAgent::SensorEvents::AppSensorMetaEvent.build(
                   request, content_length, status_code, response_headers
                 )

data/lib/tcell_agent/rails/routes/grape.rb

@@ -15,7 +15,7 @@ module TCellAgent
           end
 
           return true
-        rescue
+        rescue StandardError
         end
       end
 
@@ -30,7 +30,7 @@ module TCellAgent
           route_path = "#{grape_mount_endpoint}#{route_info[:path]}"
           route_method = route_info[:method]
 
-          route_id = TCellAgent::SensorEvents::Util.calculateRouteId(route_method.downcase, route_path)
+          route_id = TCellAgent::SensorEvents::Util.calculateRouteId(route_method, route_path)
           TCellAgent.send_event(
             TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
               route_path, route_method, route_id, nil, nil
@@ -44,13 +44,13 @@ module TCellAgent
       major, minor, tiny = Gem.loaded_specs['grape'].version.to_s.split('.')
       if major.to_i == 0 && minor.to_i < 16
         {
-          method: route.route_method,
-          path: route.route_path
+          :method => route.route_method,
+          :path => route.route_path
         }
       else
         {
-          method: route.request_method,
-          path: route.path
+          :method => route.request_method,
+          :path => route.path
         }
       end
     end

data/lib/tcell_agent/rails/routes.rb

@@ -65,7 +65,7 @@ module TCellAgent
       end
 
       class TCellRoute4 < TCellRoute
-        METHODS = %w{ DELETE GET HEAD OPTIONS PATCH POST PUT TRACE CONNECT }
+        METHODS = %w[DELETE GET HEAD OPTIONS PATCH POST PUT TRACE CONNECT]
 
         def report?
           @route.constraints.has_key?(:request_method) || grape_route?
@@ -115,7 +115,7 @@ module TCellAgent
             else
               tcell_route.route_methods.each do |route_method|
                 route_id =
-                  TCellAgent::SensorEvents::Util.calculateRouteId(route_method.downcase, tcell_route.route_path_raw)
+                  TCellAgent::SensorEvents::Util.calculateRouteId(route_method, tcell_route.route_path_raw)
                 TCellAgent.send_event(
                   TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
                     tcell_route.route_path, route_method, route_id, nil, tcell_route.route_destination
@@ -159,9 +159,8 @@ module TCellAgent
                     TCellAgent::Instrumentation::RouteId.update_context(env, parameters, route)
                   end
 
-                  patches_response = TCellAgent::Instrumentation::Patches.block?(request)
-                  if patches_response
-                    return head(patches_response)
+                  if TCellAgent::Instrumentation::Patches.block?(request)
+                    return head(403)
                   end
 
                   TCellAgent::DLP.handle_request_dlp_parameters(request)
@@ -217,9 +216,8 @@ module TCellAgent
                 TCellAgent::Instrumentation::RouteId.update_context(req.env, parameters, route)
               end
 
-              patches_response = TCellAgent::Instrumentation::Patches.block?(req)
-              if patches_response
-                return [patches_response, {}, []]
+              if TCellAgent::Instrumentation::Patches.block?(req)
+                return [403, {}, []]
               end
 
               TCellAgent::DLP.handle_request_dlp_parameters(req)
@@ -249,9 +247,8 @@ module TCellAgent
 
               tcell_request = Rack::Request.new(env)
 
-              patches_response = TCellAgent::Instrumentation::Patches.block?(tcell_request)
-              if patches_response
-                return [patches_response, {}, []]
+              if TCellAgent::Instrumentation::Patches.block?(tcell_request)
+                return [403, {}, []]
               end
 
               TCellAgent::DLP.handle_request_dlp_parameters(tcell_request)

data/lib/tcell_agent/rust/models.rb

@@ -56,6 +56,22 @@ module TCellAgent
         return request_response
       end
 
+      def self.create_patches_request(appsensor_meta)
+        post_params = convert_params(appsensor_meta.flattened_post_dict) +
+          convert_params(appsensor_meta.flattened_body_dict)
+
+        {
+          "method" =>  appsensor_meta.method,
+          "path" =>  appsensor_meta.path,
+          "remote_address" =>  appsensor_meta.remote_address,
+          "request_bytes_length" =>  appsensor_meta.request_content_bytes_len,
+          "query_params" =>  convert_params(appsensor_meta.flattened_get_dict),
+          "post_params" =>  post_params,
+          "headers" =>  convert_params(appsensor_meta.flattened_headers_dict),
+          "cookies" =>  convert_params(appsensor_meta.flattened_cookie_dict)
+        }
+      end
+
     end
   end
 end