tcell_agent 0.2.18 → 0.2.19

data/lib/tcell_agent/policies/appsensor/user_agent_sensor.rb

@@ -0,0 +1,47 @@
+require 'tcell_agent/policies/appsensor/sensor'
+
+module TCellAgent
+  module Policies
+
+    class UserAgentSensor < Sensor
+      DP_CODE = "uaempty"
+
+      attr_accessor :enabled, :empty_enabled, :excluded_route_ids
+
+      def initialize(policy_json=nil)
+        @enabled = false
+        @empty_enabled = false
+        @excluded_route_ids = {}
+
+        if policy_json
+          @enabled = policy_json.fetch("enabled", false)
+          @empty_enabled = policy_json.fetch("empty_enabled", false)
+
+          policy_json.fetch("exclude_routes", []).each do |excluded_route|
+            @excluded_route_ids[excluded_route] = true
+          end
+        end
+      end
+
+      def check(appsensor_meta)
+        return unless @enabled && @empty_enabled
+
+        return if @excluded_route_ids.fetch(appsensor_meta.route_id, false)
+
+        user_agent = appsensor_meta.user_agent
+        if !user_agent || user_agent.strip == ""
+          send_event(
+            appsensor_meta,
+            DP_CODE,
+            nil,
+            nil)
+        end
+      end
+
+      def to_s
+        "<#{self.class.name} enabled: #{@enabled} empty_enabled: #{@empty_enabled} dp_code: #{DP_CODE}>"
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/policies/appsensor_policy.rb

@@ -1,13 +1,16 @@
 require 'tcell_agent/instrumentation'
+require 'tcell_agent/policies/appsensor/cmdi_sensor'
+require 'tcell_agent/policies/appsensor/database_sensor'
+require 'tcell_agent/policies/appsensor/fpt_sensor'
 require 'tcell_agent/policies/appsensor/login_sensor'
+require 'tcell_agent/policies/appsensor/misc_sensor'
+require 'tcell_agent/policies/appsensor/nullbyte_sensor'
 require 'tcell_agent/policies/appsensor/request_size_sensor'
 require 'tcell_agent/policies/appsensor/response_codes_sensor'
 require 'tcell_agent/policies/appsensor/response_size_sensor'
-require 'tcell_agent/policies/appsensor/cmdi_sensor'
-require 'tcell_agent/policies/appsensor/fpt_sensor'
-require 'tcell_agent/policies/appsensor/nullbyte_sensor'
 require 'tcell_agent/policies/appsensor/retr_sensor'
 require 'tcell_agent/policies/appsensor/sqli_sensor'
+require 'tcell_agent/policies/appsensor/user_agent_sensor'
 require 'tcell_agent/policies/appsensor/xss_sensor'
 
 
@@ -25,7 +28,10 @@ module TCellAgent
         "fpt",
         "null",
         "retr",
-        "login_failure"]
+        "login_failure",
+        "ua",
+        "errors",
+        "database"]
 
       DETECTION_POINTS_V2 = {
           "req_size" => RequestSizeSensor,
@@ -37,7 +43,11 @@ module TCellAgent
           "fpt" => FptSensor,
           "nullbyte" => NullbyteSensor,
           "retr" => RetrSensor,
-          "login" => LoginSensor}
+          "login" => LoginSensor,
+          "ua" => UserAgentSensor,
+          "errors" => MiscSensor,
+          "database" => DatabaseSensor
+      }
 
       attr_accessor :policy_id, :options, :enabled
 
@@ -56,6 +66,16 @@ module TCellAgent
         check_params_for_injections(appsensor_meta)
       end
 
+      def process_db_rows(tcell_data, number_of_records)
+        return unless @enabled
+
+        TCellAgent::Instrumentation.safe_block("AppSensor Testing Number of DB Rows") do
+          if self.options.has_key?("database")
+            self.options["database"].check(tcell_data, number_of_records)
+          end
+        end
+      end
+
       def check_request_size(appsensor_meta)
         TCellAgent::Instrumentation.safe_block("AppSensor Testing Response Size") do
           if self.options.has_key?("req_size")
@@ -98,6 +118,20 @@ module TCellAgent
       end
 
       def check_params_for_injections(appsensor_meta)
+        path_param_type =
+          if (appsensor_meta.method || "get").to_s.downcase == "get"
+            InjectionSensor::GET_PARAM
+          else
+            InjectionSensor::POST_PARAM
+          end
+
+        (appsensor_meta.path_parameters || {}).each do |param_name, param_value|
+          TCellAgent::Instrumentation.safe_block("AppSensor Check Path Params injections") do
+            next if param_name == :controller || param_name == :action
+            check_param_for_injections(path_param_type, appsensor_meta, param_name.to_s, param_value)
+          end
+        end
+
         (appsensor_meta.get_dict || {}).each do |param_name, param_value|
           TCellAgent::Instrumentation.safe_block("AppSensor Check GET var injections") do
             check_param_for_injections(InjectionSensor::GET_PARAM, appsensor_meta, param_name, param_value)
@@ -123,8 +157,25 @@ module TCellAgent
         end
       end
 
+      def csrf_rejected(tcell_data)
+        TCellAgent::Instrumentation.safe_block("AppSensor CSRF Exception processing") do
+          if self.options.has_key?("errors")
+            self.options["errors"].csrf_rejected(tcell_data)
+          end
+        end
+      end
+
+      def sql_exception_detected(tcell_data, exception)
+        TCellAgent::Instrumentation.safe_block("AppSensor SQL Exception processing") do
+          if self.options.has_key?("errors")
+            self.options["errors"].sql_exception_detected(tcell_data, exception)
+          end
+        end
+      end
+
       def self.from_json(policy_json)
         return nil unless policy_json
+        policy_json = policy_json.deep_dup
 
         sensor_policy = AppSensorPolicy.new
         if policy_json.has_key?("policy_id")
@@ -193,6 +244,18 @@ module TCellAgent
                     enabled = options_json.fetch(sensor_name, false)
                     sensor_policy.options["login"] = LoginSensor.new({"enabled" => enabled})
 
+                  elsif "ua" == sensor_name
+                    sensor_policy.options[sensor_name] = UserAgentSensor.new({
+                      "enabled" => false, "empty_enabled" => false
+                    })
+
+                  elsif "errors" == sensor_name
+                    sensor_policy.options[sensor_name] = MiscSensor.new({
+                      "enabled" => false,
+                      "csrf_exception_enabled" => false,
+                      "sql_exception_enabled" => false
+                    })
+
                   else
                     enabled = options_json.fetch(sensor_name, false)
                     clazz = DETECTION_POINTS_V2[sensor_name]

data/lib/tcell_agent/policies/patches_policy.rb

@@ -11,8 +11,8 @@ module TCellAgent
         @blocked_ips = {}
       end
 
-      def block_ip?(request)
-        @ip_blocking_enabled && @blocked_ips[request.ip]
+      def block_ip?(ip_address)
+        @ip_blocking_enabled && @blocked_ips[ip_address]
       end
 
       def self.from_json(policy_json)

data/lib/tcell_agent/rails.rb

@@ -9,6 +9,7 @@ require 'tcell_agent/sensor_events/server_agent'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/sensor_events/util/redirect_utils'
 
+require 'tcell_agent/rails/better_ip'
 require 'tcell_agent/rails/middleware/global_middleware'
 require 'tcell_agent/rails/middleware/body_filter_middleware'
 require 'tcell_agent/rails/middleware/headers_middleware'
@@ -16,6 +17,7 @@ require 'tcell_agent/rails/middleware/context_middleware'
 
 require 'tcell_agent/rails/settings_reporter'
 require 'tcell_agent/rails/dlp'
+require 'tcell_agent/rails/csrf_exception'
 
 
 require 'tcell_agent/userinfo'
@@ -30,6 +32,7 @@ module TCellAgent
             require 'tcell_agent/rails/auth/devise' if defined?(Devise)
             require 'tcell_agent/authlogic' if defined?(Authlogic)
             require 'tcell_agent/rails/auth/authlogic'  if defined?(Authlogic)
+            require 'tcell_agent/rails/path_parameters_setter'
           end
           app.config.middleware.insert_before(0, "TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware")
           app.config.middleware.insert_after(0, "TCellAgent::Instrumentation::Rails::Middleware::HeadersMiddleware")

data/lib/tcell_agent/rails/auth/authlogic.rb

@@ -30,7 +30,7 @@ module TCellAgent
                 if (login_fraud_policy.login_failed_enabled)
                   request = Authlogic::Session::Base.controller.request
                   response = Authlogic::Session::Base.controller.response
-                  hmac_session_id = request.env["tcell.request_data"].hmac_session_id
+                  hmac_session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
                   event = TCellAgent::SensorEvents::LoginFailure.new(request, response, user_id, hmac_session_id)
                   TCellAgent.send_event(event)
                 end
@@ -38,7 +38,7 @@ module TCellAgent
                 if (login_fraud_policy.login_success_enabled)
                   request = Authlogic::Session::Base.controller.request
                   response = Authlogic::Session::Base.controller.response
-                  hmac_session_id = request.env["tcell.request_data"].hmac_session_id
+                  hmac_session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
                   event = TCellAgent::SensorEvents::LoginSuccess.new(request, response, user_id, hmac_session_id)
                   TCellAgent.send_event(event)
                 end

data/lib/tcell_agent/rails/auth/devise.rb

@@ -22,9 +22,9 @@ module TCellAgent
             tcell_username = _get_tcell_username
             login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
             if (login_fraud_policy && login_fraud_policy.enabled && login_fraud_policy.login_success_enabled)
-              hmac_session_id = request.env["tcell.request_data"].hmac_session_id
-              request.env["tcell.request_data"].user_id = TCellAgent::UserInformation.getUserFromRequest(request)
-              user_id = tcell_username || request.env["tcell.request_data"].user_id
+              hmac_session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
+              request.env[TCellAgent::Instrumentation::TCELL_ID].user_id = TCellAgent::UserInformation.getUserFromRequest(request)
+              user_id = tcell_username || request.env[TCellAgent::Instrumentation::TCELL_ID].user_id
               event = TCellAgent::SensorEvents::LoginSuccess.new(request, response, user_id, hmac_session_id)
               TCellAgent.send_event(event)
             end
@@ -58,7 +58,7 @@ module TCellAgent
             login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
             if (login_fraud_policy && login_fraud_policy.enabled && login_fraud_policy.login_failed_enabled)
               if failed_login?
-                hmac_session_id = request.env["tcell.request_data"].hmac_session_id
+                hmac_session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
                 event = TCellAgent::SensorEvents::LoginFailure.new(request, response, tcell_username, hmac_session_id)
                 TCellAgent.send_event(event)
               end

data/lib/tcell_agent/rails/better_ip.rb

@@ -0,0 +1,36 @@
+require "tcell_agent/utils/strings"
+require 'tcell_agent/instrumentation'
+
+
+module TCellAgent
+  module Utils
+    module Rails
+
+      def self.better_ip(request)
+        if TCellAgent.configuration.reverse_proxy
+          TCellAgent::Instrumentation.safe_block("Extracting reverse proxy IP") do
+            reverse_proxy_header = TCellAgent.configuration.reverse_proxy_ip_address_header
+            if TCellAgent::Utils::Strings.present?(reverse_proxy_header)
+              reverse_proxy_header = "HTTP_" + reverse_proxy_header.upcase().gsub('-','_')
+            else
+              reverse_proxy_header = "HTTP_X_FORWARDED_FOR"
+            end
+
+            x_forwarded_for = request.env[reverse_proxy_header]
+
+            if TCellAgent::Utils::Strings.present?(x_forwarded_for)
+              ip = x_forwarded_for.split(',')[0].strip()
+            else
+              ip = request.ip
+            end
+
+            return ip
+          end
+        end
+
+        request.ip
+      end
+
+    end
+  end
+end

data/lib/tcell_agent/rails/csrf_exception.rb

@@ -0,0 +1,30 @@
+require 'tcell_agent/instrumentation'
+
+module TCellAgent
+  class MyRailtie < Rails::Railtie
+
+    initializer "tcell.sensors" do |app|
+      ActiveSupport.on_load :action_controller do
+
+        ActionController::RequestForgeryProtection.module_eval do
+          alias_method :tcell_handle_unverified_request, :handle_unverified_request
+          def handle_unverified_request
+            TCellAgent::Instrumentation.safe_block("AppSensor CSRF Exception processing") do
+              appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
+              if appsensor_policy
+                tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
+                if tcell_data
+                  appsensor_policy.csrf_rejected(tcell_data)
+                end
+              end
+            end
+
+            tcell_handle_unverified_request
+          end
+        end
+
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/rails/dlp.rb

@@ -22,31 +22,12 @@ require 'tcell_agent/rails/middleware/context_middleware'
 require 'tcell_agent/rails/routes'
 require 'tcell_agent/rails/settings_reporter'
 
+require 'tcell_agent/instrumentation'
+
 require 'tcell_agent/userinfo'
 require 'cgi'
 require 'thread'
 
-# if defined?(SQLite3)
-#   require 'active_record/connection_adapters/sqlite3_adapter'
-#   ActiveRecord::ConnectionAdapters::SQLite3Adapter.class_eval do
-#         alias_method :original_exec, :exec_query
-#         def exec_query(sql, name = nil, binds = [])
-#           puts "----v----"
-#           puts sql
-#           puts name
-#           puts binds
-#           puts "----^----"
-#           result = original_exec(sql, name, binds)
-#           puts result.inspect
-#           puts ";-----------------------;"
-#           result
-#         end
-#         def postgresql_version
-#           80200
-#         end
-#   end
-# end
-
 require 'tcell_agent/configuration'
 
 
@@ -55,50 +36,25 @@ module TCellAgent
     initializer 'activeservice.autoload', :after => :set_autoload_paths do |app|
 
       if defined?(ActiveRecord)
-        #ActiveRecord::Calculations.module_eval do
-          #alias_method :tcell_pluck, :pluck
-          #def pluck(*column_names)
-            #puts "PLUCK"
-            #puts table
-            #puts model
-            #puts column_names
-            #tcell_pluck(*column_names)
-          #end
-        #end
-
-        #ActiveRecord::Relation.class_eval do
-          #alias_method :tcell_exec_queries, :exec_queries
-          #def exec_queries
-            #puts "RELATION"
-            #results = tcell_exec_queries
-
-            #results
-          #end
-        #end
-
-        #ActiveRecord::Scoping::Default::ClassMethods.module_eval do
-          #alias_method :tcell_build_default_scope, :build_default_scope
-          #def build_default_scope(base_rel = relation)
-            ##puts base_rel.inspect
-            #tcell_build_default_scope
-          #end
-        #end
-
-        #ActiveRecord::Scoping::Named::ClassMethods.module_eval do
-          #alias_method :tcell_default_scoped, :default_scoped
-          #def default_scoped
-            #tcell_default_scoped
-          #end
-        #end
-
-        #ActiveRecord::ConnectionAdapters::PostgreSQL::DatabaseStatements.module_eval do
-          #alias_method :tcell_execute, :execute
-          #def execute(sql, name = nil)
-            #pus caller
-
-            #tcell_execute(sql, name)
-          #end
-        #end
+        ActiveRecord::ConnectionAdapters::AbstractAdapter.class_eval do
+          alias_method :tcell_translate_exception, :translate_exception
+          def translate_exception(e, message)
+            result = tcell_translate_exception(e, message)
+
+            TCellAgent::Instrumentation.safe_block("Call AppSensorPolicy.sql_exception_detected") do
+              appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
+              if appsensor_policy
+                request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+                tcell_data = request_env[TCellAgent::Instrumentation::TCELL_ID]
+                if tcell_data && e.is_a?(ActiveRecord::StatementInvalid)
+                  appsensor_policy.sql_exception_detected(tcell_data, result)
+                end
+              end
+            end
+
+            result
+          end
+        end
 
         ActiveRecord::Querying.module_eval do
 
@@ -114,20 +70,29 @@ module TCellAgent
                     TCellAgent.configuration.should_intercept_requests?
 
                   dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+                  appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
 
-                  if dlp_policy
-                    request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
+                  if dlp_policy || appsensor_policy
+                    request_env =
+                      TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+                    tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
 
-                    if request_env
-                      tcell_context = request_env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+                    if tcell_context
+                      if appsensor_policy
+                        appsensor_policy.process_db_rows(tcell_context, results.size)
+                      end
 
-                      if tcell_context
+                      if dlp_policy
                         first_record = results.first
                         database_name = first_record.class.connection_config().fetch(:database,"*").split('/').last
                         model = first_record.class
                         column_names = model.columns.map { |col| col.name }
                         table_name = model.table_name
 
+                        if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
+                          TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
+                        end
+
                         if dlp_policy.database_discovery_enabled
                           TCellAgent.discover_database_fields(
                             tcell_context.route_id,
@@ -150,10 +115,6 @@ module TCellAgent
                           memo
                         end
 
-                        if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
-                          TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
-                        end
-
                         results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |record|
                           column_name_to_rules.each do |column_name, rules|
                             if rules
@@ -171,6 +132,7 @@ module TCellAgent
                           end
                         end
                       end
+
                     end
                   end
                 end
@@ -294,7 +256,7 @@ module TCellAgent
               TCellAgent.configuration.should_intercept_requests?
 
             TCellAgent::Instrumentation.safe_block("Running DLP Logging Filters") {
-              tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+              tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
               if tcell_context
                 response.body = tcell_context.filter_body(response.body)
               end
@@ -330,7 +292,7 @@ class Logger
         dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
         request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
         if message && dlp_policy && request_env
-          tcell_context = request_env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+          tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
           if tcell_context
             tcell_context.filter_log(message)
           end