tcell_agent 0.2.19 → 0.2.21

data/lib/tcell_agent/policies/appsensor/injection_sensor.rb

@@ -5,7 +5,19 @@ require 'tcell_agent/utils/params'
 module TCellAgent
   module Policies
     class InjectionSensor < Sensor
-      include TCellAgent::Utils::Params
+      GET_PARAM = TCellAgent::Utils::Params::GET_PARAM
+      POST_PARAM = TCellAgent::Utils::Params::POST_PARAM
+      JSON_PARAM = TCellAgent::Utils::Params::JSON_PARAM
+      COOKIE_PARAM = TCellAgent::Utils::Params::COOKIE_PARAM
+      URI_PARAM = TCellAgent::Utils::Params::URI_PARAM
+
+      PARAM_TYPE_TO_L = {
+        GET_PARAM => 'query',
+        POST_PARAM => 'body',
+        JSON_PARAM => 'body',
+        URI_PARAM => 'uri',
+        COOKIE_PARAM => 'cookie'
+      }
 
       attr_accessor :enabled, :detection_point, :exclude_headers, :exclude_forms,
         :exclude_cookies, :exclusions, :active_pattern_ids, :v1_compatability_enabled,
@@ -54,17 +66,19 @@ module TCellAgent
         rules = get_ruleset
         return nil unless rules
 
-        param_deep_loop(param_name, param_value) do |name, value|
-          rules.check_violation(name, value, @active_pattern_ids, @v1_compatability_enabled)
-        end
+        rules.check_violation(param_name, param_value, @active_pattern_ids, @v1_compatability_enabled)
       end
 
-      def check(type_of_param, appsensor_meta, param_name, param_value)
+      def check(type_of_param, appsensor_meta, param_name, param_value, payloads_policy)
         return false unless @enabled
 
         return false if @excluded_route_ids.fetch(appsensor_meta.route_id, false)
 
-        if @exclude_forms && (GET_PARAM == type_of_param or POST_PARAM == type_of_param or JSON_PARAM == type_of_param)
+        if @exclude_forms &&
+            (GET_PARAM == type_of_param ||
+             POST_PARAM == type_of_param ||
+             JSON_PARAM == type_of_param ||
+             URI_PARAM == type_of_param)
           return false
         end
 
@@ -76,34 +90,23 @@ module TCellAgent
 
         if vuln_results
           vuln_param = vuln_results["param"]
-          payload = nil
-
-          if TCellAgent.configuration.allow_unencrypted_appfirewall_payloads
-            payload = vuln_results["value"]
-          end
 
           if vuln_param
-            unless payload.nil?
-              if TCellAgent.configuration.blacklisted_params.has_key?(vuln_param.downcase)
-                payload = "BLACKLISTED"
+            meta = {"l" => PARAM_TYPE_TO_L[type_of_param]}
+            pattern = vuln_results["pattern"]
 
-              elsif TCellAgent.configuration.whitelist_present &&
-                  !TCellAgent.configuration.whitelisted_params.has_key?(vuln_param.downcase)
-                payload = "NOT_WHITELISTED"
-              end
-            end
-
-            log_appsensor_events(type_of_param, appsensor_meta, vuln_param, vuln_results["value"])
-
-            send_event(
-              appsensor_meta,
+            payload = payloads_policy.apply(
               @detection_point,
+              appsensor_meta,
+              type_of_param,
               vuln_param,
-              {"t" => type_of_param}.to_json,
-              payload,
-              vuln_results["pattern"]
+              vuln_results["value"],
+              meta,
+              pattern
             )
 
+            send_event(appsensor_meta, @detection_point, vuln_param, meta, payload, pattern)
+
             return true
           end
         end
@@ -111,25 +114,6 @@ module TCellAgent
         return false
       end
 
-      def log_appsensor_events(type_of_param, appsensor_meta, vuln_param, vuln_value)
-        if TCellAgent.configuration.allow_unencrypted_appfirewall_payloads_logging
-          event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
-            appsensor_meta.location,
-            @detection_point,
-            appsensor_meta.method,
-            appsensor_meta.remote_address,
-            vuln_param,
-            appsensor_meta.route_id,
-            {"t" => type_of_param}.to_json,
-            appsensor_meta.session_id,
-            appsensor_meta.user_id,
-            vuln_value
-          )
-          event.post_process
-          TCellAgent.appfirewall_payloads_logger.info(event.to_json)
-        end
-      end
-
       def to_s
         "<#{self.class.name} enabled: #{@enabled} dp: #{@detection_point} " +
         "exclude_headers: #{@exclude_headers} exclude_forms: #{exclude_forms} " +

data/lib/tcell_agent/policies/appsensor/login_sensor.rb

@@ -23,10 +23,7 @@ module TCellAgent
         return unless self.enabled
 
         if username
-          username = TCellAgent::SensorEvents::Util.hmac(
-            username,
-            TCellAgent::SensorEvents::Util.getHmacKey()
-          )
+          username = TCellAgent::SensorEvents::Util.hmac(username)
         end
 
         send_event(appsensor_meta, LOGIN_FAILURE_DP, username, nil)

data/lib/tcell_agent/policies/appsensor/misc_sensor.rb

@@ -1,7 +1,9 @@
+require 'tcell_agent/policies/appsensor/sensor'
+
 module TCellAgent
   module Policies
 
-    class MiscSensor
+    class MiscSensor < Sensor
 
       attr_accessor :enabled, :csrf_exception_enabled, :sql_exception_enabled, :excluded_route_ids
 
@@ -22,12 +24,13 @@ module TCellAgent
         end
       end
 
-      def csrf_rejected(tcell_data)
+      def csrf_rejected(tcell_data, exception_class)
         return unless @enabled && @csrf_exception_enabled
 
         return if tcell_data && @excluded_route_ids.fetch(tcell_data.route_id, false)
 
-        send_event("excsrf", tcell_data)
+        meta = nil
+        send_event_from_tcell_data(tcell_data, "excsrf", exception_class.name, meta)
       end
 
       def sql_exception_detected(tcell_data, exception)
@@ -35,25 +38,8 @@ module TCellAgent
 
         return if tcell_data && @excluded_route_ids.fetch(tcell_data.route_id, false)
 
-        send_event("exsql", tcell_data)
-      end
-
-      def send_event(detection_point, tcell_data)
-        event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
-          tcell_data.uri,
-          detection_point,
-          tcell_data.request_method,
-          tcell_data.ip_address,
-          nil,
-          tcell_data.route_id,
-          nil,
-          tcell_data.transaction_id,
-          tcell_data.session_id,
-          tcell_data.user_id,
-          nil
-        )
-
-        TCellAgent.send_event(event)
+        meta = nil
+        send_event_from_tcell_data(tcell_data, "exsql", exception.class.name, meta)
       end
 
       def to_s

data/lib/tcell_agent/policies/appsensor/payloads_policy.rb

@@ -0,0 +1,143 @@
+require 'tcell_agent/utils/params'
+
+module TCellAgent
+  module Policies
+
+    class PayloadsPolicy
+      PARAM_TYPE_MAP = {
+        TCellAgent::Utils::Params::GET_PARAM => "form",
+        TCellAgent::Utils::Params::POST_PARAM => "form",
+        TCellAgent::Utils::Params::JSON_PARAM => "form",
+        TCellAgent::Utils::Params::COOKIE_PARAM => "cookie"
+      }
+
+      attr_accessor :send_payloads, :send_blacklist, :send_whitelist, :use_send_whitelist,
+        :log_payloads, :log_blacklist, :log_whitelist, :use_log_whitelist
+
+      def initialize
+        @send_payloads = false
+        @log_payloads = false
+
+        @send_blacklist = {}
+        @log_blacklist = {}
+        @send_whitelist = {}
+        @log_whitelist = {}
+
+        @use_send_whitelist = false
+        @use_log_whitelist = false
+      end
+
+      def apply(dp, appsensor_meta, type_of_param, vuln_param, vuln_value, meta, pattern)
+        payload = nil
+
+        if @send_payloads && TCellAgent.configuration.allow_unencrypted_appfirewall_payloads
+
+          blacklisted_locations = @send_blacklist[vuln_param.downcase]
+          param_location = PARAM_TYPE_MAP[type_of_param]
+
+          if blacklisted_locations &&
+              ( blacklisted_locations.include?(param_location) ||
+               blacklisted_locations.include?("*") )
+            payload = "BLACKLISTED"
+
+          elsif use_send_whitelist
+            whitelisted_locations = @send_whitelist[vuln_param.downcase]
+            if whitelisted_locations &&
+               ( whitelisted_locations.include?(param_location) ||
+                whitelisted_locations.include?("*") )
+
+              payload = vuln_value
+
+            else
+              payload = "NOT_WHITELISTED"
+            end
+
+          else
+            payload = vuln_value
+          end
+
+        end
+
+        log(dp, appsensor_meta, type_of_param, vuln_param, vuln_value, meta, pattern)
+
+        payload
+      end
+
+      def log(dp, appsensor_meta, type_of_param, vuln_param, vuln_value, meta, pattern)
+        if @log_payloads && TCellAgent.configuration.allow_unencrypted_appfirewall_payloads_logging
+          blacklisted_locations = @log_blacklist[vuln_param.downcase]
+          param_location = PARAM_TYPE_MAP[type_of_param]
+
+          if !blacklisted_locations ||
+              ( !blacklisted_locations.include?(param_location) &&
+               !blacklisted_locations.include?("*") )
+
+            whitelisted_locations = @log_whitelist[vuln_param.downcase]
+            if !use_log_whitelist ||
+                (whitelisted_locations && (whitelisted_locations.include?(param_location) ||
+                whitelisted_locations.include?("*")))
+
+              event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
+                appsensor_meta.location,
+                dp,
+                appsensor_meta.method,
+                appsensor_meta.remote_address,
+                vuln_param,
+                appsensor_meta.route_id,
+                meta,
+                appsensor_meta.session_id,
+                appsensor_meta.user_id,
+                vuln_value,
+                pattern
+              )
+              event.post_process
+              TCellAgent.appfirewall_payloads_logger.info(event.to_json)
+            end
+          end
+        end
+      end
+
+      def self.from_json(policy_json)
+        policy = PayloadsPolicy.new
+
+        if policy_json
+          payloads_json = policy_json.fetch("payloads", {})
+          policy.send_payloads = payloads_json.fetch("send_payloads", false)
+          policy.log_payloads = payloads_json.fetch("log_payloads", false)
+
+          if policy.send_payloads
+            payloads_json.fetch("send_blacklist", {}).each do |param_name, locations|
+              policy.send_blacklist[param_name.downcase] = Set.new(locations)
+            end
+
+            send_whitelist = payloads_json["send_whitelist"]
+            if send_whitelist
+              send_whitelist.each do |param_name, locations|
+                policy.send_whitelist[param_name.downcase] = Set.new(locations)
+              end
+              policy.use_send_whitelist = true
+            end
+          end
+
+          if policy.log_payloads
+            payloads_json.fetch("log_blacklist", {}).each do |param_name, locations|
+              policy.log_blacklist[param_name.downcase] = Set.new(locations)
+            end
+
+            log_whitelist = payloads_json["log_whitelist"]
+            if log_whitelist
+              log_whitelist.each do |param_name, locations|
+                policy.log_whitelist[param_name.downcase] = Set.new(locations)
+              end
+
+              policy.use_log_whitelist = true
+            end
+          end
+        end
+
+        policy
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/policies/appsensor/response_codes_sensor.rb

@@ -51,7 +51,9 @@ module TCellAgent
         end
 
         if dp
-          send_event(appsensor_meta, dp, response_code.to_s, nil)
+          param = payload = pattern = nil
+          meta = { code: response_code }
+          send_event(appsensor_meta, dp, param, meta, payload, pattern)
         end
 
         def to_s

data/lib/tcell_agent/policies/appsensor/sensor.rb

@@ -4,7 +4,7 @@ module TCellAgent
   module Policies
 
     class Sensor
-      def send_event(appsensor_meta, detection_point, parameter, data, payload=nil, pattern=nil)
+      def send_event(appsensor_meta, detection_point, parameter, meta, payload, pattern)
         event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
           appsensor_meta.location,
           detection_point,
@@ -12,7 +12,7 @@ module TCellAgent
           appsensor_meta.remote_address,
           parameter,
           appsensor_meta.route_id,
-          data,
+          meta,
           appsensor_meta.session_id,
           appsensor_meta.user_id,
           payload,
@@ -21,6 +21,25 @@ module TCellAgent
 
         TCellAgent.send_event(event)
       end
+
+      def send_event_from_tcell_data(tcell_data, detection_point, parameter, meta)
+        payload = pattern = nil
+        event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
+          tcell_data.uri,
+          detection_point,
+          tcell_data.request_method,
+          tcell_data.ip_address,
+          parameter,
+          tcell_data.route_id,
+          meta,
+          tcell_data.hmac_session_id,
+          tcell_data.user_id,
+          payload,
+          pattern
+        )
+
+        TCellAgent.send_event(event)
+      end
     end
 
   end

data/lib/tcell_agent/policies/appsensor/size_sensor.rb

@@ -29,7 +29,9 @@ module TCellAgent
         end
 
         if content_length && content_length > @limit
-          send_event(appsensor_meta, @dp_code, content_length.to_s, nil)
+          param = payload = pattern = nil
+          meta = { "sz" => content_length }
+          send_event(appsensor_meta, @dp_code, param, meta, payload, pattern)
         end
       end
 

data/lib/tcell_agent/policies/appsensor/sqli_sensor.rb

@@ -1,3 +1,4 @@
+require 'libinjection/libinjection'
 require 'tcell_agent/policies/appsensor/injection_sensor'
 
 module TCellAgent
@@ -19,6 +20,14 @@ module TCellAgent
           @libinjection = policy_json.fetch("libinjection", false)
         end
       end
+
+      def find_vulnerability(param_name, param_value)
+        if @libinjection && Libinjection.is_sqli(param_value) == 1
+          return {"param" => param_name, "value" => param_value, "pattern" => "li"}
+        end
+
+        super(param_name, param_value)
+      end
     end
 
   end

data/lib/tcell_agent/policies/appsensor/user_agent_sensor.rb

@@ -30,11 +30,7 @@ module TCellAgent
 
         user_agent = appsensor_meta.user_agent
         if !user_agent || user_agent.strip == ""
-          send_event(
-            appsensor_meta,
-            DP_CODE,
-            nil,
-            nil)
+          send_event(appsensor_meta, DP_CODE, nil, nil, nil, nil)
         end
       end
 

data/lib/tcell_agent/policies/appsensor/xss_sensor.rb

@@ -1,6 +1,6 @@
+require 'libinjection/libinjection'
 require 'tcell_agent/policies/appsensor/injection_sensor'
 
-
 module TCellAgent
   module Policies
 
@@ -20,6 +20,14 @@ module TCellAgent
           @libinjection = policy_json.fetch("libinjection", false)
         end
       end
+
+      def find_vulnerability(param_name, param_value)
+        if @libinjection && Libinjection.is_xss(param_value) == 1
+          return {"param" => param_name, "value" => param_value, "pattern" => "li"}
+        end
+
+        super(param_name, param_value)
+      end
     end
 
   end

data/lib/tcell_agent/policies/appsensor_policy.rb

@@ -5,6 +5,7 @@ require 'tcell_agent/policies/appsensor/fpt_sensor'
 require 'tcell_agent/policies/appsensor/login_sensor'
 require 'tcell_agent/policies/appsensor/misc_sensor'
 require 'tcell_agent/policies/appsensor/nullbyte_sensor'
+require 'tcell_agent/policies/appsensor/payloads_policy'
 require 'tcell_agent/policies/appsensor/request_size_sensor'
 require 'tcell_agent/policies/appsensor/response_codes_sensor'
 require 'tcell_agent/policies/appsensor/response_size_sensor'
@@ -12,6 +13,7 @@ require 'tcell_agent/policies/appsensor/retr_sensor'
 require 'tcell_agent/policies/appsensor/sqli_sensor'
 require 'tcell_agent/policies/appsensor/user_agent_sensor'
 require 'tcell_agent/policies/appsensor/xss_sensor'
+require 'tcell_agent/utils/params'
 
 
 module TCellAgent
@@ -49,17 +51,19 @@ module TCellAgent
           "database" => DatabaseSensor
       }
 
-      attr_accessor :policy_id, :options, :enabled
+      attr_accessor :policy_id, :options, :payloads_policy, :enabled
 
       def initialize
         @policy_id = nil
         @options = Hash.new
         @enabled = false
+        @payloads_policy = PayloadsPolicy.new
       end
 
       def process_meta_event(appsensor_meta)
         return unless @enabled
 
+        check_user_agent(appsensor_meta)
         check_request_size(appsensor_meta)
         check_response_size(appsensor_meta)
         check_response_code(appsensor_meta)
@@ -76,6 +80,14 @@ module TCellAgent
         end
       end
 
+      def check_user_agent(appsensor_meta)
+        TCellAgent::Instrumentation.safe_block("AppSensor Checking User Agent") do
+          if self.options.has_key?("ua")
+            self.options["ua"].check(appsensor_meta)
+          end
+        end
+      end
+
       def check_request_size(appsensor_meta)
         TCellAgent::Instrumentation.safe_block("AppSensor Testing Response Size") do
           if self.options.has_key?("req_size")
@@ -101,66 +113,65 @@ module TCellAgent
       end
 
       def check_param_for_injections(param_type, appsensor_meta, param_name, param_value)
-        return if @options["xss"].check(param_type, appsensor_meta, param_name, param_value)
-        return if @options["sqli"].check(param_type, appsensor_meta, param_name, param_value)
+        return if @options["xss"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
+        return if @options["sqli"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
         if InjectionSensor::COOKIE_PARAM != param_type
-          return if @options["cmdi"].check(param_type, appsensor_meta, param_name, param_value)
+          return if @options["cmdi"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
         end
         if InjectionSensor::COOKIE_PARAM != param_type
-          return if @options["fpt"].check(param_type, appsensor_meta, param_name, param_value)
+          return if @options["fpt"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
         end
         if InjectionSensor::COOKIE_PARAM != param_type
-          return if @options["nullbyte"].check(param_type, appsensor_meta, param_name, param_value)
+          return if @options["nullbyte"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
         end
         if InjectionSensor::POST_PARAM != param_type && InjectionSensor::JSON_PARAM != param_type
-          return if @options["retr"].check(param_type, appsensor_meta, param_name, param_value)
+          return if @options["retr"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
         end
       end
 
       def check_params_for_injections(appsensor_meta)
-        path_param_type =
-          if (appsensor_meta.method || "get").to_s.downcase == "get"
-            InjectionSensor::GET_PARAM
-          else
-            InjectionSensor::POST_PARAM
-          end
 
-        (appsensor_meta.path_parameters || {}).each do |param_name, param_value|
+        TCellAgent::Utils::Params.flatten(appsensor_meta.path_parameters || {}).each do |param_name, param_value|
           TCellAgent::Instrumentation.safe_block("AppSensor Check Path Params injections") do
+            param_name = param_name[-1]
             next if param_name == :controller || param_name == :action
-            check_param_for_injections(path_param_type, appsensor_meta, param_name.to_s, param_value)
+            check_param_for_injections(InjectionSensor::URI_PARAM, appsensor_meta, param_name.to_s, param_value)
           end
         end
 
-        (appsensor_meta.get_dict || {}).each do |param_name, param_value|
+        TCellAgent::Utils::Params.flatten(appsensor_meta.get_dict || {}).each do |param_name, param_value|
           TCellAgent::Instrumentation.safe_block("AppSensor Check GET var injections") do
+            param_name = param_name[-1]
             check_param_for_injections(InjectionSensor::GET_PARAM, appsensor_meta, param_name, param_value)
           end
         end
 
         (appsensor_meta.post_dict || {}).each do |param_name, param_value|
           TCellAgent::Instrumentation.safe_block("AppSensor Check POST var injections") do
+            param_name = param_name[-1]
             check_param_for_injections(InjectionSensor::POST_PARAM, appsensor_meta, param_name, param_value)
           end
         end
 
         (appsensor_meta.body_dict || {}).each do |param_name, param_value|
           TCellAgent::Instrumentation.safe_block("AppSensor Check JSON var injections") do
+            param_name = param_name[-1]
             check_param_for_injections(InjectionSensor::JSON_PARAM, appsensor_meta, param_name, param_value)
           end
         end
 
-        (appsensor_meta.cookie_dict || {}).each do |param_name, param_value|
+        TCellAgent::Utils::Params.flatten(appsensor_meta.cookie_dict || {}).each do |param_name, param_value|
           TCellAgent::Instrumentation.safe_block("AppSensor Check COOKIE var injections") do
+            param_name = param_name[-1]
             check_param_for_injections(InjectionSensor::COOKIE_PARAM, appsensor_meta, param_name, param_value)
           end
         end
       end
 
-      def csrf_rejected(tcell_data)
+      def csrf_rejected(tcell_data, exception_class)
         TCellAgent::Instrumentation.safe_block("AppSensor CSRF Exception processing") do
           if self.options.has_key?("errors")
-            self.options["errors"].csrf_rejected(tcell_data)
+            self.options["errors"].csrf_rejected(tcell_data, exception_class)
           end
         end
       end
@@ -198,6 +209,9 @@ module TCellAgent
 
               else
                 sensor_policy.enabled = true
+                sensor_policy.payloads_policy = PayloadsPolicy.from_json(
+                  data_json.fetch("options", {})
+                )
 
                 DETECTION_POINTS_V2.each do |sensor_name, sensor_class|
                   settings = sensors_json.fetch(sensor_name, {})
@@ -217,6 +231,13 @@ module TCellAgent
 
               else
                 sensor_policy.enabled = true
+                sensor_policy.payloads_policy = PayloadsPolicy.from_json({
+                  "payloads" => {
+                    "send_payloads" => true,
+                    "log_payloads" => true
+                  }
+                })
+
                 DETECTION_POINTS_V1.each do |sensor_name|
                   if "req_res_size" == sensor_name
                     enabled = options_json.fetch(sensor_name, false)

data/lib/tcell_agent/policies/http_redirect_policy.rb

@@ -41,7 +41,7 @@ module TCellAgent
         end
         return true
       end
-      def enforce(target_url, current_host, current_path, method, route_id, status_code, remote_addr, session_id=nil)
+      def enforce(target_url, current_host, current_path, method, route_id, status_code, remote_addr, hmac_session_id=nil)
         if @enabled == false
           return nil
         end
@@ -51,7 +51,17 @@ module TCellAgent
           return nil
         end
         begin
-          event = TCellAgent::SensorEvents::TCellRedirectSensorEvent.new(host, current_host, current_path, method, route_id, status_code, remote_addr, session_id, nil)
+          event = TCellAgent::SensorEvents::TCellRedirectSensorEvent.new(
+            host,
+            current_host,
+            current_path,
+            method,
+            route_id,
+            status_code,
+            remote_addr,
+            hmac_session_id,
+            nil)
+
           TCellAgent.send_event(event)
         rescue Exception => ie
           TCellAgent.logger.error("uncaught exception while creating redirect event: #{ie.message}")

data/lib/tcell_agent/rails/csrf_exception.rb

@@ -14,7 +14,7 @@ module TCellAgent
               if appsensor_policy
                 tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
                 if tcell_data
-                  appsensor_policy.csrf_rejected(tcell_data)
+                  appsensor_policy.csrf_rejected(tcell_data, ActionController::InvalidAuthenticityToken)
                 end
               end
             end