tcell_agent 0.2.21 → 0.2.22

data/lib/tcell_agent/policies/appsensor/nullbyte_sensor.rb

@@ -16,6 +16,10 @@ module TCellAgent
         @rule_manager.get_ruleset_for("nullbyte")
       end
 
+      def applicable_for_param_type?(param_type)
+        InjectionSensor::COOKIE_PARAM != param_type
+      end
+
     end
 
   end

data/lib/tcell_agent/policies/appsensor/payloads_policy.rb

@@ -1,3 +1,5 @@
+require 'json'
+
 require 'tcell_agent/utils/params'
 
 module TCellAgent
@@ -91,7 +93,7 @@ module TCellAgent
                 pattern
               )
               event.post_process
-              TCellAgent.appfirewall_payloads_logger.info(event.to_json)
+              TCellAgent.appfirewall_payloads_logger.info(JSON.dump(event))
             end
           end
         end

data/lib/tcell_agent/policies/appsensor/response_codes_sensor.rb

@@ -1,10 +1,10 @@
-require 'tcell_agent/policies/appsensor/sensor'
+require 'tcell_agent/appsensor/sensor'
 
 
 module TCellAgent
   module Policies
 
-    class ResponseCodesSensor < Sensor
+    class ResponseCodesSensor
 
       RESPONSE_CODE_DP_DICT = {
         401 => "s401",
@@ -53,7 +53,7 @@ module TCellAgent
         if dp
           param = payload = pattern = nil
           meta = { code: response_code }
-          send_event(appsensor_meta, dp, param, meta, payload, pattern)
+          TCellAgent::AppSensor::Sensor.send_event(appsensor_meta, dp, param, meta, payload, pattern)
         end
 
         def to_s

data/lib/tcell_agent/policies/appsensor/retr_sensor.rb

@@ -12,6 +12,10 @@ module TCellAgent
         )
       end
 
+      def applicable_for_param_type?(param_type)
+        InjectionSensor::POST_PARAM != param_type && InjectionSensor::JSON_PARAM != param_type
+      end
+
     end
 
   end

data/lib/tcell_agent/policies/appsensor/size_sensor.rb

@@ -1,9 +1,9 @@
-require 'tcell_agent/policies/appsensor/sensor'
+require 'tcell_agent/appsensor/sensor'
 
 module TCellAgent
   module Policies
 
-    class SizeSensor < Sensor
+    class SizeSensor
 
       attr_accessor :enabled, :limit, :excluded_route_ids, :dp_code
 
@@ -31,7 +31,13 @@ module TCellAgent
         if content_length && content_length > @limit
           param = payload = pattern = nil
           meta = { "sz" => content_length }
-          send_event(appsensor_meta, @dp_code, param, meta, payload, pattern)
+          TCellAgent::AppSensor::Sensor.send_event(
+            appsensor_meta,
+            @dp_code,
+            param,
+            meta,
+            payload,
+            pattern)
         end
       end
 

data/lib/tcell_agent/policies/appsensor/user_agent_sensor.rb

@@ -1,9 +1,9 @@
-require 'tcell_agent/policies/appsensor/sensor'
+require 'tcell_agent/appsensor/sensor'
 
 module TCellAgent
   module Policies
 
-    class UserAgentSensor < Sensor
+    class UserAgentSensor
       DP_CODE = "uaempty"
 
       attr_accessor :enabled, :empty_enabled, :excluded_route_ids
@@ -30,7 +30,7 @@ module TCellAgent
 
         user_agent = appsensor_meta.user_agent
         if !user_agent || user_agent.strip == ""
-          send_event(appsensor_meta, DP_CODE, nil, nil, nil, nil)
+          TCellAgent::AppSensor::Sensor.send_event(appsensor_meta, DP_CODE, nil, nil, nil, nil)
         end
       end
 

data/lib/tcell_agent/policies/appsensor_policy.rb

@@ -1,8 +1,8 @@
 require 'tcell_agent/instrumentation'
+require 'tcell_agent/appsensor/injections_reporter'
 require 'tcell_agent/policies/appsensor/cmdi_sensor'
 require 'tcell_agent/policies/appsensor/database_sensor'
 require 'tcell_agent/policies/appsensor/fpt_sensor'
-require 'tcell_agent/policies/appsensor/login_sensor'
 require 'tcell_agent/policies/appsensor/misc_sensor'
 require 'tcell_agent/policies/appsensor/nullbyte_sensor'
 require 'tcell_agent/policies/appsensor/payloads_policy'
@@ -20,17 +20,9 @@ module TCellAgent
   module Policies
 
     class AppSensorPolicy
-
       DETECTION_POINTS_V1 = [
         "req_res_size",
         "resp_codes",
-        "xss",
-        "sqli",
-        "cmdi",
-        "fpt",
-        "null",
-        "retr",
-        "login_failure",
         "ua",
         "errors",
         "database"]
@@ -45,19 +37,28 @@ module TCellAgent
           "fpt" => FptSensor,
           "nullbyte" => NullbyteSensor,
           "retr" => RetrSensor,
-          "login" => LoginSensor,
           "ua" => UserAgentSensor,
           "errors" => MiscSensor,
           "database" => DatabaseSensor
       }
 
-      attr_accessor :policy_id, :options, :payloads_policy, :enabled
+      DETECTION_POINTS_V2_NON_INJECTION = {
+          "req_size" => RequestSizeSensor,
+          "resp_size" => ResponseSizeSensor,
+          "resp_codes" => ResponseCodesSensor,
+          "ua" => UserAgentSensor,
+          "errors" => MiscSensor,
+          "database" => DatabaseSensor
+      }
+
+      attr_accessor :policy_id, :options, :enabled, :injections_reporter
 
       def initialize
         @policy_id = nil
         @options = Hash.new
         @enabled = false
-        @payloads_policy = PayloadsPolicy.new
+        @injections_reporter =
+          TCellAgent::AppSensor::InjectionsReporter.from_json(0, {}, PayloadsPolicy.new)
       end
 
       def process_meta_event(appsensor_meta)
@@ -67,7 +68,8 @@ module TCellAgent
         check_request_size(appsensor_meta)
         check_response_size(appsensor_meta)
         check_response_code(appsensor_meta)
-        check_params_for_injections(appsensor_meta)
+
+        @injections_reporter.check(appsensor_meta)
       end
 
       def process_db_rows(tcell_data, number_of_records)
@@ -112,62 +114,6 @@ module TCellAgent
         end
       end
 
-      def check_param_for_injections(param_type, appsensor_meta, param_name, param_value)
-        return if @options["xss"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
-        return if @options["sqli"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
-        if InjectionSensor::COOKIE_PARAM != param_type
-          return if @options["cmdi"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
-        end
-        if InjectionSensor::COOKIE_PARAM != param_type
-          return if @options["fpt"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
-        end
-        if InjectionSensor::COOKIE_PARAM != param_type
-          return if @options["nullbyte"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
-        end
-        if InjectionSensor::POST_PARAM != param_type && InjectionSensor::JSON_PARAM != param_type
-          return if @options["retr"].check(param_type, appsensor_meta, param_name, param_value, @payloads_policy)
-        end
-      end
-
-      def check_params_for_injections(appsensor_meta)
-
-        TCellAgent::Utils::Params.flatten(appsensor_meta.path_parameters || {}).each do |param_name, param_value|
-          TCellAgent::Instrumentation.safe_block("AppSensor Check Path Params injections") do
-            param_name = param_name[-1]
-            next if param_name == :controller || param_name == :action
-            check_param_for_injections(InjectionSensor::URI_PARAM, appsensor_meta, param_name.to_s, param_value)
-          end
-        end
-
-        TCellAgent::Utils::Params.flatten(appsensor_meta.get_dict || {}).each do |param_name, param_value|
-          TCellAgent::Instrumentation.safe_block("AppSensor Check GET var injections") do
-            param_name = param_name[-1]
-            check_param_for_injections(InjectionSensor::GET_PARAM, appsensor_meta, param_name, param_value)
-          end
-        end
-
-        (appsensor_meta.post_dict || {}).each do |param_name, param_value|
-          TCellAgent::Instrumentation.safe_block("AppSensor Check POST var injections") do
-            param_name = param_name[-1]
-            check_param_for_injections(InjectionSensor::POST_PARAM, appsensor_meta, param_name, param_value)
-          end
-        end
-
-        (appsensor_meta.body_dict || {}).each do |param_name, param_value|
-          TCellAgent::Instrumentation.safe_block("AppSensor Check JSON var injections") do
-            param_name = param_name[-1]
-            check_param_for_injections(InjectionSensor::JSON_PARAM, appsensor_meta, param_name, param_value)
-          end
-        end
-
-        TCellAgent::Utils::Params.flatten(appsensor_meta.cookie_dict || {}).each do |param_name, param_value|
-          TCellAgent::Instrumentation.safe_block("AppSensor Check COOKIE var injections") do
-            param_name = param_name[-1]
-            check_param_for_injections(InjectionSensor::COOKIE_PARAM, appsensor_meta, param_name, param_value)
-          end
-        end
-      end
-
       def csrf_rejected(tcell_data, exception_class)
         TCellAgent::Instrumentation.safe_block("AppSensor CSRF Exception processing") do
           if self.options.has_key?("errors")
@@ -198,9 +144,6 @@ module TCellAgent
         if policy_json.has_key?("data")
           data_json = policy_json["data"]
 
-          rule_manager = AppSensorRuleManager.new
-          rule_manager.load_default_rules_file()
-
           if policy_json["version"] && policy_json["version"] == 2
             if data_json
               sensors_json = data_json.fetch("sensors", {})
@@ -209,16 +152,21 @@ module TCellAgent
 
               else
                 sensor_policy.enabled = true
-                sensor_policy.payloads_policy = PayloadsPolicy.from_json(
-                  data_json.fetch("options", {})
-                )
 
-                DETECTION_POINTS_V2.each do |sensor_name, sensor_class|
+                DETECTION_POINTS_V2_NON_INJECTION.each do |sensor_name, sensor_class|
                   settings = sensors_json.fetch(sensor_name, {})
-                  settings["enabled"] = sensors_json.has_key?(sensor_name)
-                  settings["rule_manager"] = rule_manager
-                  sensor_policy.options[sensor_name] = sensor_class.new(settings)
+                  updated_settings = {
+                    "enabled" => sensors_json.has_key?(sensor_name)
+                  }.merge(settings)
+
+                  sensor_policy.options[sensor_name] = sensor_class.new(updated_settings)
                 end
+
+                payloads_policy = PayloadsPolicy.from_json(
+                  data_json.fetch("options", {})
+                )
+                sensor_policy.injections_reporter =
+                  TCellAgent::AppSensor::InjectionsReporter.from_json(2, sensors_json, payloads_policy)
               end
             end
 
@@ -231,64 +179,40 @@ module TCellAgent
 
               else
                 sensor_policy.enabled = true
-                sensor_policy.payloads_policy = PayloadsPolicy.from_json({
+
+                payloads_policy = PayloadsPolicy.from_json({
                   "payloads" => {
                     "send_payloads" => true,
                     "log_payloads" => true
                   }
                 })
 
-                DETECTION_POINTS_V1.each do |sensor_name|
-                  if "req_res_size" == sensor_name
-                    enabled = options_json.fetch(sensor_name, false)
-                    sensor_policy.options["req_size"] = RequestSizeSensor.new({"enabled" => enabled})
-                    sensor_policy.options["resp_size"] = ResponseSizeSensor.new({"enabled" => enabled})
-
-                  elsif "resp_codes" == sensor_name
-                      enabled = options_json.fetch(sensor_name, false)
-                      sensor_policy.options[sensor_name] = ResponseCodesSensor.new({
-                          "enabled" => enabled,
-                          "series_400_enabled" => true,
-                          "series_500_enabled" => true})
-
-                  elsif "null" == sensor_name
-                    enabled = options_json.fetch(sensor_name, false)
-                    sensor_policy.options["nullbyte"] = NullbyteSensor.new(
-                      {
-                        "enabled" => enabled,
-                        "v1_compatability_enabled" => true,
-                        "rule_manager" => rule_manager
-                      }
-                    )
-
-                  elsif "login_failure" == sensor_name
-                    enabled = options_json.fetch(sensor_name, false)
-                    sensor_policy.options["login"] = LoginSensor.new({"enabled" => enabled})
-
-                  elsif "ua" == sensor_name
-                    sensor_policy.options[sensor_name] = UserAgentSensor.new({
-                      "enabled" => false, "empty_enabled" => false
-                    })
-
-                  elsif "errors" == sensor_name
-                    sensor_policy.options[sensor_name] = MiscSensor.new({
-                      "enabled" => false,
-                      "csrf_exception_enabled" => false,
-                      "sql_exception_enabled" => false
-                    })
-
-                  else
-                    enabled = options_json.fetch(sensor_name, false)
-                    clazz = DETECTION_POINTS_V2[sensor_name]
-                    sensor_policy.options[sensor_name] = clazz.new(
-                      {
-                        "enabled" => enabled,
-                        "v1_compatability_enabled" => true,
-                        "rule_manager" => rule_manager
-                      }
-                    )
-                  end
-                end
+                sensor_policy.injections_reporter =
+                  TCellAgent::AppSensor::InjectionsReporter.from_json(1, data_json, payloads_policy)
+
+                enabled = options_json.fetch("req_res_size", false)
+                sensor_policy.options["req_size"] = RequestSizeSensor.new({"enabled" => enabled})
+                sensor_policy.options["resp_size"] = ResponseSizeSensor.new({"enabled" => enabled})
+
+                enabled = options_json.fetch("resp_codes", false)
+                sensor_policy.options["resp_codes"] = ResponseCodesSensor.new({
+                    "enabled" => enabled,
+                    "series_400_enabled" => true,
+                    "series_500_enabled" => true})
+
+                sensor_policy.options["ua"] = UserAgentSensor.new({
+                  "enabled" => false, "empty_enabled" => false
+                })
+
+                sensor_policy.options["errors"] = MiscSensor.new({
+                  "enabled" => false,
+                  "csrf_exception_enabled" => false,
+                  "sql_exception_enabled" => false
+                })
+
+                sensor_policy.options["database"] = DatabaseSensor.new({
+                  "enabled" => false
+                })
               end
             end
           end

data/lib/tcell_agent/policies/content_security_policy.rb

@@ -6,150 +6,161 @@ require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/configuration'
 
 module TCellAgent
-    module Policies
-        class ContentSecurityPolicy
-            class ContentSecurityPolicyHeader
-                @@approved_headers = [
-                    "csp",
-                    "csp-report"
-                ]
-                attr_accessor :type
-                attr_accessor :raw_value
-                attr_accessor :report_uri
-                attr_accessor :policy_id
-                def initialize(type, value, report_uri=nil, policy_id=nil)
-                    if !(type && value)
-                        raise "Type and value were not set"
-                    end
-                    if type.downcase == "content-security-policy"
-                        type = "csp"
-                    elsif type.downcase == "content-security-policy-report-only"
-                        type = "csp-report"
-                    end
-                    if not @@approved_headers.include?(type.downcase)
-                        raise "Type was not included in approved_headers"
-                    end
-                    if value != value.gsub(/[^\p{L}\w\d\-_\ :\/,;.'\*"%?@#=$]/,'')
-                        raise "Value is not valid"
-                    end
-                    if policy_id
-                        self.policy_id = policy_id
-                    end
-                    self.type = type
-                    self.raw_value = value
-                    self.report_uri = report_uri
-                end
-                def self.jhash(str)
-                  str.each_char.reduce(0) do |result, char|
-                    [((result << 5) - result) + char.ord].pack('L').unpack('l').first
-                  end 
-                end
-                def value(transaction_id=nil, route_id=nil, session_id=nil, user_id=nil)
-                    if !self.report_uri
-                        return self.raw_value
-                    end
-                    begin
-                        uri = URI.parse(self.report_uri)
-                        new_query_ar = URI.decode_www_form(uri.query || '')
-                        if transaction_id
-                            new_query_ar << ["tid", transaction_id]
-                        end
-                        if session_id and session_id.length > 0
-                            new_query_ar << ["sid", session_id]
-                        end
-                        if route_id
-                            new_query_ar << ["rid", route_id]
-                        end
-                        if new_query_ar != []
-                            uri.query = URI.encode_www_form(new_query_ar)
-                        end
-                        report_uri = uri.to_s
-                        if self.policy_id
-                            checksum = ContentSecurityPolicyHeader.jhash(self.policy_id + report_uri)
-                            if new_query_ar != []
-                                report_uri = report_uri + "&"
-                            else
-                                report_uri = report_uri + "?"
-                            end
-                            report_uri = report_uri + "c=" + checksum.to_s
-                        end
-                        return "#{self.raw_value}; report-uri #{report_uri}"
-                    rescue Exception=>e
-                        return self.raw_value
-                    end
-                end
-            end
+  module Policies
 
-            attr_accessor :headers
-            attr_accessor :policy_id
-            attr_accessor :js_agent_api_key
-
-            def each(transaction_id=nil, route_id=nil, hmac_session_id=nil, user_id=nil, &block)
-                result = []
-                headers.each do | header |
-                  header_value = header.value(transaction_id, route_id, hmac_session_id)
-                  header_names = ContentSecurityPolicy.cspHeadersForType(header.type)
-                  header_names.each do | header_name |
-                    result.push( {"name"=>header_name, "value"=>header_value} )
-                  end #doloop
-                end
-                result.each(&block)
+    class ContentSecurityPolicy
+      class ContentSecurityPolicyHeader
+        @@approved_headers = [
+          "csp",
+          "csp-report"
+        ]
+        attr_accessor :type
+        attr_accessor :raw_value
+        attr_accessor :report_uri
+        attr_accessor :policy_id
+        def initialize(type, value, report_uri=nil, policy_id=nil)
+          if !(type && value)
+            raise "Type and value were not set"
+          end
+          if type.downcase == "content-security-policy"
+            type = "csp"
+          elsif type.downcase == "content-security-policy-report-only"
+            type = "csp-report"
+          end
+          if not @@approved_headers.include?(type.downcase)
+            raise "Type was not included in approved_headers"
+          end
+          if value != value.gsub(/[^\p{L}\w\d\-_\ :\/,;.'\*"%?@#=$]/,'')
+            raise "Value is not valid"
+          end
+          if policy_id
+            self.policy_id = policy_id
+          end
+          self.type = type
+          self.raw_value = value
+          self.report_uri = report_uri
+        end
+        def self.jhash(str)
+          str.each_char.reduce(0) do |result, char|
+            [((result << 5) - result) + char.ord].pack('L').unpack('l').first
+          end
+        end
+        def value(transaction_id=nil, route_id=nil, session_id=nil, user_id=nil)
+          if !self.report_uri
+            return self.raw_value
+          end
+          begin
+            uri = URI.parse(self.report_uri)
+            new_query_ar = URI.decode_www_form(uri.query || '')
+            if transaction_id
+              new_query_ar << ["tid", transaction_id]
             end
-            def self.from_json(policy_json)
-                if (!policy_json) 
-                    return nil
-                end
-                csp = ContentSecurityPolicy.new
-                if policy_json.has_key?("policy_id")
-                    csp.policy_id = policy_json["policy_id"]
-                else
-                    raise "Policy ID missing"
-                end
-                if policy_json.has_key?("data")
-                    data_json = policy_json["data"]
-                    if data_json.has_key?("options")
-                        options_json = data_json["options"]
-                        csp.js_agent_api_key = options_json.fetch("js_agent_api_key", nil)
-                    end
-                end
-                if policy_json.has_key?("headers")
-                    headers = policy_json["headers"]
-                    csp_headers = []
-                    headers.each do |header|
-                        if header.has_key?("name") && header.has_key?("value")
-                            begin
-                                csp_header = ContentSecurityPolicyHeader.new(header["name"], header["value"], header["report-uri"], csp.policy_id)
-                                csp_headers.push(csp_header)
-                            rescue Exception => secure_header_exception
-                                #pass
-                            end
-                        end
-                    end
-                    csp.headers = csp_headers
-                end
-                return csp
+            if session_id and session_id.length > 0
+              new_query_ar << ["sid", session_id]
             end
-            def self.cspHeadersForType(csp_type)
-                if (!csp_type)
-                    return []
-                end
-                if csp_type == "csp"
-                    return ["Content-Security-Policy"]#,"X-Content-Security-Policy","X-WebKit-CSP"]
-                elsif csp_type == "csp-report"
-                    return ["Content-Security-Policy-Report-Only"]#,"X-Content-Security-Policy-Report-Only","X-WebKit-CSP-Report-Only"]
-                else
-                    return []
-                end
+            if route_id
+              new_query_ar << ["rid", route_id]
             end
-            def js_agent_app_id
-                return TCellAgent.configuration.app_id
+            if new_query_ar != []
+              uri.query = URI.encode_www_form(new_query_ar)
             end
-            def js_agent_api_base_url
-                return TCellAgent.configuration.js_agent_api_base_url
+            report_uri = uri.to_s
+            if self.policy_id
+              checksum = ContentSecurityPolicyHeader.jhash(self.policy_id + report_uri)
+              if new_query_ar != []
+                report_uri = report_uri + "&"
+              else
+                report_uri = report_uri + "?"
+              end
+              report_uri = report_uri + "c=" + checksum.to_s
             end
-            def js_agent_url
-                return TCellAgent.configuration.js_agent_url
+            return "#{self.raw_value}; report-uri #{report_uri}"
+          rescue Exception
+            return self.raw_value
+          end
+        end
+      end
+
+      attr_accessor :headers
+      attr_accessor :policy_id
+      attr_accessor :js_agent_api_key
+
+      def each(transaction_id=nil, route_id=nil, hmac_session_id=nil, user_id=nil, &block)
+        result = []
+        headers.each do | header |
+          header_value = header.value(transaction_id, route_id, hmac_session_id)
+          header_names = ContentSecurityPolicy.cspHeadersForType(header.type)
+          header_names.each do | header_name |
+            result.push( {"name"=>header_name, "value"=>header_value} )
+          end #doloop
+        end
+        result.each(&block)
+      end
+      def self.from_json(policy_json)
+        if (!policy_json)
+          return nil
+        end
+        csp = ContentSecurityPolicy.new
+        if policy_json.has_key?("policy_id")
+          csp.policy_id = policy_json["policy_id"]
+        else
+          raise "Policy ID missing"
+        end
+
+        if policy_json.has_key?("data")
+          data_json = policy_json["data"]
+          if data_json.has_key?("options")
+            options_json = data_json["options"]
+            csp.js_agent_api_key = options_json.fetch("js_agent_api_key", nil)
+          end
+        end
+
+        if policy_json.has_key?("headers")
+          TCellAgent.configuration.js_agent_url = TCellAgent.configuration.startup_js_agent_url
+
+          headers = policy_json["headers"]
+          csp_headers = []
+
+
+          headers.each do |header|
+            if header.has_key?("name") && header.has_key?("value")
+              begin
+                if TCellAgent.configuration.startup_js_agent_url == "https://api.tcell.io/tcellagent.min.js" && header["value"] =~ /https:\/\/jsagent.tcell.io/
+                  TCellAgent.configuration.js_agent_url = "https://jsagent.tcell.io/tcellagent.min.js"
+                end
+
+                csp_header = ContentSecurityPolicyHeader.new(header["name"], header["value"], header["report-uri"], csp.policy_id)
+                csp_headers.push(csp_header)
+              rescue
+              end
             end
+          end
+          csp.headers = csp_headers
         end
+        return csp
+      end
+      def self.cspHeadersForType(csp_type)
+        if (!csp_type)
+          return []
+        end
+        if csp_type == "csp"
+          return ["Content-Security-Policy"]#,"X-Content-Security-Policy","X-WebKit-CSP"]
+        elsif csp_type == "csp-report"
+          return ["Content-Security-Policy-Report-Only"]#,"X-Content-Security-Policy-Report-Only","X-WebKit-CSP-Report-Only"]
+        else
+          return []
+        end
+      end
+      def js_agent_app_id
+        return TCellAgent.configuration.app_id
+      end
+      def js_agent_api_base_url
+        return TCellAgent.configuration.js_agent_api_base_url
+      end
+      def js_agent_url
+        return TCellAgent.configuration.js_agent_url
+      end
     end
+
+  end
 end