tcell_agent 0.2.29 → 0.4.0

data/lib/tcell_agent/appsensor/sensor.rb

@@ -1,52 +0,0 @@
-require 'tcell_agent/sensor_events/appsensor_event'
-
-module TCellAgent
-  module AppSensor
-
-    class Sensor
-      class << self
-        def send_event(appsensor_meta, detection_point, parameter, meta,
-                       payload, pattern, collect_full_uri)
-          event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
-            appsensor_meta.location,
-            detection_point,
-            appsensor_meta.method,
-            appsensor_meta.remote_address,
-            parameter,
-            appsensor_meta.route_id,
-            meta,
-            appsensor_meta.session_id,
-            appsensor_meta.user_id,
-            payload,
-            pattern,
-            collect_full_uri
-          )
-
-          TCellAgent.send_event(event)
-        end
-
-        def send_event_from_tcell_data(tcell_data, detection_point, parameter,
-                                       meta, collect_full_uri)
-          payload = pattern = nil
-          event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
-            tcell_data.uri,
-            detection_point,
-            tcell_data.request_method,
-            tcell_data.ip_address,
-            parameter,
-            tcell_data.route_id,
-            meta,
-            tcell_data.session_id,
-            tcell_data.user_id,
-            payload,
-            pattern,
-            collect_full_uri
-          )
-
-          TCellAgent.send_event(event)
-        end
-      end
-    end
-
-  end
-end

data/lib/tcell_agent/policies/appsensor/database_sensor.rb

@@ -1,56 +0,0 @@
-require 'tcell_agent/appsensor/sensor'
-
-module TCellAgent
-  module Policies
-
-    class DatabaseSensor
-
-      DP_CODE="dbmaxrows"
-
-      attr_accessor :enabled, :max_rows, :excluded_route_ids, :collect_full_uri
-
-      def initialize(policy_json=nil)
-        @enabled = false
-        @max_rows = 1001
-        @excluded_route_ids = {}
-        @collect_full_uri = false
-
-        if policy_json
-          @enabled = policy_json.fetch("enabled", false)
-          large_result = policy_json.fetch("large_result", {})
-          @max_rows = large_result.fetch("limit", @max_rows)
-          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
-
-          policy_json.fetch("exclude_routes", []).each do |excluded_route|
-            @excluded_route_ids[excluded_route] = true
-          end
-        end
-      end
-
-
-      def check(tcell_data, number_of_records)
-        return unless @enabled
-
-        return if @excluded_route_ids.fetch(tcell_data.route_id, false)
-
-        if number_of_records > @max_rows
-          param = nil
-          meta = { "rows" => number_of_records }
-          TCellAgent::AppSensor::Sensor.send_event_from_tcell_data(
-            tcell_data,
-            DP_CODE,
-            param,
-            meta,
-            @collect_full_uri)
-        end
-      end
-
-      def to_s
-        "<#{self.class.name} enabled: #{@enabled} max_rows: #{@max_rows} " +
-        "excluded_route_ids: #{@excluded_route_ids}>"
-      end
-
-    end
-
-  end
-end

data/lib/tcell_agent/policies/appsensor/misc_sensor.rb

@@ -1,59 +0,0 @@
-require 'tcell_agent/appsensor/sensor'
-
-module TCellAgent
-  module Policies
-
-    class MiscSensor
-
-      attr_accessor :enabled, :csrf_exception_enabled, :sql_exception_enabled,
-                    :excluded_route_ids, :collect_full_uri
-
-      def initialize(policy_json=nil)
-        @enabled = false
-        @csrf_exception_enabled = false
-        @sql_exception_enabled = false
-        @excluded_route_ids = {}
-        @collect_full_uri = false
-
-        if policy_json
-          @enabled = policy_json.fetch("enabled", false)
-          @csrf_exception_enabled = policy_json.fetch("csrf_exception_enabled", false)
-          @sql_exception_enabled = policy_json.fetch("sql_exception_enabled", false)
-          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
-
-          policy_json.fetch("exclude_routes", []).each do |excluded_route|
-            @excluded_route_ids[excluded_route] = true
-          end
-        end
-      end
-
-      def csrf_rejected(tcell_data, exception_class)
-        return unless @enabled && @csrf_exception_enabled
-
-        return if tcell_data && @excluded_route_ids.fetch(tcell_data.route_id, false)
-
-        meta = nil
-        TCellAgent::AppSensor::Sensor.send_event_from_tcell_data(
-          tcell_data, "excsrf", exception_class.name, meta, @collect_full_uri
-        )
-      end
-
-      def sql_exception_detected(tcell_data, exception)
-        return unless @enabled && @sql_exception_enabled
-
-        return if tcell_data && @excluded_route_ids.fetch(tcell_data.route_id, false)
-
-        meta = nil
-        TCellAgent::AppSensor::Sensor.send_event_from_tcell_data(
-          tcell_data, "exsql", exception.class.name, meta, @collect_full_uri
-        )
-      end
-
-      def to_s
-        "<#{self.class.name} enabled: #{@enabled} csrf_exception_enabled: #{@csrf_exception_enabled} " +
-        "sql_exception_enabled: #{sql_exception_enabled}>"
-      end
-    end
-
-  end
-end

data/lib/tcell_agent/policies/appsensor/payloads_policy.rb

@@ -1,150 +0,0 @@
-require 'json'
-
-require 'tcell_agent/utils/params'
-
-module TCellAgent
-  module Policies
-
-    class PayloadsPolicy
-      PARAM_TYPE_MAP = {
-        TCellAgent::Utils::Params::GET_PARAM => "form",
-        TCellAgent::Utils::Params::POST_PARAM => "form",
-        TCellAgent::Utils::Params::JSON_PARAM => "form",
-        TCellAgent::Utils::Params::COOKIE_PARAM => "cookie"
-      }
-
-      attr_accessor :send_payloads, :send_blacklist, :send_whitelist, :use_send_whitelist,
-                    :log_payloads, :log_blacklist, :log_whitelist, :use_log_whitelist,
-                    :collect_full_uri
-
-      def initialize
-        @send_payloads = false
-        @log_payloads = false
-        @collect_full_uri = false
-
-        @send_blacklist = {}
-        @log_blacklist = {}
-        @send_whitelist = {}
-        @log_whitelist = {}
-
-        @use_send_whitelist = false
-        @use_log_whitelist = false
-      end
-
-      def apply(dp, appsensor_meta, type_of_param, vuln_param, vuln_value, meta, pattern)
-        payload = nil
-
-        if @send_payloads && TCellAgent.configuration.allow_unencrypted_appfirewall_payloads
-
-          blacklisted_locations = @send_blacklist[vuln_param.downcase]
-          param_location = PARAM_TYPE_MAP[type_of_param]
-
-          if blacklisted_locations &&
-              ( blacklisted_locations.include?(param_location) ||
-               blacklisted_locations.include?("*") )
-            payload = "BLACKLISTED"
-
-          elsif use_send_whitelist
-            whitelisted_locations = @send_whitelist[vuln_param.downcase]
-            if whitelisted_locations &&
-               ( whitelisted_locations.include?(param_location) ||
-                whitelisted_locations.include?("*") )
-
-              payload = vuln_value
-
-            else
-              payload = "NOT_WHITELISTED"
-            end
-
-          else
-            payload = vuln_value
-          end
-
-        end
-
-        log(dp, appsensor_meta, type_of_param, vuln_param, vuln_value, meta, pattern)
-
-        payload
-      end
-
-      def log(dp, appsensor_meta, type_of_param, vuln_param, vuln_value, meta, pattern)
-        if @log_payloads
-          blacklisted_locations = @log_blacklist[vuln_param.downcase]
-          param_location = PARAM_TYPE_MAP[type_of_param]
-
-          if !blacklisted_locations ||
-              ( !blacklisted_locations.include?(param_location) &&
-               !blacklisted_locations.include?("*") )
-
-            whitelisted_locations = @log_whitelist[vuln_param.downcase]
-            if !use_log_whitelist ||
-                (whitelisted_locations && (whitelisted_locations.include?(param_location) ||
-                whitelisted_locations.include?("*")))
-
-              event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
-                appsensor_meta.location,
-                dp,
-                appsensor_meta.method,
-                appsensor_meta.remote_address,
-                vuln_param,
-                appsensor_meta.route_id,
-                meta,
-                appsensor_meta.session_id,
-                appsensor_meta.user_id,
-                vuln_value,
-                pattern,
-                @collect_full_uri
-              )
-              event.post_process
-              TCellAgent.appfirewall_payloads_logger.info(JSON.dump(event))
-            end
-          end
-        end
-      end
-
-      def self.from_json(policy_json)
-        policy = PayloadsPolicy.new
-
-        if policy_json
-          policy.collect_full_uri = policy_json.fetch("uri_options", {}).fetch("collect_full_uri", false)
-
-          payloads_json = policy_json.fetch("payloads", {})
-          policy.send_payloads = payloads_json.fetch("send_payloads", false)
-          policy.log_payloads = payloads_json.fetch("log_payloads", false)
-
-          if policy.send_payloads
-            payloads_json.fetch("send_blacklist", {}).each do |param_name, locations|
-              policy.send_blacklist[param_name.downcase] = Set.new(locations)
-            end
-
-            send_whitelist = payloads_json["send_whitelist"]
-            if send_whitelist
-              send_whitelist.each do |param_name, locations|
-                policy.send_whitelist[param_name.downcase] = Set.new(locations)
-              end
-              policy.use_send_whitelist = true
-            end
-          end
-
-          if policy.log_payloads
-            payloads_json.fetch("log_blacklist", {}).each do |param_name, locations|
-              policy.log_blacklist[param_name.downcase] = Set.new(locations)
-            end
-
-            log_whitelist = payloads_json["log_whitelist"]
-            if log_whitelist
-              log_whitelist.each do |param_name, locations|
-                policy.log_whitelist[param_name.downcase] = Set.new(locations)
-              end
-
-              policy.use_log_whitelist = true
-            end
-          end
-        end
-
-        policy
-      end
-    end
-
-  end
-end

data/lib/tcell_agent/policies/appsensor/request_size_sensor.rb

@@ -1,25 +0,0 @@
-require 'tcell_agent/policies/appsensor/size_sensor'
-
-
-module TCellAgent
-  module Policies
-
-    class RequestSizeSensor < SizeSensor
-      MAX_NORMAL_REQUEST_BYTES = 1024*512
-      DP_UNUSUAL_REQUEST_SIZE = "reqsz"
-
-      def initialize(policy_json=nil)
-        super(
-          MAX_NORMAL_REQUEST_BYTES,
-          DP_UNUSUAL_REQUEST_SIZE,
-          policy_json
-        )
-      end
-
-      def get_content_length(appsensor_meta)
-        appsensor_meta.request_content_bytes_len
-      end
-    end
-
-  end
-end

data/lib/tcell_agent/policies/appsensor/response_codes_sensor.rb

@@ -1,73 +0,0 @@
-require 'tcell_agent/appsensor/sensor'
-
-
-module TCellAgent
-  module Policies
-
-    class ResponseCodesSensor
-
-      RESPONSE_CODE_DP_DICT = {
-        401 => "s401",
-        403 => "s403",
-        404 => "s404",
-        4 => "s4xx",
-        500 => "s500",
-        5 => "s5xx"
-      }
-
-      attr_accessor :enabled, :series_400_enabled, :series_500_enabled, :excluded_route_ids,
-                    :collect_full_uri
-
-      def initialize(policy_json=nil)
-        @enabled = false
-        @series_400_enabled = false
-        @series_500_enabled = false
-        @excluded_route_ids = {}
-        @collect_full_uri = false
-
-        if policy_json
-          @enabled = policy_json.fetch("enabled", false)
-          @series_400_enabled = policy_json.fetch("series_400_enabled", false)
-          @series_500_enabled = policy_json.fetch("series_500_enabled", false)
-          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
-
-          policy_json.fetch("exclude_routes", []).each do |excluded_route|
-            @excluded_route_ids[excluded_route] = true
-          end
-        end
-      end
-
-      def check(appsensor_meta, response_code)
-        return unless @enabled
-
-        return if @excluded_route_ids.fetch(appsensor_meta.route_id, false)
-
-        return if response_code == 200
-        return if !self.series_400_enabled && (response_code >= 400 && response_code < 500)
-        return if !self.series_500_enabled and (response_code >= 500 && response_code < 600)
-
-        dp = RESPONSE_CODE_DP_DICT.fetch(response_code, nil)
-
-        if dp.nil?
-          code_series = (response_code / 100).to_i
-          dp = RESPONSE_CODE_DP_DICT.fetch(code_series, nil)
-        end
-
-        if dp
-          param = payload = pattern = nil
-          meta = { code: response_code }
-          TCellAgent::AppSensor::Sensor.send_event(
-            appsensor_meta, dp, param, meta, payload, pattern, @collect_full_uri
-          )
-        end
-
-        def to_s
-          "<#{self.class.name} enabled: #{@enabled} series_400_enabled: #{@series_400_enabled} " +
-          "series_500_enabled: #{@series_500_enabled}>"
-        end
-      end
-
-    end
-
-  end
-end

data/lib/tcell_agent/policies/appsensor/response_size_sensor.rb

@@ -1,25 +0,0 @@
-require 'tcell_agent/policies/appsensor/size_sensor'
-
-
-module TCellAgent
-  module Policies
-
-    class ResponseSizeSensor < SizeSensor
-      MAX_NORMAL_RESPONSE_BYTES = 1024*1024*2
-      DP_UNUSUAL_RESPONSE_SIZE = "rspsz"
-
-      def initialize(policy_json=nil)
-        super(
-          MAX_NORMAL_RESPONSE_BYTES,
-          DP_UNUSUAL_RESPONSE_SIZE,
-          policy_json
-        )
-      end
-
-      def get_content_length(appsensor_meta)
-        appsensor_meta.response_content_bytes_len
-      end
-    end
-
-  end
-end

data/lib/tcell_agent/policies/appsensor/size_sensor.rb

@@ -1,71 +0,0 @@
-require 'tcell_agent/appsensor/sensor'
-
-module TCellAgent
-  module Policies
-
-    class SizeSensor
-
-      attr_accessor :enabled, :limit, :excluded_route_ids, :dp_code, :collect_full_uri
-
-      def initialize(default_limit, dp_code, policy_json)
-        @enabled = false
-        @limit = default_limit
-        @excluded_route_ids = {}
-        @dp_code = dp_code
-        @collect_full_uri = false
-
-        if policy_json
-          @enabled = policy_json.fetch("enabled", false)
-          @limit = policy_json.fetch("limit", @limit)
-          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
-
-          policy_json.fetch("exclude_routes", []).each do |route_id|
-            @excluded_route_ids[route_id] = true
-          end
-        end
-      end
-
-      def get_content_length(appsensor_meta)
-        throw Exception("Not Implemented")
-      end
-
-      def check(appsensor_meta)
-        if !@enabled || @excluded_route_ids.fetch(appsensor_meta.route_id, false)
-          return
-        end
-
-        content_length_bytes = get_content_length(appsensor_meta)
-        content_length_KiB = convert_to_kibibytes(content_length_bytes)
-
-        if content_length_KiB > @limit
-          param = payload = pattern = nil
-          meta = { "sz" => content_length_bytes }
-          TCellAgent::AppSensor::Sensor.send_event(
-            appsensor_meta,
-            @dp_code,
-            param,
-            meta,
-            payload,
-            pattern,
-            @collect_full_uri
-          )
-        end
-      end
-
-      def convert_to_kibibytes(content_length)
-        if content_length
-          content_length / 1024.0
-        else
-          0
-        end
-      end
-
-      def to_s
-        "<#{self.class.name} enabled: #{@enabled} limit: #{@limit} dp_code: #{@dp_code} " +
-        "excluded_route_ids: #{@excluded_route_ids}>"
-      end
-
-    end
-
-  end
-end

data/lib/tcell_agent/policies/appsensor/user_agent_sensor.rb

@@ -1,47 +0,0 @@
-require 'tcell_agent/appsensor/sensor'
-
-module TCellAgent
-  module Policies
-
-    class UserAgentSensor
-      DP_CODE = "uaempty"
-
-      attr_accessor :enabled, :empty_enabled, :excluded_route_ids, :collect_full_uri
-
-      def initialize(policy_json=nil)
-        @enabled = false
-        @empty_enabled = false
-        @excluded_route_ids = {}
-        @collect_full_uri = false
-
-        if policy_json
-          @enabled = policy_json.fetch("enabled", false)
-          @empty_enabled = policy_json.fetch("empty_enabled", false)
-          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
-
-          policy_json.fetch("exclude_routes", []).each do |excluded_route|
-            @excluded_route_ids[excluded_route] = true
-          end
-        end
-      end
-
-      def check(appsensor_meta)
-        return unless @enabled && @empty_enabled
-
-        return if @excluded_route_ids.fetch(appsensor_meta.route_id, false)
-
-        user_agent = appsensor_meta.user_agent
-        if !user_agent || user_agent.strip == ""
-          TCellAgent::AppSensor::Sensor.send_event(
-            appsensor_meta, DP_CODE, nil, nil, nil, nil, @collect_full_uri
-          )
-        end
-      end
-
-      def to_s
-        "<#{self.class.name} enabled: #{@enabled} empty_enabled: #{@empty_enabled} dp_code: #{DP_CODE}>"
-      end
-    end
-
-  end
-end

data/lib/tcell_agent/rails/auth/hooks.rb

@@ -1,79 +0,0 @@
-require 'tcell_agent/agent'
-require 'tcell_agent/sensor_events/login_fraud'
-require 'tcell_agent/policies/appsensor_policy'
-require 'tcell_agent/sensor_events/login_fraud'
-
-if defined?(TCellAgent::Hooks::V1::Frameworks::Rails::Login)
-  TCellAgent::Hooks::V1::Frameworks::Rails::Login.module_eval do
-    class << self
-      alias_method :tcell_register_login_event, :register_login_event
-      def register_login_event(status, rails_request, user_id, user_valid=nil)
-        TCellAgent::Instrumentation.safe_block("Rails Auth Hooks") do
-          if (TCellAgent.configuration.enabled && TCellAgent.configuration.should_intercept_requests?)
-            login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
-            if (login_fraud_policy && login_fraud_policy.enabled && login_fraud_policy.login_failed_enabled)
-              tcell_data = rails_request.env[TCellAgent::Instrumentation::TCELL_ID]
-              if tcell_data
-                if status == TCellAgent::Hooks::V1::Login::LOGIN_FAILURE
-                  TCellAgent.send_event(
-                    TCellAgent::SensorEvents::LoginFailure.new(rails_request.env, tcell_data, user_id)
-                  )
-                elsif status == TCellAgent::Hooks::V1::Login::LOGIN_SUCCESS
-                  TCellAgent.send_event(
-                    TCellAgent::SensorEvents::LoginSuccess.new(rails_request.env, tcell_data, user_id)
-                  )
-                else
-                  TCellAgent.logger.error("Unkown login status: #{status}")
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end
-
-if defined?(TCellAgent::Hooks::V1::Login)
-  TCellAgent::Hooks::V1::Login.module_eval do
-    class << self
-      alias_method :tcell_register_login_event, :register_login_event
-      def register_login_event(
-          status,
-          session_id,
-          user_agent,
-          referrer,
-          remote_address,
-          header_keys,
-          user_id,
-          document_uri,
-          user_valid=nil)
-        TCellAgent::Instrumentation.safe_block("Login Auth Hooks") do
-          if (TCellAgent.configuration.enabled && TCellAgent.configuration.should_intercept_requests?)
-            login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
-            if (login_fraud_policy && login_fraud_policy.enabled && login_fraud_policy.login_failed_enabled)
-              tcell_data = TCellAgent::Instrumentation::TCellData.new
-              tcell_data.user_agent = user_agent
-              tcell_data.referrer = referrer
-              tcell_data.ip_address = remote_address
-              tcell_data.path = document_uri
-              tcell_data.hmac_session_id = TCellAgent::SensorEvents::Util.hmac(session_id)
-
-              if status == TCellAgent::Hooks::V1::Login::LOGIN_FAILURE
-                TCellAgent.send_event(
-                  TCellAgent::SensorEvents::LoginFailure.new(header_keys, tcell_data, user_id)
-                )
-              elsif status == TCellAgent::Hooks::V1::Login::LOGIN_SUCCESS
-                TCellAgent.send_event(
-                  TCellAgent::SensorEvents::LoginSuccess.new(header_keys, tcell_data, user_id)
-                )
-              else
-                TCellAgent.logger.error("Unkown login status: #{status}")
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/tcell_agent/sensor_events/util/redirect_utils.rb

@@ -1,22 +0,0 @@
-# See the file "LICENSE" for the full license governing this code.
-
-require 'logger'
-require 'cgi'
-require 'uri'
-require 'openssl'
-
-module TCellAgent
-  module SensorEvents
-    module Util
-      def self.wildcardMatch(target, wildcardPattern)
-        escaped = Regexp.escape(wildcardPattern).gsub('\*','.*?')
-        regex = Regexp.new "^#{escaped}$", Regexp::IGNORECASE
-        !!(target =~ regex)
-      end
-      def self.domainFromUrl(url)
-        uri = URI.parse(url)
-        uri.host
-      end
-    end
-  end
-end