tcell_agent 0.2.29 → 0.4.0

data/lib/tcell_agent/cmdi.rb

@@ -0,0 +1,113 @@
+require 'tcell_agent/agent/policy_types'
+require 'tcell_agent/utils/strings'
+
+
+module TCellAgent
+  module Cmdi
+    def self.block_command?(cmd)
+      TCellAgent::Instrumentation.safe_block("Checking Command Injection Policy") do
+        if TCellAgent::Utils::Strings.present?(cmd)
+          command_injection_policy = TCellAgent.policy(TCellAgent::PolicyTypes::CommandInjection)
+          if command_injection_policy && command_injection_policy.enabled
+            request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+            tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
+            return command_injection_policy.block?(cmd, tcell_context)
+          end
+        end
+      end
+
+      return false
+    end
+
+    def self.parse_command(*args)
+      cmd = ""
+
+      TCellAgent::Instrumentation.safe_block("CMDI Parsing *args") do
+        if args.size > 0
+          args_copy = Array.new(args)
+          args_copy.shift if args_copy.first.is_a?(Hash)
+          args_copy.pop if args_copy.last.is_a?(Hash)
+
+          if args_copy.first.is_a?(Array)
+            cmd_n_argv0 = args_copy.shift
+            args_copy.unshift(cmd_n_argv0.first)
+          end
+
+          cmd = args_copy.join(" ")
+        end
+      end
+
+      cmd
+    end
+  end
+end
+
+module Kernel
+  alias_method :tcell_original_backtick, :`
+  def `(cmd)
+    if TCellAgent::Cmdi.block_command?(cmd)
+      raise Errno::ENOENT.new("tCell.io Agent: Command not allowed by policy: #{cmd}")
+    end
+
+    tcell_original_backtick(cmd)
+  end
+
+  alias_method :tcell_original_exec, :exec
+  def exec(*args)
+    cmd = TCellAgent::Cmdi.parse_command(*args)
+    if TCellAgent::Cmdi.block_command?(cmd)
+      raise Errno::ENOENT.new("tCell.io Agent: Command not allowed by policy: #{cmd}")
+    end
+
+    tcell_original_exec(*args)
+  end
+
+  alias_method :tcell_original_system, :system
+  def system(*args)
+    cmd = TCellAgent::Cmdi.parse_command(*args)
+    if TCellAgent::Cmdi.block_command?(cmd)
+      raise Errno::ENOENT.new("tCell.io Agent: Command not allowed by policy: #{cmd}")
+    end
+
+    tcell_original_system(*args)
+  end
+
+  alias_method :tcell_original_spawn, :spawn
+  def spawn(*args)
+    cmd = TCellAgent::Cmdi.parse_command(*args)
+    if TCellAgent::Cmdi.block_command?(cmd)
+      raise Errno::ENOENT.new("tCell.io Agent: Command not allowed by policy: #{cmd}")
+    end
+
+    return tcell_original_spawn(*args)
+  end
+end
+
+class IO
+  class << self
+    alias_method :tcell_original_popen, :popen
+    def popen(*args)
+      if args.size > 0
+        cmd = ""
+
+        TCellAgent::Instrumentation.safe_block("CMDI Parsing popen *args") do
+          args_copy = Array.new(args)
+          args_copy.shift if args_copy.first.is_a?(Hash)
+          args_copy.pop if args_copy.last.is_a?(Hash)
+
+          if args_copy.first.is_a?(String)
+            cmd = args_copy.shift
+          else
+            cmd = TCellAgent::Cmdi.parse_command(*args_copy.shift)
+          end
+        end
+
+        if TCellAgent::Cmdi.block_command?(cmd)
+          raise Errno::ENOENT.new("tCell.io Agent: Command not allowed by policy: #{cmd}")
+        end
+      end
+
+      return tcell_original_popen(*args)
+    end
+  end
+end

data/lib/tcell_agent/config/unknown_options.rb

@@ -21,6 +21,7 @@ module TCellAgent
           "TCELL_AGENT_CONFIG",
           "TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS",
           "TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS",
+          "TCELL_AGENT_ALLOW_PAYLOADS",
           "TCELL_AGENT_HOME_OWNER"])
 
         ENV.keys.each do |environment_key|
@@ -64,6 +65,7 @@ module TCellAgent
                   "event_batch_size_limit",
                   "allow_unencrypted_appsensor_payloads",
                   "allow_unencrypted_appfirewall_payloads",
+                  "allow_payloads",
                   "reverse_proxy",
                   "reverse_proxy_ip_address_header",
                   "demomode",

data/lib/tcell_agent/configuration.rb

@@ -36,8 +36,6 @@ module TCellAgent
       :js_agent_api_base_url,
       :js_agent_url,
       :startup_js_agent_url,
-      :raise_exceptions,
-      :allow_unencrypted_appfirewall_payloads,
       :config_filename,
       :agent_log_dir,
       :max_data_ex_db_records_per_request,
@@ -48,7 +46,8 @@ module TCellAgent
       :log_file_name,
       :log_tag,
       :max_csp_header_bytes,
-      :demomode
+      :demomode,
+      :allow_payloads
 
     attr_accessor :disable_all,
       :enabled,
@@ -145,12 +144,10 @@ module TCellAgent
       @event_batch_size_limit = 50
       @event_time_limit_seconds = 15
 
-      @raise_exceptions = false
-
       @max_data_ex_db_records_per_request = 1000
       @reverse_proxy = true
       @reverse_proxy_ip_address_header = nil
-      @allow_unencrypted_appfirewall_payloads = false
+      @allow_payloads = true
 
       @max_csp_header_bytes = nil
 
@@ -158,14 +155,21 @@ module TCellAgent
       read_config_from_file(@config_filename)
 
       if ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"]
-        puts "tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS is deprecated, please switch to TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS."
+        puts "tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS is deprecated and will be removed in a future release. Please switch to TCELL_AGENT_ALLOW_PAYLOADS."
+      end
+
+      if (ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"])
+        puts "tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS is deprecated and will be removed in a future release. Please switch to TCELL_AGENT_ALLOW_PAYLOADS."
       end
 
       if (ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"] != nil)
-        @allow_unencrypted_appfirewall_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"])
+        @allow_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"])
       end
       if (ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"] != nil)
-        @allow_unencrypted_appfirewall_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"])
+        @allow_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"])
+      end
+      if (ENV["TCELL_AGENT_ALLOW_PAYLOADS"] != nil)
+        @allow_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_PAYLOADS"])
       end
 
       @tcell_api_url ||= "https://api.tcell.io/api/v1"
@@ -213,8 +217,6 @@ module TCellAgent
         @event_batch_size_limit = 2
         @event_time_limit_seconds = 5
       end
-
-      @raise_exceptions = [true, "true", "yes", "1"].include?(ENV["TCELL_RAISE_EXCEPTIONS"])
     end
 
     def read_config_from_file(filename)
@@ -265,15 +267,17 @@ module TCellAgent
 
             @max_csp_header_bytes = app_data.fetch("max_csp_header_bytes", @max_csp_header_bytes)
 
-            @allow_unencrypted_appfirewall_payloads =
-              app_data.fetch('allow_unencrypted_appsensor_payloads', @allow_unencrypted_appfirewall_payloads)
-            @allow_unencrypted_appfirewall_payloads =
-              app_data.fetch('allow_unencrypted_appfirewall_payloads', @allow_unencrypted_appfirewall_payloads)
+            @allow_payloads =
+              app_data.fetch('allow_unencrypted_appsensor_payloads', @allow_payloads)
+            @allow_payloads =
+              app_data.fetch('allow_unencrypted_appfirewall_payloads', @allow_payloads)
+            @allow_payloads =
+              app_data.fetch('allow_payloads', @allow_payloads)
 
             data_exposure = app_data.fetch('data_exposure', {})
             @max_data_ex_db_records_per_request = data_exposure.fetch('max_data_ex_db_records_per_request', @max_data_ex_db_records_per_request)
 
-            @enabled_instrumentations = app_data.fetch('enabled_instrumentations', {})
+            @enabled_instrumentations = app_data.fetch('enabled_instrumentations', @enabled_instrumentations)
 
             @reverse_proxy = app_data.fetch('reverse_proxy', @reverse_proxy)
             @reverse_proxy_ip_address_header = app_data.fetch('reverse_proxy_ip_address_header', @reverse_proxy_ip_address_header)
@@ -315,6 +319,16 @@ module TCellAgent
       end # filename exist
     end #def read
 
+    # old value could be set via initializers, this makes sure those initializers still work
+    # properly
+    def allow_unencrypted_appfirewall_payloads=(val)
+      @allow_payloads = val
+    end
+    # keep this around in case the value was read as well
+    def allow_unencrypted_appfirewall_payloads
+      @allow_payloads
+    end
+
     def log_filename
       @agent_log_dir ||= File.join(@agent_home_dir, "logs")
       File.join(@agent_log_dir, @log_file_name)

data/lib/tcell_agent/hooks/login_fraud.rb

@@ -0,0 +1,79 @@
+require 'tcell_agent/agent'
+require 'tcell_agent/sensor_events/login_fraud'
+
+module TCellAgent
+  module Hooks
+    module LoginFraud
+
+      def self.report_login_event(status, env_or_header_keys, tcell_data, user_id)
+        if (TCellAgent.configuration.enabled && TCellAgent.configuration.should_intercept_requests?)
+          login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
+
+          if (login_fraud_policy && login_fraud_policy.enabled)
+            if tcell_data
+              if ![TCellAgent::Hooks::V1::Login::LOGIN_FAILURE, TCellAgent::Hooks::V1::Login::LOGIN_SUCCESS].include?(status)
+                TCellAgent.logger.error("Unkown login status: #{status}")
+              elsif (status == TCellAgent::Hooks::V1::Login::LOGIN_FAILURE) && login_fraud_policy.login_failed_enabled
+                TCellAgent.send_event(
+                  TCellAgent::SensorEvents::LoginFailure.new(env_or_header_keys, tcell_data, user_id)
+                )
+              elsif (status == TCellAgent::Hooks::V1::Login::LOGIN_SUCCESS) && login_fraud_policy.login_success_enabled
+                TCellAgent.send_event(
+                  TCellAgent::SensorEvents::LoginSuccess.new(env_or_header_keys, tcell_data, user_id)
+                )
+              end
+            end
+          end
+        end
+      end
+
+    end
+  end
+end
+
+if defined?(TCellAgent::Hooks::V1::Frameworks::Rails::Login)
+  TCellAgent::Hooks::V1::Frameworks::Rails::Login.module_eval do
+    class << self
+
+      alias_method :tcell_register_login_event, :register_login_event
+      def register_login_event(status, rails_request, user_id, user_valid=nil)
+        TCellAgent::Instrumentation.safe_block("Rails Auth Hooks") do
+          tcell_data = rails_request.env[TCellAgent::Instrumentation::TCELL_ID]
+          TCellAgent::Hooks::LoginFraud.report_login_event(status, rails_request.env, tcell_data, user_id)
+        end
+      end
+
+    end
+  end
+end
+
+if defined?(TCellAgent::Hooks::V1::Login)
+  TCellAgent::Hooks::V1::Login.module_eval do
+    class << self
+
+      alias_method :tcell_register_login_event, :register_login_event
+      def register_login_event(
+          status,
+          session_id,
+          user_agent,
+          referrer,
+          remote_address,
+          header_keys,
+          user_id,
+          document_uri,
+          user_valid=nil)
+        TCellAgent::Instrumentation.safe_block("Login Auth Hooks") do
+          tcell_data = TCellAgent::Instrumentation::TCellData.new
+          tcell_data.user_agent = user_agent
+          tcell_data.referrer = referrer
+          tcell_data.ip_address = remote_address
+          tcell_data.path = document_uri
+          tcell_data.hmac_session_id = TCellAgent::SensorEvents::Util.hmac(session_id)
+
+          TCellAgent::Hooks::LoginFraud.report_login_event(status, header_keys, tcell_data, user_id)
+        end
+      end
+
+    end
+  end
+end

data/lib/tcell_agent/instrumentation.rb

@@ -61,7 +61,8 @@ module TCellAgent
     class TCellData
       attr_accessor :transaction_id, :session_id, :hmac_session_id, :user_id, :route_id,
         :uri, :context_filters_by_term, :database_filters, :ip_address, :user_agent, :request_method,
-        :path_parameters, :ip_blocking_triggered, :grape_mount_endpoint, :referrer, :path
+        :path_parameters, :ip_blocking_triggered, :grape_mount_endpoint, :referrer, :path,
+        :csrf_exception_name, :sql_exceptions, :database_result_sizes
 
       def self.filterx(sanitize_string, event_flag, replace_flag, term)
         send_event = false
@@ -80,6 +81,8 @@ module TCellAgent
       def initialize
         @ip_blocking_triggered = false
         @context_filters_by_term = Hash.new{|h,k| h[k] = Set.new}
+        @sql_exceptions = []
+        @database_result_sizes = []
       end
       def is_valid_term?(term)
         if term != nil && term != '' and term.to_s.length >= 5
@@ -222,13 +225,8 @@ module TCellAgent
         block.call()
 
       rescue Exception => ex
-        if TCellAgent.configuration.raise_exceptions
-          raise ex
-
-        else
-          TCellAgent.logger.debug "Exception in safe_block #{message}: #{ex.class} happened, message is #{ex.message}"
-          TCellAgent.logger.debug(ex.backtrace)
-        end
+        TCellAgent.logger.debug "Exception in safe_block #{message}: #{ex.class} happened, message is #{ex.message}"
+        TCellAgent.logger.debug(ex.backtrace)
       end
     end
 
@@ -236,9 +234,6 @@ module TCellAgent
       begin
         block.call()
       rescue Exception => e
-        if TCellAgent.configuration.raise_exceptions
-          raise e
-        end
       end
     end
   end

data/lib/tcell_agent/patches/meta_data.rb

@@ -12,23 +12,27 @@ module TCellAgent
 
       class << self
         def build(request)
-          meta_event = MetaData.new
+          tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
+          meta_event = MetaData.new(
+            request.request_method,
+            TCellAgent::Utils::Rails.better_ip(request),
+            tcell_context.route_id,
+            tcell_context.hmac_session_id,
+            tcell_context.user_id,
+            tcell_context.transaction_id
+          )
 
-          meta_event.remote_address = TCellAgent::Utils::Rails.better_ip(request)
-          meta_event.method = request.request_method
           meta_event.path = request.path
           meta_event.user_agent = request.env['HTTP_USER_AGENT']
+
           meta_event.get_dict = request.GET
           meta_event.cookie_dict = request.cookies
+          meta_event.set_headers_dict(request.env)
 
           meta_event.post_dict = request.POST
 
           meta_event.path_parameters = request.env[TCellAgent::Instrumentation::TCELL_ID].path_parameters
 
-          meta_event.route_id = request.env[TCellAgent::Instrumentation::TCELL_ID].route_id
-          meta_event.transaction_id = request.env[TCellAgent::Instrumentation::TCELL_ID].transaction_id
-          meta_event.session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
-          meta_event.user_id = request.env[TCellAgent::Instrumentation::TCELL_ID].user_id
 
           # Positions strio to the beginning of input, resetting lineno to zero.
           # rails 4.1 seems to read the stringIO directly and so body.gets is empty
@@ -46,11 +50,10 @@ module TCellAgent
         end
       end
 
-      attr_accessor :remote_address, :method, :path, :route_id, :session_id, :user_id, :transaction_id,
-        :request_content_bytes_len, :user_agent
+      attr_accessor :path, :request_content_bytes_len, :user_agent
 
-      def initialize
-        super
+      def initialize(method, remote_address, route_id, session_id, user_id, transaction_id)
+        super(method, remote_address, route_id, session_id, user_id, transaction_id)
 
         @request_content_bytes_len = 0
         @user_agent = nil

data/lib/tcell_agent/policies/appsensor/injection_sensor.rb

@@ -1,5 +1,4 @@
 require 'tcell_agent/utils/params'
-require 'tcell_agent/appsensor/sensor'
 
 
 module TCellAgent
@@ -25,14 +24,7 @@ module TCellAgent
       JSON_PARAM = TCellAgent::Utils::Params::JSON_PARAM
       COOKIE_PARAM = TCellAgent::Utils::Params::COOKIE_PARAM
       URI_PARAM = TCellAgent::Utils::Params::URI_PARAM
-
-      PARAM_TYPE_TO_L = {
-        GET_PARAM => 'query',
-        POST_PARAM => 'body',
-        JSON_PARAM => 'body',
-        URI_PARAM => 'uri',
-        COOKIE_PARAM => 'cookie'
-      }
+      HEADER_PARAM = TCellAgent::Utils::Params::HEADER_PARAM
 
       attr_accessor :enabled, :detection_point, :exclude_headers, :exclude_forms,
         :exclude_cookies, :exclusions, :active_pattern_ids, :v1_compatability_enabled,
@@ -99,6 +91,10 @@ module TCellAgent
           return false
         end
 
+        if @exclude_headers && HEADER_PARAM == type_of_param
+          return false
+        end
+
         vuln_results = find_vulnerability(param_name, param_value)
 
         if vuln_results