tcell_agent 0.2.6 → 0.2.7

data/lib/tcell_agent/rails/on_start.rb

@@ -0,0 +1,101 @@
+# See the file "LICENSE" for the full license governing this code.
+
+#require 'tcell_agent/authlogic' if defined?(Authlogic)
+#require 'tcell_agent/devise' if defined?(Devise)
+
+require 'rails/all'
+
+module TCellAgent
+  module Instrumentation
+    module Rails
+      METHODS = ['GET','POST','PUT','DELETE','HEAD',
+                 'PATCH','TRACE','CONNECT','OPTIONS']
+
+      def self.instrument_route(route)
+        if (route.constraints.has_key? :request_method)
+          route_path = "#{route.path.spec}"
+          if (route_path.end_with?("(.:format)"))
+            route_path = route_path.chomp("(.:format)")
+          end
+
+          route_destination = route.defaults.to_json.to_s
+
+          route_methods = METHODS.select { |x| route.verb.match(x) }
+          route_methods.each { |route_method|
+            route_id = TCellAgent::SensorEvents::Util.calculateRouteId(route_method.downcase, route.path.spec)
+            TCellAgent.send_event(
+              TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
+                route_path, route_method, route_id, nil, route_destination
+              )
+            )
+          }
+        end
+      end
+
+      def self.instrument_routes
+        if ::Rails.application
+          ::Rails.application.routes.routes.each do |route|
+            self.instrument_route(route)
+          end
+        end
+      end
+
+      if (::Rails::VERSION::MAJOR == 3)
+        ActionDispatch::Routing::RouteSet.class_eval do
+            alias_method :original_add_route, :add_route
+            def add_route(app, conditions = {}, requirements = {}, defaults = {}, name = nil, anchor = true)
+              route = original_add_route(app, conditions, requirements, defaults, name, anchor)
+
+              TCellAgent::Instrumentation::Rails.instrument_route(route)
+
+              route
+            end
+        end
+      end
+
+      if (::Rails::VERSION::MAJOR == 4)
+        ActionDispatch::Journey::Routes.class_eval do
+          alias_method :original_add_route, :add_route
+          def add_route(app, path, conditions, defaults, name = nil)
+            route = original_add_route(app, path, conditions, defaults, name)
+
+            TCellAgent::Instrumentation::Rails.instrument_route(route)
+
+            route
+          end
+        end
+      end
+
+    end
+  end
+end
+
+
+TCellAgent::Instrumentation::Rails.send_framework_info
+if (Rails.application)
+  TCellAgent::Instrumentation::Rails.send_settings(Rails.application)
+  require 'tcell_agent/devise' if defined?(Devise)
+  require 'tcell_agent/rails/auth/devise' if defined?(Devise)
+  require 'tcell_agent/authlogic' if defined?(Authlogic)
+  require 'tcell_agent/rails/auth/authlogic'  if defined?(Authlogic)
+
+else
+  module TCellAgent
+    class MyRailtie < Rails::Railtie
+      initializer 'activeservice.autoload', :after => :set_autoload_paths do |app|
+        if (TCellAgent.configuration.enabled)
+          Rails.application.config.to_prepare do
+            require 'tcell_agent/devise' if defined?(Devise)
+            require 'tcell_agent/rails/auth/devise' if defined?(Devise)
+            require 'tcell_agent/authlogic' if defined?(Authlogic)
+            require 'tcell_agent/rails/auth/authlogic'  if defined?(Authlogic)
+          end
+
+          Rails.application.config.after_initialize do
+            TCellAgent::Instrumentation::Rails.send_settings(Rails.application)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tcell_agent/rails/routes.rb

@@ -1,90 +1,99 @@
-
 module TCellAgent
   ActiveSupport.on_load(:action_controller) do
-
     ActionController::Base.class_eval do
 
-      around_filter :tell_around_filter_routes
+      prepend_around_filter :tell_around_filter_routes
       def tell_around_filter_routes
         begin
           TCellAgent::Instrumentation.safe_block("Determining Rails Route ID") {
             route = Rails.application.routes.router.recognize(request) { |r, _| r }.first
-
             if route
               route_path = route[2].path.spec
               tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
-              tcell_context.route_id = TCellAgent::SensorEvents::Util.calculateRouteId(request.method.downcase, route_path)
+              if tcell_context
+                tcell_context.route_id = TCellAgent::SensorEvents::Util.calculateRouteId(request.method.downcase, route_path)
+              end
             end
           }
+              def loop_params_hash(method, param_hash, prefix, &block)
+                  param_hash.each do |param_name, param_value|
+                      if param_value && param_value.is_a?(Hash)
+                          loop_params_hash(method, param_value, 'hash', &block)
+                      elsif !param_value || !param_value.instance_of?(String) || param_value == ""
+                          next
+                      else
+                          block.call(method, param_name, param_value)
+                      end
+                  end
+              end
+              def for_params(request, &block)
+                  get_params = request.GET
+                  if get_params
+                    self.loop_params_hash('get', get_params, nil, &block)
+                  end
+                  post_params = request.POST
+                  if post_params
+                    self.loop_params_hash('post', post_params, nil, &block)
+                  end
+              end
+              def _handle_dataexpsure_forms(request)
+                dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+                tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+                if tcell_context && dataex_policy && dataex_policy.has_actions_for_form_parameter?
+                  for_params(request) { |method, param_name, param_value|
+                    actions = dataex_policy.get_actions_for_request("form",param_name)
+                    if actions
+                      actions.each { |action|
+                        tcell_context.add_filter_for_request_parameter(param_value, action, param_name)
+                      }
+                    end
+                  }
+                end
+              end
+              TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request forms)") {
+                _handle_dataexpsure_forms(request)
+              }
+
+              def _handle_dataexpsure_headers(request)
+                dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+                tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+                if tcell_context && dataex_policy && dataex_policy.has_actions_for_headers?
+                  headers = request.env.select {|k,v| k.start_with? 'HTTP_'}
+                  headers.each { |header_name, header_value|
+                    header_name = header_name.sub(/^HTTP_/, '').gsub('_','-')
+                    actions = dataex_policy.get_actions_for_header(header_name)
+                    if actions
+                      actions.each { |action|
+                        tcell_context.add_filter_for_header_value(header_value, action, header_name)
+                      }
+                    end
+                  }
+                end
+              end
+              TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request headers)") {
+                _handle_dataexpsure_headers(request)
+              }
+
+              def _handler_dataexposure_cookies(request)
+                dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+                tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]               
+                if tcell_context && dataex_policy && dataex_policy.has_actions_for_cookie?
+                  request.cookies.each { |cookie_name, cookie_value|
+                    actions = dataex_policy.get_actions_for_cookie(cookie_name)
+                    if actions
+                      actions.each { |action|
+                        tcell_context.add_filter_for_cookie_value(cookie_value, action, cookie_name)
+                      }
+                    end
+                  }
+                end
+              end
+              TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request cookies)") {
+                _handler_dataexposure_cookies(request)
+              }
           yield
         end
       end
-
-    end
-  end
-end
-
-module TCellAgent
-
-  module Instrumentation
-
-    module Rails
-      METHODS = ['GET','POST','PUT','DELETE','HEAD',
-                 'PATCH','TRACE','CONNECT','OPTIONS']
-
-      def self.instrument_route(route)
-        if (route.constraints.has_key? :request_method)
-          route_path = "#{route.path.spec}"
-          if (route_path.end_with?("(.:format)"))
-            route_path = route_path.chomp("(.:format)")
-          end
-
-          route_destination = route.defaults.to_json.to_s
-
-          route_methods = METHODS.select { |x| route.verb.match(x) }
-          route_methods.each { |route_method|
-            route_id = TCellAgent::SensorEvents::Util.calculateRouteId(route_method.downcase, route.path.spec)
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
-                route_path, route_method, route_id, nil, route_destination
-              )
-            )
-          }
-        end
-      end
-
-      def self.instrument_routes
-        ::Rails.application.routes.routes.each do |route|
-          self.instrument_route(route)
-        end
-      end
-
-      if (::Rails::VERSION::MAJOR == 3)
-        ActionDispatch::Routing::RouteSet.class_eval do
-            alias_method :original_add_route, :add_route
-            def add_route(app, conditions = {}, requirements = {}, defaults = {}, name = nil, anchor = true)
-              route = original_add_route(app, conditions, requirements, defaults, name, anchor)
-
-              TCellAgent::Instrumentation::Rails.instrument_route(route)
-
-              route
-            end
-        end
-      end
-
-      if (::Rails::VERSION::MAJOR == 4)
-        ActionDispatch::Journey::Routes.class_eval do
-          alias_method :original_add_route, :add_route
-          def add_route(app, path, conditions, defaults, name = nil)
-            route = original_add_route(app, path, conditions, defaults, name)
-
-            TCellAgent::Instrumentation::Rails.instrument_route(route)
-
-            route
-          end
-        end
-      end
-
     end
   end
 end

data/lib/tcell_agent/sensor_events/app_sensor.rb

@@ -11,6 +11,7 @@ require 'tcell_agent/policies/appsensor_policy'
 require 'tcell_agent/appsensor'
 
 require 'tcell_agent/instrumentation'
+require 'tcell_agent/configuration'
 
 # Some Rules Originate from ModSecurity
 #   ModSecurity for Apache 2.x, http://www.modsecurity.org/
@@ -19,7 +20,7 @@ require 'tcell_agent/instrumentation'
 module TCellAgent
     module SensorEvents
         class TCellAppSensorEvent < TCellSensorEvent
-            def initialize(location, detection_point, remote_addr, param, route_id, data=nil, transaction_id=nil, session_id=nil, user_id=nil)
+            def initialize(location, detection_point, method, remote_addr, param, route_id, data=nil, transaction_id=nil, session_id=nil, user_id=nil, payload=nil)
                 super("as")
                 self["dp"] = detection_point
                 self["param"] = param
@@ -27,10 +28,13 @@ module TCellAgent
                 if (route_id)
                     self["rou"] = route_id
                 end
+                self["m"] = method
                 @raw_location = location
                 @user_id = user_id
                 @transaction_id = transaction_id
                 @raw_session_id = session_id
+                @payload = payload
+
             end
             def post_process
                 self["loc"] = Util.strip_uri_values(@raw_location)
@@ -44,12 +48,16 @@ module TCellAgent
                     hmac_key = Util.getHmacKey()
                     self["sid"] = Util.hmac(@raw_session_id, hmac_key)
                 end
+                if @payload
+                    self["payload"] = @payload[0..150]
+                end
             end
         end
         class TCellAppSensorEventProcessor < TCellSensorEvent
             attr_accessor :request_headers, :request_size, :remote_addr, 
                             :uri, :get_params, :post_params, :cookies,
-                            :request_content_length, :request_content_type
+                            :request_content_length, :request_content_type,
+                            :request_method
             attr_accessor :status_code, :response_headers,
                             :response_content_length,
                             :route_id, :transaction_id, :session_id, :user_id
@@ -58,18 +66,20 @@ module TCellAgent
                 @response_content_length = 0
                 @send = false
             end
-            def appsensor_event(dp, param, data)
+            def appsensor_event(dp, param, data, payload=nil)
                 TCellAgent::Instrumentation.safe_block("AppSensor Sending Event") {
                     event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
                         @uri,
                         dp,
+                        @request_method,
                         @remote_addr,
                         param,
                         @route_id,
                         data=nil,
                         transaction_id=@transaction_id,
                         session_id=@session_id,
-                        user_id=@user_id)
+                        user_id=@user_id,
+                        payload=payload)
                     @sensor_triggered = true
                     TCellAgent.send_event(event)
                 }
@@ -148,20 +158,21 @@ module TCellAgent
                     }
                 end
 
-
                 if (!@sensor_triggered && appsensor_policy.option_enabled?("resp_codes"))
                     TCellAgent::Instrumentation.safe_block("AppSensor Resting Response Code") {
                         test_response_code
                     }
                 end
 
-
-
                 if (!@sensor_triggered && appsensor_policy.option_enabled?("xss"))
                     TCellAgent::Instrumentation.safe_block("AppSensor Testing for XSS") {
                         for_params { | param_type, param_name, param_value |
                             if (TCellAgent::AppSensor.isXss(param_value))
-                                appsensor_event("xss", param_name, nil)
+                                send_payload = nil
+                                if (TCellAgent.configuration.allow_unencrypted_appsensor_payloads)
+                                    send_payload = param_value
+                                end
+                                appsensor_event("xss", param_name, nil, payload=send_payload)
                                 return
                             end
                         }
@@ -171,7 +182,11 @@ module TCellAgent
                     TCellAgent::Instrumentation.safe_block("AppSensor Testing for CMDI") {
                         for_params { | param_type, param_name, param_value |
                             if (TCellAgent::AppSensor.isCmdi(param_value))
-                                appsensor_event("cmdi", param_name, nil)
+                                send_payload = nil
+                                if (TCellAgent.configuration.allow_unencrypted_appsensor_payloads)
+                                    send_payload = param_value
+                                end
+                                appsensor_event("cmdi", param_name, nil, payload=send_payload)
                                 return
                             end
                         }
@@ -182,7 +197,11 @@ module TCellAgent
                     TCellAgent::Instrumentation.safe_block("AppSensor Testing for SQLI") {
                         for_params { | param_type, param_name, param_value |
                             if (TCellAgent::AppSensor.isSqli(param_value))
-                                appsensor_event("sqli", param_name, nil)
+                                send_payload = nil
+                                if (TCellAgent.configuration.allow_unencrypted_appsensor_payloads)
+                                    send_payload = param_value
+                                end
+                                appsensor_event("sqli", param_name, nil, payload=send_payload)
                                 return
                             end
                         }
@@ -193,7 +212,11 @@ module TCellAgent
                     TCellAgent::Instrumentation.safe_block("AppSensor Testing for Return Chars") {
                         for_get_params { | param_type, param_name, param_value |
                             if (TCellAgent::AppSensor.containsReturnChars(param_value))
-                                appsensor_event("retr", param_name, nil)
+                                send_payload = nil
+                                if (TCellAgent.configuration.allow_unencrypted_appsensor_payloads)
+                                    send_payload = param_value
+                                end
+                                appsensor_event("retr", param_name, nil, payload=send_payload)
                                 return
                             end
                         }
@@ -204,18 +227,26 @@ module TCellAgent
                     TCellAgent::Instrumentation.safe_block("AppSensor Testing for Null") {
                         for_params { | param_type, param_name, param_value |
                             if (TCellAgent::AppSensor.containsNull(param_value))
-                                appsensor_event("null", param_name, nil)
+                                send_payload = nil
+                                if (TCellAgent.configuration.allow_unencrypted_appsensor_payloads)
+                                    send_payload = param_value
+                                end
+                                appsensor_event("null", param_name, nil, payload=send_payload)
                                 return
                             end
                         }
                     }
                 end
 
-                if (!@sensor_triggered && appsensor_policy.option_enabled?("null"))
+                if (!@sensor_triggered && appsensor_policy.option_enabled?("fpt"))
                     TCellAgent::Instrumentation.safe_block("AppSensor Testing for File path traversal") {
                         for_params { | param_type, param_name, param_value |
                             if (TCellAgent::AppSensor.isPathTraversal(param_value))
-                                appsensor_event("null", param_name, nil)
+                                send_payload = nil
+                                if (TCellAgent.configuration.allow_unencrypted_appsensor_payloads)
+                                    send_payload = param_value
+                                end
+                                appsensor_event("fpt", param_name, nil, payload=send_payload)
                                 return
                             end
                         }

data/lib/tcell_agent/sensor_events/metrics.rb

@@ -1,24 +1,126 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-    module SensorEvents
-        class RequestRouteTimer < TCellSensorEvent
-            attr_accessor :route_id
-            attr_accessor :response_time
-            def initialize(route_id, response_time)
-                super("RequestRouteTimer")
-                self.route_id = route_id
-                self.response_time = response_time
-                @send = false
-            end
+  module SensorEvents
+    class Counter
+      attr_reader :counter
+      def initialize
+        @counter = 0
+      end
+
+      def add_object
+        @counter += 1
+      end
+
+      def reset
+        @counter = 0
+      end
+    end
+
+    class RequestRouteTimer < TCellSensorEvent
+      attr_accessor :route_id
+      attr_accessor :response_time
+      def initialize(route_id, response_time)
+        super("RequestRouteTimer")
+        self.route_id = route_id
+        self.response_time = response_time
+        @send = false
+      end
+    end
+
+    class MetricsEvent < TCellSensorEvent
+      def initialize()
+        super("metrics")
+      end
+      def set_route_count_table(route_count_table)
+        self["rct"] = route_count_table
+      end
+    end
+
+    class SessionsMetric < TCellSensorEvent
+      class UserSessionTrackMetric < Hash
+        def initialize(object_counter, user_id)
+          @object_counter = object_counter
+          @user_agents = {}
+          self["uid"] = user_id
+          self["track"] = []
         end
-        class MetricsEvent < TCellSensorEvent
-            def initialize()
-                super("metrics")
-            end
-            def set_route_count_table(route_count_table)
-                self["rct"] = route_count_table
+
+        def add_user_agent_ip(truncated_agent, ip_address)
+          if @user_agents.has_key?(truncated_agent)
+            tracked_agents = @user_agents[truncated_agent]
+            ips = tracked_agents[1]
+            unless ips.include?(ip_address)
+              @object_counter.add_object
+              ips.push(ip_address)
             end
+
+          else
+            @object_counter.add_object
+            @user_agents[truncated_agent] = [truncated_agent, [ip_address]]
+            self["track"].push(@user_agents[truncated_agent])
+          end
         end
+      end
+
+      class UserSessionMetric < Array
+        def initialize(object_counter)
+          @user_ids = {}
+          @object_counter = object_counter
+        end
+
+        def add_user_id_user_agent_ip(user_id, truncated_agent, ip_address)
+          if @user_ids.has_key?(user_id)
+            user_id_info = @user_ids[user_id]
+            user_id_info.add_user_agent_ip(truncated_agent, ip_address)
+
+          else
+            @object_counter.add_object
+
+            @user_ids[user_id] = user_id_info = UserSessionTrackMetric.new(@object_counter, user_id)
+            user_id_info.add_user_agent_ip(truncated_agent, ip_address)
+
+            self.push(user_id_info)
+          end
+        end
+      end
+
+      def initialize()
+        super("metrics")
+        @send = false
+        @flush = false
+
+        self["sessions"] = {}
+        @has_sessions = false
+        @object_counter = Counter.new
+      end
+
+      def has_sessions?
+        @has_sessions
+      end
+
+      def add_session_info(hmac_session_id, user_id, ip_address, user_agent)
+        if @object_counter.counter >= 250
+          TCellAgent.logger.warn("Sessions Metric is full. Information dropped")
+
+        else
+          self["sessions"][hmac_session_id] =
+            self["sessions"].fetch(hmac_session_id, UserSessionMetric.new(@object_counter))
+
+          @has_sessions = true
+
+          truncated_agent = truncated_user_agent(user_agent)
+          self["sessions"][hmac_session_id].add_user_id_user_agent_ip(user_id, truncated_agent, ip_address)
+
+          if @object_counter.counter >= 200
+            @flush = true
+          end
+        end
+      end
+
+      def truncated_user_agent(user_agent)
+        user_agent[0...256]
+      end
     end
-end
+  end
+end