tcell_agent 0.2.19 → 0.2.21

data/lib/tcell_agent/rails/dlp.rb

@@ -32,6 +32,87 @@ require 'tcell_agent/configuration'
 
 
 module TCellAgent
+  module DLP
+    def self.instrument_find_by_sql(results)
+      if results.size > 0
+
+        if TCellAgent.configuration.enabled &&
+            TCellAgent.configuration.should_instrument? &&
+            TCellAgent.configuration.should_intercept_requests?
+
+          dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+          appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
+
+          if dlp_policy || appsensor_policy
+            request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+            tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
+
+            if tcell_context
+              if appsensor_policy
+                appsensor_policy.process_db_rows(tcell_context, results.size)
+              end
+
+              if dlp_policy
+                first_record = results.first
+                database_name = first_record.class.connection_config().fetch(:database,"*").split('/').last
+                model = first_record.class
+                column_names = model.columns.map { |col| col.name }
+                table_name = model.table_name
+
+                if dlp_policy.database_discovery_enabled
+                  TCellAgent.discover_database_fields(
+                    tcell_context.route_id,
+                    database_name,
+                    "*",
+                    table_name,
+                    column_names
+                  )
+                end
+
+                if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
+                  TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
+                end
+
+                column_name_to_rules = column_names.inject({}) do |memo, column_name|
+                  rules = dlp_policy.get_actions_for_table(
+                    database_name,
+                    "*",
+                    table_name,
+                    column_name,
+                    tcell_context.route_id
+                  )
+
+                  memo[column_name] = rules if rules
+
+                  memo
+                end
+
+                return if column_name_to_rules.empty?
+
+                results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |record|
+                  column_name_to_rules.each do |column_name, rules|
+                    if rules
+                      rules.each do |rule|
+                        tcell_context.add_response_db_filter(
+                          record[column_name.to_sym],
+                          rule,
+                          database_name,
+                          "*",
+                          table_name,
+                          column_name
+                        )
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
   class MyRailtie < Rails::Railtie
     initializer 'activeservice.autoload', :after => :set_autoload_paths do |app|
 
@@ -58,88 +139,29 @@ module TCellAgent
 
         ActiveRecord::Querying.module_eval do
 
-          alias_method :tcell_find_by_sql, :find_by_sql
-          def find_by_sql(sql, binds = [])
-            results = tcell_find_by_sql(sql, binds)
-
-            TCellAgent::Instrumentation.safe_block("Running DLP on find_by_sql") do
-              if results.size > 0
-
-                if TCellAgent.configuration.enabled &&
-                    TCellAgent.configuration.should_instrument? &&
-                    TCellAgent.configuration.should_intercept_requests?
+          if (::Rails::VERSION::MAJOR == 5)
+            alias_method :tcell_find_by_sql, :find_by_sql
+            def find_by_sql(*args)
+              results = tcell_find_by_sql(*args)
 
-                  dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-                  appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
-
-                  if dlp_policy || appsensor_policy
-                    request_env =
-                      TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
-                    tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
+              TCellAgent::Instrumentation.safe_block("Running DLP on find_by_sql") do
+                TCellAgent::DLP.instrument_find_by_sql(results)
+              end
 
-                    if tcell_context
-                      if appsensor_policy
-                        appsensor_policy.process_db_rows(tcell_context, results.size)
-                      end
+              results
+            end
 
-                      if dlp_policy
-                        first_record = results.first
-                        database_name = first_record.class.connection_config().fetch(:database,"*").split('/').last
-                        model = first_record.class
-                        column_names = model.columns.map { |col| col.name }
-                        table_name = model.table_name
-
-                        if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
-                          TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
-                        end
-
-                        if dlp_policy.database_discovery_enabled
-                          TCellAgent.discover_database_fields(
-                            tcell_context.route_id,
-                            database_name,
-                            "*",
-                            table_name,
-                            column_names
-                          )
-                        end
-
-                        column_name_to_rules = first_record.attributes.keys.inject({}) do |memo, column_name|
-                          memo[column_name] = dlp_policy.get_actions_for_table(
-                            database_name,
-                            "*",
-                            table_name,
-                            column_name,
-                            tcell_context.route_id
-                          )
-
-                          memo
-                        end
-
-                        results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |record|
-                          column_name_to_rules.each do |column_name, rules|
-                            if rules
-                              rules.each do |rule|
-                                tcell_context.add_response_db_filter(
-                                  record[column_name.to_sym],
-                                  rule,
-                                  database_name,
-                                  "*",
-                                  table_name,
-                                  column_name
-                                )
-                              end
-                            end
-                          end
-                        end
-                      end
+          elsif (::Rails::VERSION::MAJOR < 5)
+            alias_method :tcell_find_by_sql, :find_by_sql
+            def find_by_sql(sql, binds = [])
+              results = tcell_find_by_sql(sql, binds)
 
-                    end
-                  end
-                end
+              TCellAgent::Instrumentation.safe_block("Running DLP on find_by_sql") do
+                TCellAgent::DLP.instrument_find_by_sql(results)
               end
-            end
 
-            results
+              results
+            end
           end
 
         end

data/lib/tcell_agent/rails/middleware/global_middleware.rb

@@ -38,10 +38,9 @@ module TCellAgent
               }
 
               TCellAgent::Instrumentation.safe_block("Setting hmac_session_id") {
-                hmac_key = TCellAgent::SensorEvents::Util.getHmacKey()
                 if request.env[TCellAgent::Instrumentation::TCELL_ID].session_id
                   env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id =
-                    TCellAgent::SensorEvents::Util.hmac(request.env[TCellAgent::Instrumentation::TCELL_ID].session_id, hmac_key)
+                    TCellAgent::SensorEvents::Util.hmac(request.env[TCellAgent::Instrumentation::TCELL_ID].session_id)
                 end
               }
 

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -125,7 +125,7 @@ module TCellAgent
               if http_redirect_policy && headers.has_key?("Location")
                 local_uri = URI.parse(request.url)
                 route_id = request.env[TCellAgent::Instrumentation::TCELL_ID].route_id
-                session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].session_id
+                hmac_session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
                 new_location = http_redirect_policy.enforce(
                   headers["Location"],
                   local_uri.host,
@@ -134,7 +134,7 @@ module TCellAgent
                   route_id,
                   status,
                   TCellAgent::Utils::Rails.better_ip(request),
-                  session_id)
+                  hmac_session_id)
                 # Enforcement
                 if (new_location)
                   headers["Location"] = new_location

data/lib/tcell_agent/rails/on_start.rb

@@ -16,23 +16,43 @@ module TCellAgent
 
       def self.instrument_route(route)
         if TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument?
-          if (route.constraints.has_key? :request_method)
-            route_path = "#{route.path.spec}"
-            if (route_path.end_with?("(.:format)"))
-              route_path = route_path.chomp("(.:format)")
-            end
-
-            route_destination = route.defaults.to_json.to_s
-
-            route_methods = METHODS.select { |x| route.verb.match(x) }
-            route_methods.each { |route_method|
-              route_id = TCellAgent::SensorEvents::Util.calculateRouteId(route_method.downcase, route.path.spec)
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
-                  route_path, route_method, route_id, nil, route_destination
+          if (::Rails::VERSION::MAJOR == 5)
+              route_path = "#{route.path.spec}"
+              if (route_path.end_with?("(.:format)"))
+                route_path = route_path.chomp("(.:format)")
+              end
+
+              route_destination = route.defaults.to_json.to_s
+
+              route_methods = (route.verb || "").split('|')
+              route_methods.each do |route_method|
+                route_id = TCellAgent::SensorEvents::Util.calculateRouteId(route_method.downcase, route.path.spec)
+                TCellAgent.send_event(
+                  TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
+                    route_path, route_method, route_id, nil, route_destination
+                  )
+                )
+              end
+
+          elsif (::Rails::VERSION::MAJOR < 5)
+            if (route.constraints.has_key? :request_method)
+              route_path = "#{route.path.spec}"
+              if (route_path.end_with?("(.:format)"))
+                route_path = route_path.chomp("(.:format)")
+              end
+
+              route_destination = route.defaults.to_json.to_s
+
+              route_methods = METHODS.select { |x| route.verb.match(x) }
+              route_methods.each do |route_method|
+                route_id = TCellAgent::SensorEvents::Util.calculateRouteId(route_method.downcase, route.path.spec)
+                TCellAgent.send_event(
+                  TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
+                    route_path, route_method, route_id, nil, route_destination
+                  )
                 )
-              )
-            }
+              end
+            end
           end
         end
       end
@@ -47,9 +67,9 @@ module TCellAgent
 
       if (::Rails::VERSION::MAJOR == 3)
         ActionDispatch::Routing::RouteSet.class_eval do
-            alias_method :original_add_route, :add_route
+            alias_method :tcell_add_route, :add_route
             def add_route(app, conditions = {}, requirements = {}, defaults = {}, name = nil, anchor = true)
-              route = original_add_route(app, conditions, requirements, defaults, name, anchor)
+              route = tcell_add_route(app, conditions, requirements, defaults, name, anchor)
 
               TCellAgent::Instrumentation::Rails.instrument_route(route)
 
@@ -60,9 +80,22 @@ module TCellAgent
 
       if (::Rails::VERSION::MAJOR == 4)
         ActionDispatch::Journey::Routes.class_eval do
-          alias_method :original_add_route, :add_route
+          alias_method :tcell_add_route, :add_route
           def add_route(app, path, conditions, defaults, name = nil)
-            route = original_add_route(app, path, conditions, defaults, name)
+            route = tcell_add_route(app, path, conditions, defaults, name)
+
+            TCellAgent::Instrumentation::Rails.instrument_route(route)
+
+            route
+          end
+        end
+      end
+
+      if (::Rails::VERSION::MAJOR == 5)
+        ActionDispatch::Journey::Routes.class_eval do
+          alias_method :tcell_add_route, :add_route
+          def add_route(name, mapping)
+            route = tcell_add_route(name, mapping)
 
             TCellAgent::Instrumentation::Rails.instrument_route(route)
 

data/lib/tcell_agent/sensor_events/appsensor_event.rb

@@ -12,8 +12,8 @@ module TCellAgent
                      remote_addr,
                      param,
                      route_id,
-                     data=nil,
-                     session_id=nil,
+                     meta=nil,
+                     hmac_session_id=nil,
                      user_id=nil,
                      payload=nil,
                      pattern=nil)
@@ -21,31 +21,24 @@ module TCellAgent
         self["dp"] = detection_point
         self["param"] = param
         self["remote_addr"] = remote_addr
-        if (route_id)
-          self["rou"] = route_id
-        end
         self["m"] = method
         @raw_location = location
         @user_id = user_id
-        @raw_session_id = session_id
+        @hmac_session_id = hmac_session_id
         @payload = payload
-        if pattern
-          self["pattern"] = pattern
-        end
+
+        self["pattern"] = pattern if pattern
+        self["meta"] = meta if meta
+        self["rid"] = route_id if route_id
       end
 
       def post_process
-        self["loc"] = Util.strip_uri_values(@raw_location)
-        if @user_id
-          self["uid"] = @user_id.to_s
-        end
-        if @raw_session_id
-          hmac_key = Util.getHmacKey()
-          self["sid"] = Util.hmac(@raw_session_id, hmac_key)
-        end
-        if @payload
-          self["payload"] = @payload[0..150]
+        self["uri"] = Util.strip_uri_values(@raw_location)
+        self["uid"] = @user_id.to_s if @user_id
+        if @hmac_session_id
+          self["sid"] = @hmac_session_id
         end
+        self["payload"] = @payload[0..150] if @payload
       end
 
     end

data/lib/tcell_agent/sensor_events/appsensor_meta_event.rb

@@ -6,6 +6,8 @@ require 'tcell_agent/sensor_events/sensor'
 require 'tcell_agent/agent'
 require 'tcell_agent/agent/policy_types'
 require 'tcell_agent/policies/appsensor_policy'
+require 'tcell_agent/utils/params'
+
 
 # Some Rules Originate from ModSecurity
 #   ModSecurity for Apache 2.x, http://www.modsecurity.org/
@@ -28,9 +30,11 @@ module TCellAgent
           meta_event.request_content_len = (request.content_length || "0").to_i
           meta_event.response_content_len = (rack_response.length || "0").to_i
           meta_event.get_dict = request.GET
-          meta_event.post_dict = request.POST
           meta_event.cookie_dict = request.cookies
 
+          # don't enqueue parameter values of unknown type to avoid any serialization issues
+          meta_event.post_dict = TCellAgent::Utils::Params.flatten(request.POST)
+
           meta_event.response_code = response_code
           meta_event.response_headers = response_headers
 
@@ -76,7 +80,8 @@ module TCellAgent
         else
           if request_content_type =~ %r{application/json}i && request_body
             begin
-              @body_dict = JSON.parse(request_body)
+              # don't enqueue parameter values of unknown type to avoid any serialization issues
+              @body_dict = TCellAgent::Utils::Params.flatten(JSON.parse(request_body))
             rescue
               TCellAgent.logger.debug("JSON body parameter parsing failed")
               @body_dict = {}

data/lib/tcell_agent/sensor_events/sensor.rb

@@ -44,7 +44,7 @@ module TCellAgent
             end
         end
         class TCellRedirectSensorEvent < TCellSensorEvent
-            def initialize(redirect_domain, original_domain, original_url, method, route_id, status_code, remote_addr, session_id=nil, user_id=nil)
+            def initialize(redirect_domain, original_domain, original_url, method, route_id, status_code, remote_addr, hmac_session_id=nil, user_id=nil)
                 super("redirect")
                 @raw_original_url = original_url
                 self["method"] = method
@@ -56,36 +56,35 @@ module TCellAgent
                 end
                 @raw_redirect_domain = redirect_domain
                 @user_id = user_id
-                @raw_session_id = session_id
+                @hmac_session_id = hmac_session_id
             end
             def post_process
                 self["from"] = Util.strip_uri_values(@raw_original_url)
                 self["to"] = @raw_redirect_domain
-                if @raw_session_id
-                    hmac_key = Util.getHmacKey()
-                    self["sid"] = Util.hmac(@raw_session_id, hmac_key)
+                if @hmac_session_id
+                  self["sid"] = @hmac_session_id
                 end
             end
         end
+
         class TCellFingerprintSensorEvent < TCellSensorEvent
-            def initialize(request, session_id, user_id=nil)
+            def initialize(request, hmac_session_id, user_id=nil)
                 super("fingerprint")
                 @raw_request = request
-                @raw_session_id = session_id
+                @hmac_session_id = hmac_session_id
                 @user_id = user_id
             end
             def post_process
                 if !(@raw_request.headers.key?("HTTP_USER_AGENT"))
                     raise "User Agent not Found!"
-                end 
+                end
                 self["ua"] = @raw_request.headers["HTTP_USER_AGENT"]
                 self["ip"] = @raw_request.remote_ip
-                hmac_key = Util.getHmacKey()
-                self["sid"] = Util.hmac(@raw_session_id, hmac_key)
+                self["sid"] = @hmac_session_id
                 if @user_id
                     self["uid"] = @user_id
                 end
             end
         end
     end
-end
+end

data/lib/tcell_agent/sensor_events/server_agent.rb

@@ -20,25 +20,30 @@ module TCellAgent
                 super("server_agent_details")
                 @flush = true
                 @ensure = true
+
+                self["user"] = "unknown"
+                self["group"] = "unknown"
+
                 begin
                     login = Etc.getlogin
-                    self["user"] = login
-                    TCellAgent.logger.debug("User #{login}")
-                    begin
-                        info = Etc.getpwnam(login)
-                        self["group"] = info.gid.to_s
-                        TCellAgent.logger.debug("Group #{info.gid.to_s}")
-                    rescue Exception => te
-                        TCellAgent.logger.warn("Exception finding group id: #{te.message}")
-                        TCellAgent.logger.debug(te.backtrace)
-                        self["group"] = "unknown"
+                    if login
+                      self["user"] = login
+                      begin
+                          info = Etc.getpwnam(login)
+                          self["group"] = info.gid.to_s
+                      rescue Exception => te
+                          TCellAgent.logger.warn("Exception finding group id: #{te.message}")
+                          TCellAgent.logger.debug(te.backtrace)
+                      end
                     end
+
                 rescue Exception => to
-                    self["user"] = "unknown"
-                    self["group"] = "unknown"
                     TCellAgent.logger.warn("Exception finding user & group: #{to.message}")
                     TCellAgent.logger.debug(te.backtrace)
                 end
+
+                TCellAgent.logger.debug("User #{self["user"]}")
+                TCellAgent.logger.debug("Group #{self["group"]}")
             end
         end
         class ServerAgentDetailsLanguageEvent < TCellSensorEvent