tcell_agent 0.2.2 → 0.2.4

data/{config/initializers/devise_auth.rb → lib/tcell_agent/rails/auth/devise.rb}

@@ -1,13 +1,5 @@
-# See the file "LICENSE" for the full license governing this code.
-
-require 'tcell_agent/logger'
-require 'tcell_agent/configuration'
-require 'tcell_agent/userinfo'
-require 'tcell_agent/instrumentation'
-
 module TCellAgent
   if defined?(Devise)
-
     if (TCellAgent.configuration.enabled && TCellAgent.configuration.instrument_for_events)
       TCellAgent.logger.debug("Instrumenting Devise")
 
@@ -16,62 +8,15 @@ module TCellAgent
       require 'tcell_agent/sensor_events/app_sensor'
       require 'tcell_agent/policies/appsensor_policy'
 
-      # Devise::OmniauthCallbacksController.class_eval do
-      #   after_filter :log_after_login
-      #   alias_method :original_failure, :failure
-
-      #   def failure
-      #     TCellAgent::Instrumentation.safe_block("Omniauth login failed") {
-      #       login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
-      #       if (login_fraud_policy && login_fraud_policy.enabled && login_fraud_policy.login_failed_enabled)
-      #         hmac_session_id = request.env["tcell.request_data"].hmac_session_id
-      #         event = TCellAgent::SensorEvents::LoginFailure.new(request, response, nil, hmac_session_id)
-      #         TCellAgent.send_event(event)         
-      #       end
-      #       appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
-      #       if (appsensor_policy && appsensor_policy.enabled && appsensor_policy.option_enabled?("login_failure"))
-      #         hmac_session_id = request.env["tcell.request_data"].hmac_session_id
-      #         event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
-      #           request.fullpath,
-      #           TCellAgent::Policies::AppSensorPolicy::DP_LOGIN_FAILURE,
-      #           request.remote_ip,
-      #           nil,
-      #           request.env["tcell.request_data"].route_id,
-      #           data=nil,
-      #           transaction_id=nil,
-      #           session_id=hmac_session_id,
-      #           user_id=nil)
-      #         TCellAgent.send_event(event)
-      #       end            
-      #     }
-      #     original_failure
-      #   end
-      #   private
-      #   def log_after_login
-      #     TCellAgent::Instrumentation.safe_block("Omniauth login successful") {
-      #       login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
-      #       if (login_fraud_policy && login_fraud_policy.enabled && login_fraud_policy.login_success_enabled)
-      #         omniauth = env["omniauth.auth"]          
-      #         if (omniauth)
-      #           hmac_session_id = request.env["tcell.request_data"].hmac_session_id
-      #           user_id = request.env["tcell.request_data"].user_id
-      #           event = TCellAgent::SensorEvents::LoginSuccess.new(request, response, user_id, hmac_session_id)
-      #           TCellAgent.send_event(event)         
-      #         end
-      #       end
-      #     }
-      #   end 
-      # end
-
       Devise::SessionsController.class_eval do
-        after_filter :log_failed_login, :only => :new
 
+        after_filter :log_failed_login, :only => :new
         alias_method :original_new, :new
         def new
           original_new
         end
 
-        alias_method :original_create, :create
+       alias_method :original_create, :create
         def create(&block)
           results = original_create(&block)
           TCellAgent::Instrumentation.safe_block("Devise login successful") {
@@ -137,31 +82,7 @@ module TCellAgent
         def failed_login?
           (options = env["warden.options"]) && options[:action] == "unauthenticated"
         end 
-
       end
-      # Devise::PasswordsController.class_eval do
-
-      #      after_filter :send_results
-      #      def send_results
-      #         puts response
-      #      end
-
-      #      def new
-      #         #::TCellAgent::Sensors::LoginFraud.use_request(request)
-      #         self.resource = resource_class.new
-      #       end
-
-      #       def create
-      #         self.resource = resource_class.send_reset_password_instructions(resource_params)
-      #         yield resource if block_given?
-
-      #         if successfully_sent?(resource)
-      #           respond_with({}, location: after_sending_reset_password_instructions_path_for(resource_name))
-      #         else
-      #           respond_with(resource)
-      #         end
-      #       end
-      # end
     end # if instrument
   end #if defined devise
 end

data/lib/tcell_agent/rails/dlp.rb

@@ -51,34 +51,39 @@ require 'thread'
 #   end
 # end
 
-# class ActiveRecord::Base
-#   after_initialize do |user|
-#     puts "You have initialized an object!"
-#     puts user
-#   end
-
-#   after_find do |record|
-#     dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-#     if dlp_policy
-#       request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
-#       if request_env
-#         model = record.class
-#         model.columns.each do |col|
-#           #puts "#{model.table_name} .. #{col.name}"
-#           actions = dlp_policy.get_actions_for(model.table_name, col.name)
-#           if (actions.include?("body_redact"))
-#             (request_env["filter_body_set"] ||= Set.new).add(record[col.name.to_sym])
-#             #request_env["filter_body_set"].add()
-#           end
-#           if (actions.include?("log_redact"))
-#             (request_env["filter_log_set"] ||= Set.new).add(record[col.name.to_sym])
-#             #request_env["filter_log_set"].add(record[col.name.to_sym])
-#           end
-#         end
-#       end
-#     end
-#   end
-# end
+class ActiveRecord::Base
+  # after_initialize do |user|
+  #   puts "You have initialized an object!"
+  #   puts "ASDF"
+  # end
+  after_find do |record|
+    database_name = self.class.connection_config().fetch(:database,"*").split('/').last
+    #puts record
+    dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+    if dlp_policy
+      request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
+      if request_env
+        tcell_context = request_env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+        if tcell_context
+          model = record.class
+          column_names = model.columns.map { |col| col.name }
+          if (dlp_policy.database_discovery_enabled)
+            TCellAgent.discover_database_fields(tcell_context.route_id, database_name,"*",model.table_name, column_names)
+          end
+          model.columns.each do |col|
+            #puts "#{model.table_name} .. #{col.name}"
+            action_objs = dlp_policy.get_actions_for_table(database_name, "*", model.table_name, col.name, tcell_context.route_id)
+            if action_objs
+              action_objs.each do |action_obj|
+                tcell_context.add_response_db_filter(record[col.name.to_sym], action_obj, database_name, "*", model.table_name, col.name)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
 
 # - Request
 # -   Session Id event
@@ -158,25 +163,22 @@ module TCellAgent
 end
 
 module TCellAgent
-  class Engine < Rails::Engine
     ActiveSupport.on_load(:action_controller) do
       ActionController::Base.class_eval do
         around_filter :global_request_logging
         def global_request_logging 
           begin 
             yield
-            TCellAgent::Instrumentation.safe_block("Handling JSAgent add") {
-              dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-              if dlp_policy
-                tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
-                response.body = dlp_policy.response_body_enforce(tcell_context, response.body)
+            TCellAgent::Instrumentation.safe_block("Running DLP Logging Filters") {
+              tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+              if tcell_context
+                response.body = tcell_context.filter_body(response.body)
               end
             }
           end
         end
       end
     end
-  end
 end
 
 class Logger
@@ -196,7 +198,9 @@ class Logger
       request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
       if message && dlp_policy && request_env
         tcell_context = request_env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
-        dlp_policy.log_enforce(tcell_context, message)
+        if tcell_context
+          tcell_context.filter_log(message)
+        end
       end
     }
     tcell_old_add(severity, message, progname)

data/lib/tcell_agent/rails/middleware/body_filter_middleware.rb

@@ -49,9 +49,9 @@ module TCellAgent
             TCellAgent::Instrumentation.safe_block("Handling JSAgent add") {
               status, headers, rack_body = response
               if (headers.fetch("Content-Type","").start_with?'text/html')
-                  script_tag_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AddScriptTag)
+                  script_tag_policy = TCellAgent.policy(TCellAgent::PolicyTypes::CSP)
                   if (script_tag_policy &&
-                      script_tag_policy.enabled)
+                      script_tag_policy.js_agent_api_key)
                     newbody = []
                     rack_body.each { |str| 
                       newbody << self.replace_in_body(script_tag_policy, str) 

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -54,6 +54,7 @@ module TCellAgent
               if content_security_policy
                 content_security_policy.each(
                   request.env["tcell.request_data"].transaction_id, 
+                  request.env["tcell.request_data"].route_id,
                   request.env["tcell.request_data"].hmac_session_id, 
                   request.env["tcell.request_data"].user_id) do | header_pair | 
                   headers[header_pair["name"]] = header_pair["value"]
@@ -141,7 +142,6 @@ module TCellAgent
                 event = TCellAgent::SensorEvents::TCellAppSensorEventProcessor.new                
                 event.request_headers = request.env
                 event.request_content_length = (request.content_length || "0").to_i
-                event.request_body = request.body
                 event.response_content_length = (rack_response.length || "0").to_i
                 event.remote_addr = request.ip
                 event.uri = request.fullpath
@@ -149,6 +149,9 @@ module TCellAgent
                 event.post_params = request.POST
                 event.cookies = request.cookies
 
+                request_body = request.body
+                event.request_size = event.request_content_length
+                
                 #status, headers, active_response = response
                 #if active_response.class != ActionDispatch::Response
                 #  return response
@@ -156,13 +159,16 @@ module TCellAgent
 
                 event.status_code = status_code
                 event.response_headers = response_headers
-                event.response_body = response_body
 
                 event.route_id = request.env["tcell.request_data"].route_id
                 event.transaction_id = request.env["tcell.request_data"].transaction_id
                 event.session_id = request.env["tcell.request_data"].hmac_session_id
                 event.user_id = request.env["tcell.request_data"].user_id
 
+                #puts event.response_headers
+                #puts event.cookies, event.post_params, event.get_params, event.uri, event.remote_addr
+                #puts event.request_headers
+
                 TCellAgent.send_event(event)
               end
             }

data/lib/tcell_agent/rails/routes.rb

@@ -1,29 +1,25 @@
 
 module TCellAgent
-  class Engine < Rails::Engine
-    ActiveSupport.on_load(:action_controller) do
-      ActionController::Base.class_eval do
-        around_filter :tell_around_filter_routes
-        def tell_around_filter_routes 
-          begin 
-            TCellAgent::Instrumentation.safe_block("Determining Rails Route ID") {
-              route = Rails.application.routes.router.recognize(request) { |route, _| route }.first
-              if route
-                route_path = route[2].path.spec
-                tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
-                tcell_context.route_id = TCellAgent::SensorEvents::Util.calculateRouteId(request.method.downcase, route_path)
-              end
-            }
-            yield
+  ActiveSupport.on_load(:action_controller) do
+    ActionController::Base.class_eval do      
+          around_filter :tell_around_filter_routes
+          def tell_around_filter_routes 
+            begin 
+              TCellAgent::Instrumentation.safe_block("Determining Rails Route ID") {
+                route = Rails.application.routes.router.recognize(request) { |route, _| route }.first
+                if route
+                  route_path = route[2].path.spec
+                  tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+                  tcell_context.route_id = TCellAgent::SensorEvents::Util.calculateRouteId(request.method.downcase, route_path)
+                end
+              }
+              yield
+            end
           end
-        end
-      end
     end
   end
 end
 
-
-
 module TCellAgent
   module Instrumentation
     module Rails

data/lib/tcell_agent/routes/table.rb

@@ -0,0 +1,35 @@
+module TCellAgent
+  module Routes
+  	class FieldEndpoint
+	    attr_accessor :discovered
+	    def initialize
+	    	super()
+	    	@discovered = false
+	    end
+  	end
+
+  	class RouteEndpoint
+	    attr_accessor :database
+	    def initialize
+	    	@database = Hash.new {|h,k| # Database
+							h[k] = Hash.new {|h,k| # Schema
+								h[k] = Hash.new {|h,k| # Table
+									h[k] = Hash.new {|h,k| #Field
+										h[k] = FieldEndpoint.new
+									}
+								}
+							}
+						}
+
+	    end
+  	end
+
+  	class RouteTable
+		attr_accessor :routes
+		def initialize
+	    	@routes = Hash.new { |h,k| h[k] = RouteEndpoint.new }
+	    end
+	end 
+
+  end #/module Routes
+end #/module TCellAgent

data/lib/tcell_agent/sensor_events/app_sensor.rb

@@ -47,10 +47,10 @@ module TCellAgent
             end
         end
         class TCellAppSensorEventProcessor < TCellSensorEvent
-            attr_accessor :request_headers, :request_body, :remote_addr, 
+            attr_accessor :request_headers, :request_size, :remote_addr, 
                             :uri, :get_params, :post_params, :cookies,
                             :request_content_length, :request_content_type
-            attr_accessor :status_code, :response_headers, :response_body,
+            attr_accessor :status_code, :response_headers,
                             :response_content_length,
                             :route_id, :transaction_id, :session_id, :user_id
             def initialize
@@ -74,33 +74,28 @@ module TCellAgent
                     TCellAgent.send_event(event)
                 }
             end
-            def for_params
-                if @get_params
-                    @get_params.each do |param_name, param_value|
-                        if !param_value || !param_value.instance_of?(String) || param_value == ""
-                            next
-                        end
-                        yield('get', param_name, param_value)
+            def loop_params_hash(method, param_hash, prefix, &block)
+                param_hash.each do |param_name, param_value|
+                    if param_value && param_value.is_a?(Hash)
+                        loop_params_hash(method, param_value, 'hash', &block)
+                    elsif !param_value || !param_value.instance_of?(String) || param_value == ""
+                        next
+                    else
+                        block.call(method, param_name, param_value)
                     end
                 end
+            end
+            def for_params(&block)
+                if @get_params
+                    self.loop_params_hash('get', @get_params, nil, &block)
+                end
                 if @post_params
-                    @post_params.each do |param_name, param_value|
-                        if !param_value || !param_value.instance_of?(String) || param_value == ""
-                            next
-                        end
-                        yield('post', param_name, param_value)
-                    end
+                    self.loop_params_hash('post', @post_params, nil, &block)
                 end
             end
-
-            def for_get_params
+            def for_get_params(&block)
                 if @get_params
-                    @get_params.each do |param_name, param_value|
-                        if !param_value || !param_value.instance_of?(String) || param_value == ""
-                            next
-                        end
-                        yield('get', param_name, param_value)
-                    end
+                    self.loop_params_hash('get', @get_params, nil, &block)
                 end
             end
 
@@ -108,14 +103,7 @@ module TCellAgent
                 if (@response_content_length && @response_content_length > TCellAgent::Policies::AppSensorPolicy::MAX_NORMAL_RESPONSE_BYTES)
                     appsensor_event(TCellAgent::Policies::AppSensorPolicy::DP_UNUSUAL_RESPONSE_SIZE, response_content_length.to_s, nil)
                 end
-                request_size = 0
-                if @request_body
-                    if @request_body.kind_of?(StringIO)
-                        request_size = @request_body.size
-                    else
-                        request_size = @request_content_length
-                    end
-                end
+                request_size = @request_size || 0
                 if (request_size > TCellAgent::Policies::AppSensorPolicy::MAX_NORMAL_REQUEST_BYTES)
                     appsensor_event(TCellAgent::Policies::AppSensorPolicy::DP_UNUSUAL_REQUEST_SIZE,request_size.to_s, nil)
                 end
@@ -167,6 +155,8 @@ module TCellAgent
                     }
                 end
 
+
+
                 if (!@sensor_triggered && appsensor_policy.option_enabled?("xss"))
                     TCellAgent::Instrumentation.safe_block("AppSensor Testing for XSS") {
                         for_params { | param_type, param_name, param_value |
@@ -177,8 +167,7 @@ module TCellAgent
                         }
                     }
                 end
-                
-                if (!@sensor_triggered && appsensor_policy.option_enabled?("cmdi"))
+                    if (!@sensor_triggered && appsensor_policy.option_enabled?("cmdi"))
                     TCellAgent::Instrumentation.safe_block("AppSensor Testing for CMDI") {
                         for_params { | param_type, param_name, param_value |
                             if (TCellAgent::AppSensor.isCmdi(param_value))