tcell_agent 1.1.12 → 2.2.0

data/lib/tcell_agent/rails/dlp.rb

@@ -1,11 +1,7 @@
 # See the file "LICENSE" for the full license governing this code.
 
-require 'tcell_agent/authlogic' if defined?(Authlogic)
-require 'tcell_agent/devise' if defined?(Devise)
-
 require 'rails'
 require 'uri'
-require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/sensor_events/sensor'
 require 'tcell_agent/sensor_events/server_agent'
@@ -22,7 +18,6 @@ require 'tcell_agent/rails/settings_reporter'
 
 require 'tcell_agent/instrumentation'
 
-require 'tcell_agent/userinfo'
 require 'cgi'
 require 'thread'
 
@@ -31,11 +26,20 @@ require 'tcell_agent/rails/responses'
 
 module TCellAgent
   module DLP
+    def self.get_dlp_logger
+      unless defined?(@rails_dlp_logger)
+        @rails_dlp_logger = TCellAgent::ModuleLogger.new(
+          TCellAgent.logger, name
+        )
+      end
+
+      @rails_dlp_logger
+    end
+
     def self.instrument_pluck(results, column_names, model)
       return if results.empty?
 
-      if TCellAgent.configuration.enabled &&
-         TCellAgent.configuration.should_instrument? &&
+      if TCellAgent.configuration.should_instrument? &&
          TCellAgent.configuration.should_intercept_requests?
 
         dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
@@ -88,7 +92,7 @@ module TCellAgent
             end
 
             if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
-              TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
+              get_dlp_logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
             end
 
             return if column_name_to_rules.empty?
@@ -138,8 +142,7 @@ module TCellAgent
     def self.instrument_find_by_sql(results)
       return if results.empty?
 
-      if TCellAgent.configuration.enabled &&
-         TCellAgent.configuration.should_instrument? &&
+      if TCellAgent.configuration.should_instrument? &&
          TCellAgent.configuration.should_intercept_requests?
 
         dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
@@ -167,7 +170,7 @@ module TCellAgent
             end
 
             if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
-              TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
+              get_dlp_logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
             end
 
             column_name_to_rules = column_names.each_with_object({}) do |column_name, memo|
@@ -214,16 +217,22 @@ module TCellAgent
             result = tcell_translate_exception(exception, message)
 
             TCellAgent::Instrumentation.safe_block('Set sql_exception_detected in meta') do
-              rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
-              if rust_policies && rust_policies.appfirewall_enabled
+              appfirewall_policy = TCellAgent.policy(TCellAgent::PolicyTypes::APPSENSOR)
+              if appfirewall_policy.enabled
                 request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(
                   Thread.current.object_id, {}
                 )
                 tcell_data = request_env[TCellAgent::Instrumentation::TCELL_ID]
                 if tcell_data && result.is_a?(ActiveRecord::StatementInvalid)
-                  tcell_data.sql_exceptions.push(
-                    { 'exception_name' => result.class.name, 'exception_payload' => message }
-                  )
+                  if message.is_a? Hash
+                    tcell_data.sql_exceptions.push(
+                      { 'exception_name' => result.class.name, 'exception_payload' => message[:message] }
+                    )
+                  else
+                    tcell_data.sql_exceptions.push(
+                      { 'exception_name' => result.class.name, 'exception_payload' => message }
+                    )
+                  end
                 end
               end
             end
@@ -246,7 +255,7 @@ module TCellAgent
         end
 
         ActiveRecord::Querying.module_eval do
-          if ::Rails::VERSION::MAJOR == 5
+          if ::Rails::VERSION::MAJOR >= 5
             alias_method :tcell_find_by_sql, :find_by_sql
             def find_by_sql(*args)
               results = tcell_find_by_sql(*args)
@@ -290,8 +299,7 @@ module TCellAgent
   module Policies
     class DataLossPolicy
       def log_enforce(tcell_context, sanitize_string)
-        if TCellAgent.configuration.enabled &&
-           TCellAgent.configuration.should_instrument? &&
+        if TCellAgent.configuration.should_instrument? &&
            TCellAgent.configuration.should_intercept_requests?
           if tcell_context && tcell_context.session_id
             session_id_actions = get_actions_for_session_id
@@ -326,8 +334,7 @@ module TCellAgent
       end
 
       def response_body_enforce(tcell_context, sanitize_string)
-        if TCellAgent.configuration.enabled &&
-           TCellAgent.configuration.should_instrument? &&
+        if TCellAgent.configuration.should_instrument? &&
            TCellAgent.configuration.should_intercept_requests?
           if tcell_context && tcell_context.session_id
             session_id_actions = get_actions_for_session_id
@@ -367,6 +374,8 @@ end
 class Logger
   alias_method :tcell_old_add, :add
   def add(severity, message = nil, progname = nil)
+    return tcell_old_add(severity, message, progname) unless severity >= level
+
     if severity >= level
       progname ||= @progname
       if message.nil?
@@ -383,9 +392,12 @@ class Logger
          TCellAgent.configuration.should_intercept_requests?
 
         TCellAgent::Instrumentation.safe_block_no_log('Handling DLP log message filtering') do
-          dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
+          dataloss_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
+          return tcell_old_add(severity, message, progname) unless dataloss_policy && dataloss_policy.enabled
+
           request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
-          if message && dlp_policy && request_env
+
+          if message && request_env
             tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
             tcell_context.filter_log(message) if tcell_context
           end

data/lib/tcell_agent/rails/dlp_handler.rb

@@ -38,8 +38,7 @@ module TCellAgent
           tcell_context = nil
 
           TCellAgent::Instrumentation.safe_block('DLP Handler get handler and context') do
-            if TCellAgent.configuration.enabled &&
-               TCellAgent.configuration.should_instrument? &&
+            if TCellAgent.configuration.should_instrument? &&
                TCellAgent.configuration.should_intercept_requests?
 
               # do all this work so that dlp doesn't run at all unless it's on and there

data/lib/tcell_agent/rails/js_agent_insert.rb

@@ -44,19 +44,18 @@ module TCellAgent
           script_insert = nil
 
           TCellAgent::Instrumentation.safe_block('JSAgent get handler and script insert') do
-            if (response_headers['Content-Type'] || '').start_with?('text/html')
-              rust_policy = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
-              if rust_policy
-                script_insert = rust_policy.get_js_agent_script_tag(
-                  request.env[TCellAgent::Instrumentation::TCELL_ID]
-                )
-                if script_insert
-                  js_agent_handler = proc { |si, resp|
-                    handle_js_agent_insert(si, resp)
-                  }
-                end
-              end
-            end
+            return [nil, nil] unless (response_headers['Content-Type'] || '').start_with?('text/html')
+
+            js_agent_policy = TCellAgent.policy(TCellAgent::PolicyTypes::JSAGENTINJECTION)
+            script_insert = js_agent_policy.get_js_agent_script_tag(
+              request.env[TCellAgent::Instrumentation::TCELL_ID]
+            )
+
+            return [nil, nil] unless script_insert
+
+            js_agent_handler = proc { |si, resp|
+              handle_js_agent_insert(si, resp)
+            }
           end
 
           [js_agent_handler, script_insert]

data/lib/tcell_agent/rails/middleware/body_filter_middleware.rb

@@ -2,7 +2,6 @@
 
 require 'rails'
 require 'uri'
-require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/sensor_events/sensor'
 require 'tcell_agent/sensor_events/server_agent'
@@ -22,34 +21,14 @@ module TCellAgent
           end
 
           def call(env)
-            orig = (Time.now.to_f * 1000).to_i
+            start_time = (Time.now.to_f * 1000).to_i
 
             response = @app.call(env)
 
             if TCellAgent.configuration.should_intercept_requests?
-              response_time = (Time.now.to_f * 1000).to_i - orig
-              TCellAgent::Instrumentation.safe_block('Handling Route Time') do
-                route_id = env[TCellAgent::Instrumentation::TCELL_ID].route_id
-                if route_id
-                  TCellAgent.increment_route(route_id, response_time)
-                else
-                  TCellAgent.increment_route('', response_time)
-                end
-              end
-              TCellAgent::Instrumentation.safe_block('Handling Sessions Info') do
-                login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LOGINFRAUD)
-                if login_fraud_policy && login_fraud_policy.session_hijacking_metrics
-                  hmac_session_id = env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
-                  user_id = env[TCellAgent::Instrumentation::TCELL_ID].user_id
-                  if user_id && hmac_session_id
-                    TCellAgent.increment_session_info(
-                      hmac_session_id,
-                      user_id,
-                      env[TCellAgent::Instrumentation::TCELL_ID].ip_address,
-                      env[TCellAgent::Instrumentation::TCELL_ID].user_agent
-                    )
-                  end
-                end
+              TCellAgent::Instrumentation.safe_block('Handling reporting metrics') do
+                response_time = (Time.now.to_f * 1000).to_i - start_time
+                TCellAgent.report_metrics(response_time, env[TCellAgent::Instrumentation::TCELL_ID])
               end
             end
 

data/lib/tcell_agent/rails/middleware/context_middleware.rb

@@ -2,18 +2,8 @@
 
 require 'rails'
 require 'uri'
-require 'tcell_agent/logger'
 require 'tcell_agent/agent'
-require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/server_agent'
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
-
-require 'tcell_agent/userinfo'
-require 'cgi'
-
-require 'tcell_agent/instrumentation'
-require 'tcell_agent/configuration'
-require 'thread'
+require 'tcell_agent/rails/better_ip'
 
 module TCellAgent
   module Instrumentation
@@ -36,7 +26,7 @@ module TCellAgent
                 env[TCellAgent::Instrumentation::TCELL_ID].path = request.path
                 env[TCellAgent::Instrumentation::TCELL_ID].user_agent = request.user_agent
                 env[TCellAgent::Instrumentation::TCELL_ID].referrer = request.referrer
-                env[TCellAgent::Instrumentation::TCELL_ID].ip_address = TCellAgent::Utils::Rails.better_ip(request)
+                env[TCellAgent::Instrumentation::TCELL_ID].remote_address = TCellAgent::Utils::Rails.better_ip(request)
                 if request.request_method
                   env[TCellAgent::Instrumentation::TCELL_ID].request_method = request.request_method
                 end

data/lib/tcell_agent/rails/middleware/global_middleware.rb

@@ -2,13 +2,12 @@
 
 require 'rails'
 require 'uri'
-require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/sensor_events/sensor'
 require 'tcell_agent/sensor_events/server_agent'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 
-require 'tcell_agent/userinfo'
+require 'tcell_agent/rails/auth/userinfo'
 require 'cgi'
 
 require 'tcell_agent/instrumentation'

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -1,16 +1,3 @@
-# See the file "LICENSE" for the full license governing this code.
-
-require 'rails'
-require 'uri'
-require 'tcell_agent/agent'
-require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/appsensor_meta_event'
-require 'tcell_agent/sensor_events/server_agent'
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
-
-require 'tcell_agent/userinfo'
-require 'cgi'
-
 require 'tcell_agent/instrumentation'
 require 'tcell_agent/rails/responses'
 require 'tcell_agent/rails/js_agent_insert'
@@ -49,9 +36,9 @@ module TCellAgent
           def _set_headers(request, response)
             status, headers, active_response = response
 
-            rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
-            if rust_policies
-              policy_headers = rust_policies.get_headers(
+            TCellAgent::Instrumentation.safe_block('Handling headers') do
+              headers_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HEADERS)
+              policy_headers = headers_policy.get_headers(
                 request.env[TCellAgent::Instrumentation::TCELL_ID]
               )
               policy_headers.each do |header_info|
@@ -73,21 +60,14 @@ module TCellAgent
           def _handle_redirect(request, response)
             TCellAgent::Instrumentation.safe_block('Handling Redirect Headers') do
               status, headers, active_response = response
-              http_redirect_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HTTPREDIRECT)
-              if http_redirect_policy && headers.key?('Location')
+              if headers.key?('Location')
+                http_redirect_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HTTPREDIRECT)
                 tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
-                route_id = tcell_context.route_id
-                hmac_session_id = tcell_context.hmac_session_id
-                new_location = http_redirect_policy.enforce(
-                  headers['Location'],
-                  tcell_context.uri,
-                  tcell_context.fullpath,
-                  tcell_context.request_method,
-                  route_id,
-                  status,
-                  tcell_context.ip_address,
-                  hmac_session_id
+                from_domain = URI.parse(tcell_context.uri).host
+                new_location = http_redirect_policy.check_redirect(
+                  headers['Location'], from_domain, status, tcell_context
                 )
+
                 # Enforcement
                 headers['Location'] = new_location if new_location
               end
@@ -143,15 +123,15 @@ module TCellAgent
                 defer_appfw_due_to_streaming = true
               end
 
-              rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
-              if rust_policies && rust_policies.appfirewall_enabled
-                event = TCellAgent::SensorEvents::AppSensorMetaEvent.build(
+              appfirewall_policy = TCellAgent.policy(TCellAgent::PolicyTypes::APPSENSOR)
+              if appfirewall_policy.enabled
+                meta_data = TCellAgent::MetaData.for_appfirewall(
                   request, content_length, status_code, response_headers
                 )
                 if defer_appfw_due_to_streaming
-                  response_body.appsensor_meta_event = event
+                  response_body.meta_data = meta_data
                 else
-                  TCellAgent.send_event(event)
+                  appfirewall_policy.check_appfirewall_injections(meta_data)
                 end
               end
 

data/lib/tcell_agent/{rails.rb → rails/railties/tcell_agent_railties.rb}

@@ -1,40 +1,35 @@
 # See the file "LICENSE" for the full license governing this code.
 
 require 'rails'
-require 'uri'
-require 'tcell_agent/agent'
-require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/server_agent'
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 
-require 'tcell_agent/rails/better_ip'
 require 'tcell_agent/rails/middleware/global_middleware'
 require 'tcell_agent/rails/middleware/body_filter_middleware'
 require 'tcell_agent/rails/middleware/headers_middleware'
 require 'tcell_agent/rails/middleware/context_middleware'
 
+require 'tcell_agent/rails/routes'
 require 'tcell_agent/rails/settings_reporter'
 require 'tcell_agent/rails/dlp'
 require 'tcell_agent/rails/csrf_exception'
 
-require 'tcell_agent/userinfo'
 require 'cgi'
 require 'thread'
 
 module TCellAgent
-  class Railtie < Rails::Railtie
-    initializer 'tcell_agent.insert_middleware' do |app|
-      app.config.to_prepare do
-        require 'tcell_agent/devise' if defined?(Devise)
-        require 'tcell_agent/rails/auth/devise' if defined?(Devise)
-        require 'tcell_agent/authlogic' if defined?(Authlogic)
-        require 'tcell_agent/rails/auth/authlogic' if defined?(Authlogic)
-        require 'tcell_agent/rails/auth/doorkeeper'
-      end
+  class TCellAgentStartupRailtie < Rails::Railtie
+    initializer :tcell_insert_middleware, :before => :build_middleware_stack do |app|
       app.config.middleware.insert_before(0, TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware)
       app.config.middleware.insert_after(0, TCellAgent::Instrumentation::Rails::Middleware::HeadersMiddleware)
       app.config.middleware.use TCellAgent::Instrumentation::Rails::Middleware::BodyFilterMiddleware
       app.config.middleware.use TCellAgent::Instrumentation::Rails::Middleware::GlobalMiddleware
     end
   end
+
+  class TCellAgentCSRFRailtie < Rails::Railtie
+    initializer 'tcell.sensors' do |_app|
+      ActiveSupport.on_load :action_controller do
+        ActionController::Base.send(:include, TCellAgent::CsrfExceptionReporter)
+      end
+    end
+  end
 end

data/lib/tcell_agent/rails/railties/tcell_agent_unicorn_railties.rb

@@ -0,0 +1,8 @@
+module TCellAgent
+  class TCellAgentStartupRailtie < Rails::Railtie
+    initializer :start_tcell_agent,
+                :after => :load_config_initializers do |_app|
+      TCellAgent.thread_agent.start('Unicorn')
+    end
+  end
+end

data/lib/tcell_agent/rails/routes.rb

@@ -1,7 +1,7 @@
-require 'tcell_agent/configuration'
 require 'tcell_agent/patches'
 require 'tcell_agent/rails/routes/grape'
 require 'tcell_agent/rails/routes/route_id'
+require 'tcell_agent/rails/dlp/process_request'
 
 require 'json'
 
@@ -77,13 +77,15 @@ module TCellAgent
       end
 
       def self.create_tcell_route(route)
-        return TCellRoute5.new(route) if route && ::Rails::VERSION::MAJOR == 5
+        return TCellRoute5.new(route) if route && ::Rails::VERSION::MAJOR >= 5
         return TCellRoute4.new(route) if route && ::Rails::VERSION::MAJOR < 5
 
         TCellRoute.new
       end
 
       def self.instrument_routes
+        return unless TCellAgent.configuration.should_instrument?
+
         return unless ::Rails.application
 
         ::Rails.application.routes.routes.each do |route|
@@ -92,7 +94,7 @@ module TCellAgent
       end
 
       def self.instrument_route(route)
-        return unless TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument?
+        return unless TCellAgent.configuration.should_instrument?
 
         tcell_route = create_tcell_route(route)
 
@@ -134,9 +136,7 @@ module TCellAgent
               prepend_around_filter :tcell_around_filter_routes
             end
             def tcell_around_filter_routes
-              if TCellAgent.configuration.enabled &&
-                 TCellAgent.configuration.should_instrument? &&
-                 TCellAgent.configuration.should_intercept_requests?
+              if TCellAgent.configuration.should_intercept_requests?
                 TCellAgent::Instrumentation.safe_block('Determining Rails Route ID') do
                   _match, parameters, route = ::Rails.application.routes.router.recognize(request) { |r, _| r }.first
 
@@ -172,7 +172,7 @@ module TCellAgent
         end
       end
 
-      if ::Rails::VERSION::MAJOR == 5
+      if ::Rails::VERSION::MAJOR >= 5
         ActionDispatch::Journey::Routes.class_eval do
           alias_method :tcell_add_route, :add_route
           def add_route(name, mapping)
@@ -187,12 +187,11 @@ module TCellAgent
         end
       end
 
-      if ::Rails::VERSION::MAJOR == 5 || (::Rails::VERSION::MAJOR == 4 && ::Rails::VERSION::MINOR >= 2)
+      if ::Rails::VERSION::MAJOR >= 5 || (::Rails::VERSION::MAJOR == 4 && ::Rails::VERSION::MINOR >= 2)
         ActionDispatch::Journey::Router.class_eval do
           alias_method :tcell_serve, :serve
           def serve(req)
-            if TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument? &&
-               TCellAgent.configuration.should_intercept_requests?
+            if TCellAgent.configuration.should_intercept_requests?
               TCellAgent::Instrumentation.safe_block('Determining Rails Route ID') do
                 _match, parameters, route = find_routes(req).first
 
@@ -219,8 +218,7 @@ module TCellAgent
           def call(env)
             env['PATH_INFO'] = ActionDispatch::Journey::Router::Utils.normalize_path(env['PATH_INFO'])
 
-            if TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument? &&
-               TCellAgent.configuration.should_intercept_requests?
+            if TCellAgent.configuration.should_intercept_requests?
               TCellAgent::Instrumentation.safe_block('Determining Rails Route ID') do
                 _match, parameters, route = find_routes(env).first