tcell_agent 1.1.12 → 2.0.0

data/lib/tcell_agent/rails/middleware/global_middleware.rb

@@ -2,7 +2,6 @@
 
 require 'rails'
 require 'uri'
-require 'tcell_agent/logger'
 require 'tcell_agent/agent'
 require 'tcell_agent/sensor_events/sensor'
 require 'tcell_agent/sensor_events/server_agent'

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -1,16 +1,3 @@
-# See the file "LICENSE" for the full license governing this code.
-
-require 'rails'
-require 'uri'
-require 'tcell_agent/agent'
-require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/appsensor_meta_event'
-require 'tcell_agent/sensor_events/server_agent'
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
-
-require 'tcell_agent/userinfo'
-require 'cgi'
-
 require 'tcell_agent/instrumentation'
 require 'tcell_agent/rails/responses'
 require 'tcell_agent/rails/js_agent_insert'
@@ -49,9 +36,9 @@ module TCellAgent
           def _set_headers(request, response)
             status, headers, active_response = response
 
-            rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
-            if rust_policies
-              policy_headers = rust_policies.get_headers(
+            TCellAgent::Instrumentation.safe_block('Handling headers') do
+              headers_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HEADERS)
+              policy_headers = headers_policy.get_headers(
                 request.env[TCellAgent::Instrumentation::TCELL_ID]
               )
               policy_headers.each do |header_info|
@@ -73,21 +60,14 @@ module TCellAgent
           def _handle_redirect(request, response)
             TCellAgent::Instrumentation.safe_block('Handling Redirect Headers') do
               status, headers, active_response = response
-              http_redirect_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HTTPREDIRECT)
-              if http_redirect_policy && headers.key?('Location')
+              if headers.key?('Location')
+                http_redirect_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HTTPREDIRECT)
                 tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
-                route_id = tcell_context.route_id
-                hmac_session_id = tcell_context.hmac_session_id
-                new_location = http_redirect_policy.enforce(
-                  headers['Location'],
-                  tcell_context.uri,
-                  tcell_context.fullpath,
-                  tcell_context.request_method,
-                  route_id,
-                  status,
-                  tcell_context.ip_address,
-                  hmac_session_id
+                from_domain = URI.parse(tcell_context.uri).host
+                new_location = http_redirect_policy.check_redirect(
+                  headers['Location'], from_domain, status, tcell_context
                 )
+
                 # Enforcement
                 headers['Location'] = new_location if new_location
               end
@@ -143,15 +123,15 @@ module TCellAgent
                 defer_appfw_due_to_streaming = true
               end
 
-              rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
-              if rust_policies && rust_policies.appfirewall_enabled
-                event = TCellAgent::SensorEvents::AppSensorMetaEvent.build(
+              appfirewall_policy = TCellAgent.policy(TCellAgent::PolicyTypes::APPSENSOR)
+              if appfirewall_policy.enabled
+                meta_data = TCellAgent::MetaData.for_appfirewall(
                   request, content_length, status_code, response_headers
                 )
                 if defer_appfw_due_to_streaming
-                  response_body.appsensor_meta_event = event
+                  response_body.meta_data = meta_data
                 else
-                  TCellAgent.send_event(event)
+                  appfirewall_policy.check_appfirewall_injections(meta_data)
                 end
               end
 

data/lib/tcell_agent/rails/on_start.rb

@@ -1,41 +1,42 @@
 # See the file "LICENSE" for the full license governing this code.
 
-# require 'tcell_agent/authlogic' if defined?(Authlogic)
-
 require 'rails'
 
-require 'tcell_agent/configuration'
-
 require 'tcell_agent/rails/routes'
-require 'tcell_agent/rails/dlp/process_request'
-
-TCellAgent::Instrumentation::Rails.send_language_info
-TCellAgent::Instrumentation::Rails.send_framework_info
 
-if Rails.application
-  if TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument?
-    TCellAgent::Instrumentation::Rails.send_settings(Rails.application)
-  end
+require 'tcell_agent/rails/middleware/global_middleware'
+require 'tcell_agent/rails/middleware/body_filter_middleware'
+require 'tcell_agent/rails/middleware/headers_middleware'
+require 'tcell_agent/rails/middleware/context_middleware'
+
+require 'tcell_agent/rails/settings_reporter'
+require 'tcell_agent/rails/dlp'
+require 'tcell_agent/rails/csrf_exception'
+
+require 'tcell_agent/userinfo'
+require 'cgi'
+require 'thread'
+
+module TCellAgent
+  class TCellAgentStartupRailtie < Rails::Railtie
+    # TCellAgent config can be specified thru
+    # Rails initializer's (https://guides.rubyonrails.org/v2.3/configuring.html#using-initializers)
+    # so those need to run first since this relies on configuration
+    initializer :tcell_instrument_auth_frameworks, :after => :load_config_initializers do |_app|
+      next unless TCellAgent.configuration.should_instrument?
+
+      require 'tcell_agent/devise'
+      require 'tcell_agent/rails/auth/devise'
+      require 'tcell_agent/authlogic'
+      require 'tcell_agent/rails/auth/authlogic'
+      require 'tcell_agent/rails/auth/doorkeeper'
+    end
 
-else
-  module TCellAgent
-    class MyRailtie < Rails::Railtie
-      initializer 'activeservice.autoload', :after => :set_autoload_paths do |_app|
-        Rails.application.config.to_prepare do
-          require 'tcell_agent/devise' if defined?(Devise)
-          require 'tcell_agent/rails/auth/devise' if defined?(Devise)
-          require 'tcell_agent/authlogic' if defined?(Authlogic)
-          require 'tcell_agent/rails/auth/authlogic' if defined?(Authlogic)
-          require 'tcell_agent/rails/auth/doorkeeper'
-        end
-
-        # TODO: will this get run ever?
-        if TCellAgent.configuration.enabled
-          Rails.application.config.after_initialize do
-            TCellAgent::Instrumentation::Rails.send_settings(Rails.application)
-          end
-        end
-      end
+    initializer :tcell_insert_middleware, :before => :build_middleware_stack do |app|
+      app.config.middleware.insert_before(0, TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware)
+      app.config.middleware.insert_after(0, TCellAgent::Instrumentation::Rails::Middleware::HeadersMiddleware)
+      app.config.middleware.use TCellAgent::Instrumentation::Rails::Middleware::BodyFilterMiddleware
+      app.config.middleware.use TCellAgent::Instrumentation::Rails::Middleware::GlobalMiddleware
     end
   end
 end

data/lib/tcell_agent/rails/routes.rb

@@ -1,7 +1,7 @@
-require 'tcell_agent/configuration'
 require 'tcell_agent/patches'
 require 'tcell_agent/rails/routes/grape'
 require 'tcell_agent/rails/routes/route_id'
+require 'tcell_agent/rails/dlp/process_request'
 
 require 'json'
 
@@ -84,6 +84,8 @@ module TCellAgent
       end
 
       def self.instrument_routes
+        return unless TCellAgent.configuration.should_instrument?
+
         return unless ::Rails.application
 
         ::Rails.application.routes.routes.each do |route|
@@ -92,7 +94,7 @@ module TCellAgent
       end
 
       def self.instrument_route(route)
-        return unless TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument?
+        return unless TCellAgent.configuration.should_instrument?
 
         tcell_route = create_tcell_route(route)
 
@@ -134,8 +136,7 @@ module TCellAgent
               prepend_around_filter :tcell_around_filter_routes
             end
             def tcell_around_filter_routes
-              if TCellAgent.configuration.enabled &&
-                 TCellAgent.configuration.should_instrument? &&
+              if TCellAgent.configuration.should_instrument? &&
                  TCellAgent.configuration.should_intercept_requests?
                 TCellAgent::Instrumentation.safe_block('Determining Rails Route ID') do
                   _match, parameters, route = ::Rails.application.routes.router.recognize(request) { |r, _| r }.first
@@ -191,7 +192,7 @@ module TCellAgent
         ActionDispatch::Journey::Router.class_eval do
           alias_method :tcell_serve, :serve
           def serve(req)
-            if TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument? &&
+            if TCellAgent.configuration.should_instrument? &&
                TCellAgent.configuration.should_intercept_requests?
               TCellAgent::Instrumentation.safe_block('Determining Rails Route ID') do
                 _match, parameters, route = find_routes(req).first
@@ -219,7 +220,7 @@ module TCellAgent
           def call(env)
             env['PATH_INFO'] = ActionDispatch::Journey::Router::Utils.normalize_path(env['PATH_INFO'])
 
-            if TCellAgent.configuration.enabled && TCellAgent.configuration.should_instrument? &&
+            if TCellAgent.configuration.should_instrument? &&
                TCellAgent.configuration.should_intercept_requests?
               TCellAgent::Instrumentation.safe_block('Determining Rails Route ID') do
                 _match, parameters, route = find_routes(env).first

data/lib/tcell_agent/rails/routes/grape.rb

@@ -76,8 +76,7 @@ module TCellAgent
       Grape::Endpoint.class_eval do
         alias_method :tcell_call!, :call!
         def call!(env)
-          if TCellAgent.configuration.enabled &&
-             TCellAgent.configuration.should_instrument? &&
+          if TCellAgent.configuration.should_instrument? &&
              TCellAgent.configuration.should_intercept_requests?
 
             TCellAgent::Instrumentation.safe_block('Determining Rails Route ID') do
@@ -102,7 +101,6 @@ module TCellAgent
           tcell_call!(env)
         end
       end
-
     end
   end
 end

data/lib/tcell_agent/rails/routes/route_id.rb

@@ -16,7 +16,9 @@ module TCellAgent
           tcell_context.grape_mount_endpoint = grape_mount_endpoint
 
         else
-          tcell_context.route_id = TCellAgent::SensorEvents::Util.calculate_route_id(tcell_context.request_method, route_path)
+          tcell_context.route_id = TCellAgent::SensorEvents::Util.calculate_route_id(
+            tcell_context.request_method, route_path
+          )
         end
       end
     end

data/lib/tcell_agent/rails/settings_reporter.rb

@@ -1,15 +1,12 @@
 require 'rails'
 require 'tcell_agent'
-require 'tcell_agent/sensor_events/app_config'
+require 'tcell_agent/sensor_events/app_config_setting_event'
 require 'tcell_agent/sensor_events/server_agent'
-require 'tcell_agent/system_info'
 
 module TCellAgent
   module Instrumentation
     module Rails
       def self.send_framework_info
-        return unless TCellAgent.configuration.exp_config_settings
-
         TCellAgent.send_event(
           TCellAgent::SensorEvents::ServerAgentAppFrameworkEvent.new(
             'Rails', ::Rails.version
@@ -17,44 +14,34 @@ module TCellAgent
         )
       end
 
-      def self.send_language_info
-        return unless TCellAgent.configuration.exp_config_settings
+      def self.send_settings
+        TCellAgent::Instrumentation.safe_block('Reporting Rails settings') do
+          rails_config = ::Rails.application.config
 
-        language = TCellAgent::SystemInfo.get_language
-        language_version = TCellAgent::SystemInfo.get_language_version
-        TCellAgent.send_event(
-          TCellAgent::SensorEvents::ServerAgentDetailsLanguageEvent.new(
-            language, language_version
+          # Defaults to true
+          csrf_protection = rails_config.action_controller.allow_forgery_protection || true
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::AppConfigSettingEvent.new(
+              'Rails', 'core', '', 'csrf_protection', csrf_protection
+            )
           )
-        )
-      end
 
-      def self.send_settings(application)
-        return unless TCellAgent.configuration.exp_config_settings
-
-        # Defaults to true
-        csrf_protection = application.config.action_controller.allow_forgery_protection || true
-        TCellAgent.send_event(
-          TCellAgent::SensorEvents::AppConfigSettingEvent.new(
-            'Rails', 'core', '', 'csrf_protection', csrf_protection
+          # Defaults to false if nil
+          mass_assignment_allowed = rails_config.action_controller.permit_all_parameters || false
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::AppConfigSettingEvent.new(
+              'Rails', 'core', '', 'mass_assignment_allowed', mass_assignment_allowed
+            )
           )
-        )
 
-        # Defaults to false if nil
-        mass_assignment_allowed = application.config.action_controller.permit_all_parameters || false
-        TCellAgent.send_event(
-          TCellAgent::SensorEvents::AppConfigSettingEvent.new(
-            'Rails', 'core', '', 'mass_assignment_allowed', mass_assignment_allowed
-          )
-        )
-
-        # Defaults to never
-        session_expire = application.config.session_options[:expire_after] || -1
-        TCellAgent.send_event(
-          TCellAgent::SensorEvents::AppConfigSettingEvent.new(
-            'Rails', 'session', '', 'timeout', session_expire
+          # Defaults to never
+          session_expire = rails_config.session_options[:expire_after] || -1
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::AppConfigSettingEvent.new(
+              'Rails', 'session', '', 'timeout', session_expire
+            )
           )
-        )
+        end
       end
     end
   end

data/lib/tcell_agent/rails/start_agent_after_initializers.rb

@@ -0,0 +1,12 @@
+module TCellAgent
+  class TCellAgentStartupRailtie < Rails::Railtie
+    # TCellAgent config can be specified thru Rails initializer's
+    # (https://guides.rubyonrails.org/v2.3/configuring.html#using-initializers)
+    # so those need to run first before the agent is started
+    initializer :start_tcell_agent,
+                :after => :load_config_initializers,
+                :before => :tcell_instrument_auth_frameworks do |_app|
+      TCellAgent.thread_agent.start('Unicorn')
+    end
+  end
+end

data/lib/tcell_agent/rails/tcell_body_proxy.rb

@@ -4,7 +4,7 @@ module TCellAgent
   module Instrumentation
     module Rails
       class TCellBodyProxy
-        attr_accessor :appsensor_meta_event
+        attr_accessor :meta_data
 
         # for specs
         attr_accessor :content_length
@@ -29,9 +29,11 @@ module TCellAgent
 
         def close
           TCellAgent::Instrumentation.safe_block('Running AppSensor deferred due to streaming') do
-            if @appsensor_meta_event
-              @appsensor_meta_event.meta_data.response_content_bytes_len = @content_length
-              TCellAgent.send_event(@appsensor_meta_event)
+            if @meta_data
+              @meta_data.response_content_bytes_len = @content_length
+
+              appfirewall_policy = TCellAgent.policy(TCellAgent::PolicyTypes::APPSENSOR)
+              appfirewall_policy.check_appfirewall_injections(@meta_data)
             end
           end
 

data/lib/tcell_agent/rust/agent_config.rb

@@ -0,0 +1,49 @@
+require 'tcell_agent/version'
+
+module TCellAgent
+  module Rust
+    class AgentConfig < Hash
+      def initialize(configuration)
+        send_mode = 'Normal'
+        send_mode = 'Demo' if configuration.demomode
+
+        logging_options = configuration.clean_logging_options
+        unless configuration.js_agent_api_base_url
+          parsed_uri = URI.parse(configuration.tcell_api_url)
+          api_url = [parsed_uri.scheme, '://', parsed_uri.host]
+          api_url.push(":#{parsed_uri.port}") unless [80, 443].include?(parsed_uri.port)
+          configuration.js_agent_api_base_url = "#{api_url.join('')}/api/v1"
+        end
+
+        self['disable_event_sending'] = !configuration.should_start_event_manager?
+        self['send_mode'] = send_mode
+        self['agent_type'] = 'Ruby'
+        self['agent_version'] = TCellAgent::VERSION
+        self['diagnostics_enabled'] = false
+        self['application'] = {
+          :app_id => configuration.app_id,
+          :api_key =>  configuration.api_key,
+          :tcell_api_url => configuration.tcell_api_url,
+          :tcell_input_url => configuration.tcell_input_url,
+          :hmac_key => configuration.hmac_key,
+          :password_hmac_key => configuration.password_hmac_key,
+          :allow_payloads => configuration.allow_payloads,
+          :js_agent_api_base_url => configuration.js_agent_api_base_url,
+          :js_agent_url => configuration.js_agent_url,
+          :cache_dir => configuration.cache_folder,
+          :log_dir => configuration.agent_log_dir,
+          :logging_options => logging_options,
+          :host_identifier => configuration.host_identifier,
+          :reverse_proxy_ip_address_header => configuration.reverse_proxy_ip_address_header,
+          :fetch_policies_from_tcell => configuration.should_start_policy_poll?,
+          :preload_policy_filename => configuration.preload_policy_filename
+        }
+        self['appfirewall'] = {
+          :enable_body_json_inspection => true,
+          :allow_log_payloads => true
+        }
+        self['max_header_size'] = configuration.max_csp_header_bytes || (1024 * 1024)
+      end
+    end
+  end
+end

data/lib/tcell_agent/rust/models.rb

@@ -13,61 +13,6 @@ module TCellAgent
 
         flattened_params
       end
-
-      def self.create_request_response(appsensor_meta)
-        post_params = convert_params(appsensor_meta.flattened_post_dict)
-
-        request_response = {
-          'method' =>  appsensor_meta.method,
-          'status_code' =>  appsensor_meta.response_code.to_i,
-          'route_id' =>  appsensor_meta.route_id,
-          'path' =>  appsensor_meta.path,
-          'query_params' =>  convert_params(appsensor_meta.flattened_get_dict),
-          'post_params' =>  post_params,
-          'headers' =>  convert_params(appsensor_meta.flattened_headers_dict),
-          'cookies' =>  convert_params(appsensor_meta.flattened_cookie_dict),
-          'path_params' =>  convert_params(appsensor_meta.flattened_path_parameters),
-          'remote_address' =>  appsensor_meta.remote_address,
-          'full_uri' =>  appsensor_meta.location,
-          'session_id' =>  appsensor_meta.session_id,
-          'user_id' =>  appsensor_meta.user_id,
-          'user_agent' =>  appsensor_meta.user_agent,
-          :content_type => appsensor_meta.content_type,
-          :request_body => appsensor_meta.raw_request_body,
-          'request_bytes_length' =>  appsensor_meta.request_content_bytes_len,
-          'response_bytes_length' =>  appsensor_meta.response_content_bytes_len
-        }
-
-        if TCellAgent::Utils::Strings.present?(appsensor_meta.csrf_exception_name)
-          request_response['csrf_exception'] = { 'exception_name' => appsensor_meta.csrf_exception_name }
-        end
-
-        if appsensor_meta.sql_exceptions
-          request_response['sql_exceptions'] = appsensor_meta.sql_exceptions
-        end
-
-        if appsensor_meta.database_result_sizes
-          request_response['database_result_sizes'] = appsensor_meta.database_result_sizes
-        end
-
-        request_response
-      end
-
-      def self.create_patches_request(appsensor_meta)
-        post_params = convert_params(appsensor_meta.flattened_post_dict)
-
-        {
-          'method' =>  appsensor_meta.method,
-          'path' =>  appsensor_meta.path,
-          'remote_address' =>  appsensor_meta.remote_address,
-          'request_bytes_length' =>  appsensor_meta.request_content_bytes_len,
-          'query_params' =>  convert_params(appsensor_meta.flattened_get_dict),
-          'post_params' =>  post_params,
-          'headers' =>  convert_params(appsensor_meta.flattened_headers_dict),
-          'cookies' =>  convert_params(appsensor_meta.flattened_cookie_dict),
-          :content_type => appsensor_meta.content_type
-        }
-      end
     end
   end
 end