tcell_agent 1.1.12 → 2.0.0

data/lib/tcell_agent/configuration.rb

@@ -26,10 +26,9 @@ module TCellAgent
                   :app_id,
                   :api_key,
                   :hmac_key,
+                  :tcell_api_url,
                   :tcell_input_url,
                   :logging_options,
-                  :logger,
-                  :appfirewall_payloads_logger, # appfirewall_payloads_logger can be specified from initializers
                   :fetch_policies_from_tcell, :instrument_for_events,
                   :preload_policy_filename,
                   :host_identifier,
@@ -37,13 +36,13 @@ module TCellAgent
                   :event_batch_size_limit, :event_time_limit_seconds,
                   :base_dir,
                   :cache_filename,
+                  :cache_folder,
                   :js_agent_api_base_url,
                   :js_agent_url,
                   :config_filename,
                   :agent_log_dir,
                   :max_data_ex_db_records_per_request,
                   :agent_home_dir,
-                  :agent_home_owner,
                   :reverse_proxy,
                   :reverse_proxy_ip_address_header,
                   :log_file_name,
@@ -51,9 +50,8 @@ module TCellAgent
                   :max_csp_header_bytes,
                   :demomode,
                   :allow_payloads,
-                  :password_hmac_key
-
-    attr_reader   :tcell_api_url
+                  :password_hmac_key,
+                  :stdout_logger
 
     attr_accessor :disable_all,
                   :enabled,
@@ -61,8 +59,7 @@ module TCellAgent
                   :enable_event_consumer,      # false = Do not consume events, drop them
                   :enable_policy_polling,      # false = Do not poll for policies
                   :enable_instrumentation,     # false = Do not add instrumentation
-                  :enable_intercept_requests,  # false = Do not insert middleware
-                  :enable_child_process_events # true = Start an event processor on all processes, even children
+                  :enable_intercept_requests   # false = Do not insert middleware
 
     attr_accessor :enabled_instrumentations
 
@@ -70,23 +67,10 @@ module TCellAgent
 
     attr_accessor :disable_cmdi_exec_instrumentation # true = disable cmdi Kernel::exec instrumentation
 
-    def tcell_api_url=(value)
-      @tcell_api_url = value
-      @tcell_api_url = compose_api_url!
-    end
-
     def should_start_event_manager?
       @enabled && @enable_event_manager
     end
 
-    def should_start_event_manager_in_child_processes?
-      @enabled && @enable_event_manager && @enable_child_process_events
-    end
-
-    def should_consume_event?
-      @enabled && @enable_event_manager && @enable_event_consumer
-    end
-
     def should_start_policy_poll?
       @enabled && @enable_policy_polling && @fetch_policies_from_tcell # fetch_policies_from_tcel = legacy
     end
@@ -134,11 +118,7 @@ module TCellAgent
       @agent_log_dir = nil
       @log_tag = nil
 
-      @logger = nil
-      @appfirewall_payloads_logger = nil
-
       @version = 0
-      @exp_config_settings = true
       @demomode = false
 
       @fetch_policies_from_tcell = true
@@ -147,11 +127,9 @@ module TCellAgent
       @disable_all = false
       @enabled = true
       @enable_event_manager = true
-      @enable_event_consumer = true
       @enable_policy_polling = true
       @enable_instrumentation = true
       @enable_intercept_requests = true
-      @enable_child_process_events = false
 
       @enabled_instrumentations = {
         :doorkeeper => true,
@@ -168,13 +146,17 @@ module TCellAgent
 
       @max_data_ex_db_records_per_request = 1000
       @reverse_proxy = true
-      @reverse_proxy_ip_address_header = nil
+      @reverse_proxy_ip_address_header = 'X-Forwarded-For'
       @allow_payloads = true
 
       @max_csp_header_bytes = nil
       @password_hmac_key = nil
+      @logging_options = {}
 
       @agent_home_dir = ENV['TCELL_AGENT_HOME'] || File.join(Dir.getwd, 'tcell')
+      @cache_folder = File.join(@agent_home_dir, 'cache/')
+      @agent_log_dir = File.join(@agent_home_dir, 'logs')
+
       @config_filename = ENV['TCELL_AGENT_CONFIG'] || File.join(Dir.getwd, filename)
 
       read_config_from_file(@config_filename)
@@ -185,27 +167,9 @@ module TCellAgent
         @event_time_limit_seconds = 2
       end
 
-      if ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS']
-        puts 'tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS is deprecated and will be removed in a future release. Please switch to TCELL_AGENT_ALLOW_PAYLOADS.'
-      end
-
-      if ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS']
-        puts 'tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS is deprecated and will be removed in a future release. Please switch to TCELL_AGENT_ALLOW_PAYLOADS.'
-      end
-
-      unless ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS'].nil?
-        @allow_payloads = [true, 'true', 'yes', '1'].include?(ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS'])
-      end
-      unless ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS'].nil?
-        @allow_payloads = [true, 'true', 'yes', '1'].include?(ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS'])
-      end
-      unless ENV['TCELL_AGENT_ALLOW_PAYLOADS'].nil?
-        @allow_payloads = [true, 'true', 'yes', '1'].include?(ENV['TCELL_AGENT_ALLOW_PAYLOADS'])
-      end
-
-      @tcell_api_url = compose_api_url!
-      @tcell_input_url ||= 'https://input.tcell.io/api/v1'
-      @js_agent_url ||= 'https://jsagent.tcell.io/tcellagent.min.js'
+      @tcell_api_url ||= 'https://us.agent.tcell.insight.rapid7.com/api/v1'
+      @tcell_input_url ||= 'https://us.input.tcell.insight.rapid7.com/api/v1'
+      @js_agent_url ||= 'https://us.jsagent.tcell.insight.rapid7.com/tcellagent.min.js'
 
       if @host_identifier.nil?
         begin
@@ -218,50 +182,6 @@ module TCellAgent
       @uuid = SecureRandom.uuid
     end
 
-    def compose_api_url!
-      @tcell_api_url ||= 'https://api.tcell.io'
-      parsed_uri = URI.parse(@tcell_api_url)
-
-      api_url = [
-        parsed_uri.scheme,
-        '://',
-        parsed_uri.host
-      ]
-
-      api_url.push(":#{parsed_uri.port}") unless [80, 443].include?(parsed_uri.port)
-
-      @js_agent_api_base_url ||= "#{api_url.join('')}/api/v1"
-
-      [
-        api_url.join(''),
-        '/agents/api/v1/apps/',
-        '{app_id}',
-        '/policies/latest',
-        '?',
-        'type=jsagentinjection:v1',
-        '&type=http-redirect:v1',
-        '&type=clickjacking:v1',
-        '&type=secure-headers:v1',
-        '&type=cmdi:v1',
-        '&type=csp-headers:v1',
-        '&type=dlp:v1',
-        '&type=login:v1',
-        '&type=regex:v1',
-        '&type=appsensor:v2',
-        '&type=patches:v1'
-      ].join('')
-    end
-
-    def cache_filename_with_app_id
-      @cache_filename ||= File.join(@agent_home_dir, 'cache', 'tcell_agent.cache')
-
-      if @app_id
-        "#{@cache_filename}.#{@app_id}"
-      else
-        @cache_filename
-      end
-    end
-
     def read_config_using_env
       @app_id = ENV['TCELL_AGENT_APP_ID'] || @app_id
       @api_key = ENV['TCELL_AGENT_API_KEY'] || @api_key
@@ -272,12 +192,16 @@ module TCellAgent
       @tcell_input_url = ENV['TCELL_INPUT_URL'] || @tcell_input_url
       @demomode = ENV['TCELL_DEMOMODE'] || @demomode
 
-      @agent_home_owner = ENV['TCELL_AGENT_HOME_OWNER'] || @agent_home_owner
       @agent_log_dir = ENV['TCELL_AGENT_LOG_DIR'] || @agent_log_dir
+      @log_file_name = ENV['TCELL_AGENT_LOG_FILENAME'] || @log_file_name
+
+      @logging_options['enabled'] = to_bool(ENV['TCELL_AGENT_LOG_ENABLED']) unless to_bool(ENV['TCELL_AGENT_LOG_ENABLED']).nil?
+      @logging_options['level'] = ENV['TCELL_AGENT_LOG_LEVEL'] || @logging_options['level'] unless @logging_options.nil?
 
-      @disable_cmdi_exec_instrumentation = ENV['TCELL_CMDI_EXEC_DISABLED'] || @disable_cmdi_exec_instrumentation
+      @enabled = to_bool(ENV['TCELL_AGENT_ENABLED']) unless to_bool(ENV['TCELL_AGENT_ENABLED']).nil?
 
-      @enabled = ENV['TCELL_AGENT_ENABLED'].to_s.casecmp('true').zero? if %w[true false].include? ENV['TCELL_AGENT_ENABLED'].to_s.downcase
+      @allow_payloads = to_bool(ENV['TCELL_AGENT_ALLOW_PAYLOADS']) unless to_bool(ENV['TCELL_AGENT_ALLOW_PAYLOADS']).nil?
+      @disable_cmdi_exec_instrumentation = to_bool(ENV['TCELL_CMDI_EXEC_DISABLED']) || @disable_cmdi_exec_instrumentation
     end
 
     def read_config_from_file(filename)
@@ -306,33 +230,21 @@ module TCellAgent
           @enabled = app_data.fetch('enabled', @enabled)
 
           @enable_event_manager = app_data.fetch('enable_event_manager', @enable_event_manager)
-          @enable_event_consumer = app_data.fetch('enable_event_consumer', @enable_event_consumer)
           @enable_policy_polling = app_data.fetch('enable_policy_polling', @enable_policy_polling)
           @enable_instrumentation = app_data.fetch('enable_instrumentation', @enable_instrumentation)
           @enable_intercept_requests = app_data.fetch('enable_intercept_requests', @enable_intercept_requests)
           @fetch_policies_from_tcell = app_data.fetch('fetch_policies_from_tcell', @fetch_policies_from_tcell)
           @instrument_for_events = app_data.fetch('instrument_for_events', @instrument_for_events)
-          @enable_child_process_events = app_data.fetch('enable_child_process_events', @enable_child_process_events)
-
-          @agent_home_owner = app_data.fetch('agent_home_owner', @agent_home_owner)
 
           @logging_options = app_data.fetch('logging_options', {})
           @agent_log_dir = app_data.fetch('log_dir', @agent_log_dir)
-          @log_file_name = @logging_options.fetch('filename', @log_file_name)
+          @log_file_name = @logging_options['filename'] || @log_file_name
 
           @tcell_api_url = app_data.fetch('tcell_api_url', @tcell_api_url)
           @tcell_input_url = app_data.fetch('tcell_input_url', @tcell_input_url)
 
           @max_csp_header_bytes = app_data.fetch('max_csp_header_bytes', @max_csp_header_bytes)
 
-          @allow_payloads = app_data.fetch(
-            'allow_unencrypted_appsensor_payloads',
-            @allow_payloads
-          )
-          @allow_payloads = app_data.fetch(
-            'allow_unencrypted_appfirewall_payloads',
-            @allow_payloads
-          )
           @allow_payloads = app_data.fetch(
             'allow_payloads',
             @allow_payloads
@@ -377,15 +289,19 @@ module TCellAgent
       end
     end
 
-    # old value could be set via initializers, this makes sure those initializers still work
-    # properly
-    def allow_unencrypted_appfirewall_payloads=(val)
-      @allow_payloads = val
+    def to_bool(var)
+      return unless var
+      var.to_s.casecmp('true').zero? if %w[true false].include? var.to_s.downcase
     end
 
-    # keep this around in case the value was read as well
-    def allow_unencrypted_appfirewall_payloads
-      @allow_payloads
+    def enforce_symbol_keys(hashmap)
+      hashmap.each_with_object({}) do |(k, v), memo|
+        memo[k.to_sym] = v
+      end
+    end
+
+    def logging_enabled?
+      @enabled && enforce_symbol_keys(@logging_options || {})[:enabled]
     end
 
     def log_filename
@@ -393,9 +309,12 @@ module TCellAgent
       File.join(@agent_log_dir, @log_file_name)
     end
 
-    def appfirewall_payloads_log_filename
-      @agent_log_dir ||= File.join(@agent_home_dir, 'logs')
-      File.join(@agent_log_dir, 'tcell_agent_payloads.log')
+    def clean_logging_options
+      {
+        :enabled => true,
+        :level => 'INFO',
+        :filename => log_file_name
+      }.merge(enforce_symbol_keys(@logging_options || {}))
     end
   end
 

data/lib/tcell_agent/devise.rb

@@ -1,33 +1,31 @@
-# See the file "LICENSE" for the full license governing this code.
+if TCellAgent.configuration.should_instrument_devise? && defined?(Devise)
+  require 'devise'
+  require 'devise/rails'
+  require 'devise/strategies/database_authenticatable'
+  require 'tcell_agent/userinfo'
 
-require 'devise'
-require 'devise/rails'
-require 'devise/strategies/database_authenticatable'
-require 'tcell_agent/userinfo'
-require 'tcell_agent/logger'
-require 'tcell_agent/sensor_events/honeytokens'
-
-module TCellAgent
-  if defined?(Devise)
-    TCellAgent::UserInformation.class_eval do
-      class << self
-        alias_method :original_get_user_from_request, :get_user_from_request
-        def get_user_from_request(request)
-          orig_user_id = original_get_user_from_request(request)
-          begin
-            if request.session && request.session.key?('warden.user.user.key')
-              userkey = request.session['warden.user.user.key']
-              user_id = if userkey.length == 2
-                          userkey[0][0]
-                        else
-                          userkey[1][0]
-                        end
-              return user_id.to_s if user_id.is_a? Integer
+  module TCellAgent
+    if defined?(Devise)
+      TCellAgent::UserInformation.class_eval do
+        class << self
+          alias_method :original_get_user_from_request, :get_user_from_request
+          def get_user_from_request(request)
+            orig_user_id = original_get_user_from_request(request)
+            begin
+              if request.session && request.session.key?('warden.user.user.key')
+                userkey = request.session['warden.user.user.key']
+                user_id = if userkey.length == 2
+                            userkey[0][0]
+                          else
+                            userkey[1][0]
+                          end
+                return user_id.to_s if user_id.is_a? Integer
+              end
+            rescue StandardError
+              return orig_user_id
             end
-          rescue StandardError
-            return orig_user_id
+            orig_user_id
           end
-          orig_user_id
         end
       end
     end

data/lib/tcell_agent/hooks/login_fraud.rb

@@ -1,47 +1,44 @@
 require 'tcell_agent/agent'
-require 'tcell_agent/sensor_events/login_fraud'
 
 module TCellAgent
   module Hooks
     module LoginFraud
+      # Note: mock out in tests
+      def self.get_logger
+        unless defined?(@login_fraud_logger)
+          @login_fraud_logger = TCellAgent::ModuleLogger.new(TCellAgent.logger, name)
+        end
+
+        @login_fraud_logger
+      end
+
       def self.report_login_event(status,
                                   env_or_header_keys,
                                   tcell_data,
                                   user_id,
                                   password,
                                   user_valid)
-        if TCellAgent.configuration.enabled &&
-           TCellAgent.configuration.should_intercept_requests?
-          login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LOGINFRAUD)
-
-          if login_fraud_policy && login_fraud_policy.enabled
-            if tcell_data
-              if ![TCellAgent::Hooks::V1::Login::LOGIN_FAILURE,
-                   TCellAgent::Hooks::V1::Login::LOGIN_SUCCESS].include?(status)
-                TCellAgent.logger.error("Unkown login status: #{status}")
+        return unless TCellAgent.configuration.should_intercept_requests? && tcell_data
 
-              elsif (status == TCellAgent::Hooks::V1::Login::LOGIN_FAILURE) &&
-                    login_fraud_policy.login_failed_enabled
-                TCellAgent.send_event(
-                  TCellAgent::SensorEvents::LoginFailure.new(env_or_header_keys,
-                                                             tcell_data,
-                                                             user_id,
-                                                             password,
-                                                             user_valid)
-                )
+        login_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LOGINFRAUD)
 
-              elsif (status == TCellAgent::Hooks::V1::Login::LOGIN_SUCCESS) &&
-                    login_fraud_policy.login_success_enabled
-                TCellAgent.send_event(
-                  TCellAgent::SensorEvents::LoginSuccess.new(env_or_header_keys,
-                                                             tcell_data,
-                                                             user_id,
-                                                             password,
-                                                             user_valid)
-                )
-              end
-            end
-          end
+        if ![TCellAgent::Hooks::V1::Login::LOGIN_FAILURE,
+             TCellAgent::Hooks::V1::Login::LOGIN_SUCCESS].include?(status)
+          get_logger.error("Unkown login status: #{status}")
+        elsif status == TCellAgent::Hooks::V1::Login::LOGIN_FAILURE
+          login_policy.report_login_failure(
+            user_id,
+            password,
+            env_or_header_keys,
+            user_valid,
+            tcell_data
+          )
+        elsif status == TCellAgent::Hooks::V1::Login::LOGIN_SUCCESS
+          login_policy.report_login_success(
+            user_id,
+            env_or_header_keys,
+            tcell_data
+          )
         end
       end
     end
@@ -91,9 +88,9 @@ if defined?(TCellAgent::Hooks::V1::Login)
           tcell_data = TCellAgent::Instrumentation::TCellData.new
           tcell_data.user_agent = user_agent
           tcell_data.referrer = referrer
-          tcell_data.ip_address = remote_address
+          tcell_data.remote_address = remote_address
           tcell_data.path = document_uri
-          tcell_data.hmac_session_id = TCellAgent::SensorEvents::Util.hmac(session_id)
+          tcell_data.session_id = session_id
 
           TCellAgent::Hooks::LoginFraud.report_login_event(status,
                                                            header_keys,

data/lib/tcell_agent/instrument_servers.rb

@@ -0,0 +1,25 @@
+tcell_server = ENV['TCELL_AGENT_SERVER']
+
+if TCellAgent.configuration.should_instrument?
+  unless tcell_server && tcell_server == 'mock'
+    if (tcell_server && tcell_server == 'webrick') || defined?(Rails::Server)
+      require('tcell_agent/servers/rails_server')
+
+    elsif (tcell_server && tcell_server == 'thin') || defined?(Thin)
+      require('tcell_agent/servers/thin')
+
+    elsif (tcell_server && tcell_server == 'puma') || defined?(Puma)
+      require('tcell_agent/servers/puma')
+
+    elsif (tcell_server && tcell_server == 'unicorn') || defined?(Unicorn)
+      require('tcell_agent/servers/unicorn')
+
+    elsif (tcell_server && tcell_server == 'passenger') || defined?(PhusionPassenger)
+      require('tcell_agent/servers/passenger')
+    end
+  end
+
+elsif (tcell_server && tcell_server == 'unicorn') || defined?(Unicorn)
+  # unicorn is always instrumented to support rolling restarts
+  require('tcell_agent/servers/unicorn')
+end