tcell_agent 2.0.0 → 2.5.0

data/lib/tcell_agent/sinatra.rb

@@ -16,6 +16,7 @@ module TCellAgent
       TCellAgent::Instrumentation.safe_block('Setting Headers') do
         headers_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HEADERS)
         policy_headers = headers_policy.get_headers(
+          headers['Content-Type'],
           request.env[TCellAgent::Instrumentation::TCELL_ID]
         )
         policy_headers.each do |header_info|

data/lib/tcell_agent/tcell_context.rb

@@ -40,10 +40,11 @@ module TCellAgent
           tcell_context.request_method,
           tcell_context.remote_address,
           tcell_context.route_id,
-          tcell_context.hmac_session_id,
+          tcell_context.session_id,
           tcell_context.user_id,
           tcell_context.transaction_id,
-          tcell_context.uri
+          tcell_context.uri,
+          tcell_context.reverse_proxy_header_value
         )
         meta_data.path = tcell_context.path
 
@@ -70,7 +71,8 @@ module TCellAgent
                   :response_headers,
                   :csrf_exception_name,
                   :sql_exceptions,
-                  :database_result_sizes
+                  :database_result_sizes,
+                  :reverse_proxy_header_value
 
     attr_reader :flattened_get_dict,
                 :flattened_cookie_dict,
@@ -85,7 +87,8 @@ module TCellAgent
                    session_id,
                    user_id,
                    transaction_id,
-                   location)
+                   location,
+                   reverse_proxy_header_value)
       @method = method
       @remote_address = remote_address
       @route_id = route_id
@@ -93,7 +96,7 @@ module TCellAgent
       @user_id = user_id
       @transaction_id = transaction_id
       @location = location
-      @path = path
+      @reverse_proxy_header_value = reverse_proxy_header_value
 
       @flattened_get_dict = {}
       @flattened_cookie_dict = {}
@@ -148,6 +151,7 @@ module TCellAgent
       if !content_length.nil? && content_length > TCELL_MAX_BODY_LENGTH || request.content_type.nil?
         return nil
       end
+
       raw_post_data = nil
       # Positions strio to the beginning of input, resetting lineno to zero.
       # rails 4.1 seems to read the stringIO directly and so body.gets is empty
@@ -171,10 +175,15 @@ module TCellAgent
       self.get_dict = request.GET
       self.cookie_dict = request.cookies
 
-      self.post_dict = if @content_type.start_with?('application/json', 'application/xml')
+      self.post_dict = if @content_type.start_with?('application/json', 'application/xml') ||
+                          (@content_type.start_with?('multipart/form-data') && @request_content_bytes_len == 0)
                          {}
                        else
-                         request.POST
+                         begin
+                           request.POST
+                         rescue EOFError
+                           {}
+                         end
                        end
 
       self.headers_dict = request.env

data/lib/tcell_agent/utils/headers.rb

@@ -1,4 +1,3 @@
-
 module TCellAgent
   module Utils
     module Headers

data/lib/tcell_agent/utils/strings.rb

@@ -1,10 +1,10 @@
 module TCellAgent
   module Utils
     module Strings
-      BLANK_RE = /\A[[:space:]]*\z/
+      BLANK_RE = /\A[[:space:]]*\z/.freeze
 
       def self.blank?(str)
-        str.nil? || str.empty? || BLANK_RE === str # rubocop:disable Style/CaseEquality
+        str.nil? || str.empty? || BLANK_RE === str
       end
 
       def self.present?(str)

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = '2.0.0'.freeze
+  VERSION = '2.5.0'.freeze
 end

data/lib/tcell_agent.rb

@@ -1,26 +1,18 @@
 # See the file "LICENSE" for the full license governing this code.
-
-require 'tcell_agent/logger'
-require 'tcell_agent/utils/strings'
 require 'tcell_agent/configuration'
 
-require 'tcell_agent/agent'
-
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
-
-require 'tcell_agent/instrumentation'
+unless TCellAgent.configuration.disable_all
+  require 'tcell_agent/logger'
+  require 'tcell_agent/utils/strings'
+  require 'tcell_agent/agent'
 
-require 'tcell_agent/instrument_servers'
+  require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 
-if !TCellAgent.configuration.disable_all && TCellAgent.configuration.should_instrument?
-  require 'tcell_agent/instrumentation/cmdi'
-  require 'tcell_agent/instrumentation/lfi'
-  require 'tcell_agent/instrumentation/monkey_patches/io'
-  require 'tcell_agent/instrumentation/monkey_patches/file'
-  require 'tcell_agent/instrumentation/monkey_patches/kernel'
+  require 'tcell_agent/instrumentation'
 
+  require 'tcell_agent/instrument_servers'
   require 'tcell_agent/hooks/login_fraud'
-  require 'tcell_agent/rails/on_start' if defined?(Rails)
+  require 'tcell_agent/rails/railties/tcell_agent_railties' if defined?(Rails)
   # sinatra used to be supported, but dropped support due to no customers using it
   # require 'tcell_agent/sinatra' if defined?(Sinatra)
 end

data/spec/cruby_spec_helper.rb

@@ -0,0 +1,26 @@
+require 'tcell_agent'
+
+TCellAgent.configuration.enabled = true
+TCellAgent.configuration.instrument = true
+TCellAgent.configuration.enable_intercept_requests = true
+TCellAgent.configuration.disabled_instrumentation = []
+TCellAgent.thread_agent.instrument_built_ins
+
+# This monkey patch turns off blocking for LFI/CMDI when run in CRuby specs/
+# test suite. The original method also depends on a Rails installation,
+# which CRuby does not have installed.
+module TCellAgent
+  module Instrumentation
+    module Lfi
+      def self.block_file_access?(_path, _mode)
+        false
+      end
+    end
+  end
+
+  module Cmdi
+    def self.block_command?(_cmd)
+      false
+    end
+  end
+end

data/spec/lib/tcell_agent/configuration_spec.rb

@@ -2,228 +2,78 @@ require 'spec_helper'
 
 module TCellAgent
   describe Configuration do
-    describe '#agent_home_dir' do
-      context 'no TCELL_AGENT_HOME defined' do
-        it 'should set cache file, config, and log file to defaults' do
-          configuration = Configuration.new
+    describe 'should_instrument?' do
+      context 'with the agent disabled' do
+        it 'should return false' do
+          config = Configuration.new
+          config.enabled = false
 
-          expect(configuration.log_filename).to eq(
-            File.join(Dir.getwd, 'tcell/logs/tcell_agent.log')
-          )
-          expect(configuration.config_filename).to eq(
-            File.join(Dir.getwd, 'config/tcell_agent.config')
-          )
+          expect(config.should_instrument?).to be_falsey
         end
       end
-
-      context 'TCELL_AGENT_HOME defined' do
-        it 'should set config filename to default, cache file and log file are updated' do
-          old_tcell_agent_home = ENV['TCELL_AGENT_HOME']
-
-          ENV['TCELL_AGENT_HOME'] = 'spec_tcell_home'
-
-          configuration = Configuration.new
-
-          expect(configuration.log_filename).to eq(
-            'spec_tcell_home/logs/tcell_agent.log'
-          )
-          expect(configuration.config_filename).to eq(
-            File.join(Dir.getwd, 'config/tcell_agent.config')
-          )
-
-          ENV['TCELL_AGENT_HOME'] = old_tcell_agent_home
-        end
-      end
-
-      context 'TCELL_AGENT_HOME and TCELL_AGENT_LOG_DIR defined' do
-        it 'should set config filename to default, cache file and log file are updated' do
-          old_tcell_agent_home = ENV['TCELL_AGENT_HOME']
-          old_tcell_agent_log_dir = ENV['TCELL_AGENT_LOG_DIR']
-
-          ENV['TCELL_AGENT_HOME'] = 'spec_tcell_home'
-          ENV['TCELL_AGENT_LOG_DIR'] = 'spec_tcell_log_dir'
-
-          configuration = Configuration.new
-
-          expect(configuration.log_filename).to eq(
-            'spec_tcell_log_dir/tcell_agent.log'
-          )
-          expect(configuration.config_filename).to eq(
-            File.join(Dir.getwd, 'config/tcell_agent.config')
-          )
-
-          ENV['TCELL_AGENT_HOME'] = old_tcell_agent_home
-          ENV['TCELL_AGENT_LOG_DIR'] = old_tcell_agent_log_dir
-        end
-      end
-
-      context 'TCELL_AGENT_HOME, TCELL_AGENT_LOG_DIR, and TCELL_AGENT_CONFIG defined ' do
-        it 'should update config filename, cache file, and log file' do
-          old_tcell_agent_home = ENV['TCELL_AGENT_HOME']
-          old_tcell_agent_log_dir = ENV['TCELL_AGENT_LOG_DIR']
-          old_config_filename = ENV['TCELL_AGENT_CONFIG']
-
-          ENV['TCELL_AGENT_HOME'] = 'spec_tcell_home'
-          ENV['TCELL_AGENT_LOG_DIR'] = 'spec_tcell_log_dir'
-          ENV['TCELL_AGENT_CONFIG'] = 'spec_config/tcell_agent.config'
-
-          configuration = Configuration.new
-
-          expect(configuration.log_filename).to eq(
-            'spec_tcell_log_dir/tcell_agent.log'
-          )
-          expect(configuration.config_filename).to eq(
-            'spec_config/tcell_agent.config'
-          )
-
-          ENV['TCELL_AGENT_HOME'] = old_tcell_agent_home
-          ENV['TCELL_AGENT_LOG_DIR'] = old_tcell_agent_log_dir
-          ENV['TCELL_AGENT_CONFIG'] = old_config_filename
-        end
-      end
-    end
-
-    describe '#data_exposure' do
-      context 'no data_exposure defined' do
-        it 'should set max_data_ex_db_records_per_request to default' do
-          no_data_ex = double(
-            'no_data_ex',
-            :read => {
-              :version => 1,
-              :applications => [
-                :app_id => 'app_id',
-                :name => 'test',
-                :api_key => 'api_key'
-              ]
-            }.to_json
-          )
-          expect(File).to receive(:file?).with(
-            File.join(Dir.getwd, 'no_data_ex.config')
-          ).and_return(true)
-          expect(File).to receive(:open).with(
-            File.join(Dir.getwd, 'no_data_ex.config')
-          ).and_return(no_data_ex)
-          configuration = Configuration.new('no_data_ex.config')
-
-          expect(configuration.max_data_ex_db_records_per_request).to eq(1000)
-        end
-      end
-
-      context 'data_exposure is empty' do
-        it 'should set max_data_ex_db_records_per_request to default' do
-          no_data_ex = double(
-            'no_data_ex',
-            :read => {
-              :version => 1,
-              :applications => [
-                :app_id => 'app_id',
-                :name => 'test',
-                :api_key => 'api_key',
-                :data_exposure => {}
-              ]
-            }.to_json
-          )
-          expect(File).to receive(:file?).with(
-            File.join(Dir.getwd, 'no_data_ex.config')
-          ).and_return(true)
-          expect(File).to receive(:open).with(
-            File.join(Dir.getwd, 'no_data_ex.config')
-          ).and_return(no_data_ex)
-          configuration = Configuration.new('no_data_ex.config')
-
-          expect(configuration.max_data_ex_db_records_per_request).to eq(1000)
+      context 'with the agent enabled' do
+        context 'with all instrumentation enabled' do
+          context 'with no parameters' do
+            it 'should return true' do
+              config = Configuration.new
+              config.enabled = true
+              config.instrument = true
+
+              expect(config.should_instrument?).to be_truthy
+            end
+          end
+          context 'with parameters' do
+            it 'should return true' do
+              config = Configuration.new
+              config.enabled = true
+              config.instrument = true
+              config.disabled_instrumentation = Set.new
+
+              expect(config.should_instrument?('devise')).to be_truthy
+            end
+          end
         end
-      end
+        context 'with auth frameworks disabled' do
+          it 'should return false' do
+            config = Configuration.new
+            config.disabled_instrumentation = Set.new(%w[authlogic devise doorkeeper])
 
-      context 'data_exposure contains an override' do
-        it 'should set max_data_ex_db_records_per_request to override' do
-          no_data_ex = double(
-            'no_data_ex',
-            :read => {
-              :version => 1,
-              :applications => [
-                :app_id => 'app_id',
-                :name => 'test',
-                :api_key => 'api_key',
-                :data_exposure => {
-                  :max_data_ex_db_records_per_request => 5000
-                }
-              ]
-            }.to_json
-          )
-          expect(File).to receive(:file?).with(
-            File.join(Dir.getwd, 'no_data_ex.config')
-          ).and_return(true)
-          expect(File).to receive(:open).with(
-            File.join(Dir.getwd, 'no_data_ex.config')
-          ).and_return(no_data_ex)
-          configuration = Configuration.new('no_data_ex.config')
-
-          expect(configuration.max_data_ex_db_records_per_request).to eq(5000)
+            expect(config.should_instrument?('devise')).to be_falsey
+          end
         end
       end
     end
-
-    describe '#allow_payloads' do
-      context 'setting it via config' do
-        context 'using allow_payloads' do
-          it 'should be false' do
-            allow_payloads_enabled = double(
-              'no_data_ex',
-              :read => {
-                :version => 1,
-                :applications => [
-                  :app_id => 'app_id',
-                  :api_key => 'api_key',
-                  :allow_payloads => false
-                ]
-              }.to_json
-            )
-            expect(File).to receive(:file?).with(
-              File.join(Dir.getwd, 'config/tcell_agent.config')
-            ).and_return(true)
-            expect(File).to receive(:open).with(
-              File.join(Dir.getwd, 'config/tcell_agent.config')
-            ).and_return(allow_payloads_enabled)
-
-            configuration = Configuration.new
-
-            expect(configuration.allow_payloads).to eq(false)
-          end
+    describe 'populate_configuration' do
+      context 'with a poor native_agent_config_response' do
+        it 'should not throw an error' do
+          native_agent_config_response = {}
+
+          config = Configuration.new
+          expect do
+            config.populate_configuration(native_agent_config_response)
+          end.not_to raise_error
         end
       end
-
-      context 'setting it via env var' do
-        context 'TCELL_AGENT_ALLOW_PAYLOADS overrides everything else' do
-          it 'should be false' do
-            old_tcell_agent_allow_payloads = ENV['TCELL_AGENT_ALLOW_PAYLOADS']
-
-            ENV['TCELL_AGENT_ALLOW_PAYLOADS'] = 'false'
-
-            allow_payloads_enabled = double(
-              'no_data_ex',
-              :read => {
-                :version => 1,
-                :applications => [
-                  :app_id => 'app_id',
-                  :api_key => 'api_key',
-                  :allow_payloads => true
-                ]
-              }.to_json
-            )
-            expect(File).to receive(:file?).with(
-              File.join(Dir.getwd, 'config/tcell_agent.config')
-            ).and_return(true)
-            expect(File).to receive(:open).with(
-              File.join(Dir.getwd, 'config/tcell_agent.config')
-            ).and_return(allow_payloads_enabled)
-
-            configuration = Configuration.new
-
-            ENV['TCELL_AGENT_ALLOW_PAYLOADS'] = old_tcell_agent_allow_payloads
-
-            expect(configuration.allow_payloads).to eq(false)
-          end
+      context 'with an elaborate native_agent_config_response' do
+        it 'should set all the correct configurations' do
+          native_agent_config_response = { 'enabled' => true,
+                                           'disabled_instrumentation' => %w[devise doorkeeper],
+                                           'update_policy' => 'true',
+                                           'applications' => { 'first' => { 'app_id' => 'app_id_placeholder',
+                                                                            'api_key' => 'api_key_paceholder',
+                                                                            'hmac_key' => 'hmac_key_placeholder',
+                                                                            'password_hmac_key' => 'password_hmac_key_placeholder',
+                                                                            'proxy_config' => { 'reverse_proxy' => true,
+                                                                                                'reverse_proxy_ip_address_header' => 'X-Forwarded-For' } } },
+                                           'endpoint_config' => { 'api_url' => 'https://us.agent.tcell.insight.rapid7.com/api/v1' },
+                                           'ruby_config' => { 'enable_policy_polling' => true } }
+
+          config = Configuration.new
+          config.populate_configuration(native_agent_config_response)
+
+          expect(config.disabled_instrumentation).to be_a(Set)
+          expect(config.disabled_instrumentation).to include('devise', 'doorkeeper')
+          expect(config.enable_intercept_requests).to be_truthy
         end
       end
     end

data/spec/lib/tcell_agent/instrument_servers_spec.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+def test_rails
+  expect(Rails::Server.instance_methods.include?(:tcell_build_app)).to be_truthy
+end
+
+def test_thin
+  expect(Thin::Server.instance_methods.include?(:original_start)).to be_truthy
+end
+
+def test_unicorn
+  expect(Unicorn::HttpServer::START_CTX[0]).to be_falsy
+  expect(Unicorn::HttpServer.instance_methods.include?(:tcell_init_worker_process)).to be_truthy
+  expect(Unicorn::HttpServer.instance_methods.include?(:tcell_load_config!)).to be_truthy
+end
+
+def test_passenger
+  expect(PhusionPassenger::LoaderSharedHelpers.instance_methods.include?(:tcell_before_handling_requests))
+end
+
+def test_puma
+  expect(Puma.cli_config.options[:preload_app]).to be_falsey
+  expect(Puma::Server.instance_methods.include?(:tcell_original_run)).to be_truthy
+end
+
+def test_server(filenames, funcs)
+  fork do
+    filenames.each do |file|
+      load file
+    end
+
+    load 'tcell_agent/instrument_servers.rb'
+
+    funcs.each do |func|
+      method(func).call
+    end
+  end
+end
+
+describe 'instrument_servers' do
+  context 'with single server dependency' do
+    context 'with webrick server' do
+      it 'should instrument Webrick' do
+        mocks = ['spec/support/server_mocks/rails_mock.rb']
+        tests = [:test_rails]
+        test_server(mocks, tests)
+      end
+    end
+
+    context 'with Thin server' do
+      it 'should instrument Thin' do
+        mocks = ['spec/support/server_mocks/thin_mock.rb']
+        tests = [:test_thin]
+        test_server(mocks, tests)
+      end
+    end
+
+    context 'with Puma server' do
+      it 'should instrument Puma' do
+        mocks = ['spec/support/server_mocks/puma_mock.rb']
+        tests = [:test_puma]
+        test_server(mocks, tests)
+      end
+    end
+
+    context 'with Unicorn server' do
+      it 'should instrument Unicorn' do
+        mocks = ['spec/support/server_mocks/unicorn_mock.rb']
+        tests = [:test_unicorn]
+        test_server(mocks, tests)
+      end
+    end
+
+    context 'with Passenger server' do
+      it 'should instrument Unicorn' do
+        mocks = ['spec/support/server_mocks/passenger_mock.rb']
+        tests = [:test_passenger]
+        test_server(mocks, tests)
+      end
+    end
+  end
+  context 'with multiple server dependencies' do
+    it 'should instrument all servers available' do
+      mocks = ['spec/support/server_mocks/rails_mock.rb',
+               'spec/support/server_mocks/thin_mock.rb',
+               'spec/support/server_mocks/puma_mock.rb',
+               'spec/support/server_mocks/unicorn_mock.rb',
+               'spec/support/server_mocks/passenger_mock.rb']
+
+      tests = %i[test_rails test_thin test_puma test_unicorn test_passenger]
+
+      test_server(mocks, tests)
+    end
+  end
+end

data/spec/lib/tcell_agent/instrumentation/cmdi/io_cmdi_spec.rb

@@ -262,10 +262,10 @@ describe IO do
           IO.popen(@env, %w[echo test], 'w+', :unsetenv_others => true)
 
           expect(TCellAgent::Cmdi).to receive(:block_command?).with('echo test')
-          IO.popen([@env, 'echo', 'test', :unsetenv_others => true], 'w+')
+          IO.popen([@env, 'echo', 'test', { :unsetenv_others => true }], 'w+')
 
           expect(TCellAgent::Cmdi).to receive(:block_command?).with('echo test')
-          IO.popen(@env, [@env, 'echo', 'test', :unsetenv_others => true], 'w+', :err => %i[child out])
+          IO.popen(@env, [@env, 'echo', 'test', { :unsetenv_others => true }], 'w+', :err => %i[child out])
         end
       end
     end

data/spec/lib/tcell_agent/{cmdi_spec.rb → instrumentation/cmdi_spec.rb}

@@ -147,5 +147,55 @@ module TCellAgent
         end
       end
     end
+    describe '.parse_command_from_open' do
+      context 'with empty parameters' do
+        it 'should not return a command' do
+          cmd = TCellAgent::Cmdi.parse_command_from_open
+          expect(cmd).to eq('')
+        end
+      end
+      context 'with empty array' do
+        it 'should not return a command' do
+          args = []
+          cmd = TCellAgent::Cmdi.parse_command_from_open(*args)
+          expect(cmd).to eq('')
+        end
+      end
+      context 'with a string command' do
+        context 'with an empty string' do
+          it 'should return an empty string' do
+            cmd = TCellAgent::Cmdi.parse_command_from_open('')
+            expect(cmd).to eq('')
+          end
+        end
+        context 'with an empty command' do
+          it 'should return an empty string' do
+            cmd = TCellAgent::Cmdi.parse_command_from_open('|')
+            expect(cmd).to eq('')
+          end
+        end
+        context 'with a non-empty command' do
+          it 'should parse the command properly' do
+            cmd = TCellAgent::Cmdi.parse_command_from_open('|echo')
+            expect(cmd).to eq('echo')
+
+            cmd = TCellAgent::Cmdi.parse_command_from_open('|ls -l /tmp')
+            expect(cmd).to eq('ls -l /tmp')
+          end
+        end
+      end
+      context 'with a filename argument' do
+        it 'should not return a command' do
+          cmd = TCellAgent::Cmdi.parse_command_from_open('/tmp')
+          expect(cmd).to eq('')
+        end
+      end
+      context 'with a non-string first argument' do
+        it 'should return an empty string' do
+          cmd = TCellAgent::Cmdi.parse_command_from_open({})
+          expect(cmd).to eq('')
+        end
+      end
+    end
   end
 end