tcell_agent 0.2.18 → 0.2.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c0a92b41c861284cfbb6514ec1b0b4d6ab40dae
-  data.tar.gz: bd2eccfd22ecc2fbdf6c803e1f89bf3642556c5a
+  metadata.gz: 1b4ab2290db14be2d14b65edc01f36145e73da02
+  data.tar.gz: 35f0e04cbc8b20703cc3f5a41b77875d859e9e0c
 SHA512:
-  metadata.gz: ac335190e83b476d4c8a3cf36e0b1b1111fd370f0690e202726c0cc42d9305c0be85324a5121c7191515ca16b0bf2ba5b11400a35d847a251a6ea58f9a2cf5d7
-  data.tar.gz: a46196f6c74e1c9b80fb7f7edfa75e6c78ddd02d48d588bbc611b8ee98fa755eee44b6d9368f1843ce546dd2ea775da22e97eb7b04dd1c3e020875caf818259e
+  metadata.gz: 41fa1edda89b1cefe0ec2d6501a465077a21960fdb4fcd776fd0dc99d71507fc59159503ce98640421cd5882640173a75665f66b10ba88f754a8d3630126729f
+  data.tar.gz: 194f427b4cb851e4434f27584e612c06aa0f16188ced64481fbf196ffc58c0d6e80e7c744cdc658f520fc6fb61320d8204150c19f94c0ff84f5321fee6dc64ea

data/Rakefile

@@ -5,3 +5,14 @@ RSpec::Core::RakeTask.new(:spec)
 desc "Run tests"
 task :default => :spec
 task :test => :spec
+
+task "init-integration-tests" do
+system("docker-compose run railsintegration224 bundle install")
+system("docker-compose run railsintegration224 bundle exec rake db:create db:setup")
+system("docker-compose stop")
+end
+
+task "integration-test" do
+  system("docker-compose up railsintegration224")
+  system("docker-compose stop")
+end

data/lib/tcell_agent/configuration.rb

@@ -40,7 +40,9 @@ module TCellAgent
       :max_data_ex_db_records_per_request,
       :allow_unencrypted_appfirewall_payloads_logging,
       :agent_home_dir,
-      :agent_home_owner
+      :agent_home_owner,
+      :reverse_proxy,
+      :reverse_proxy_ip_address_header
 
     attr_accessor :disable_all,
       :enabled,
@@ -104,6 +106,8 @@ module TCellAgent
       @raise_exceptions = false
 
       @max_data_ex_db_records_per_request = 1000
+      @reverse_proxy = true
+      @reverse_proxy_ip_address_header = nil
 
       read_config_using_env
       read_config_from_file(@config_filename)
@@ -223,6 +227,9 @@ module TCellAgent
             data_exposure = app_data.fetch('data_exposure', {})
             @max_data_ex_db_records_per_request = data_exposure.fetch('max_data_ex_db_records_per_request', @max_data_ex_db_records_per_request)
 
+            @reverse_proxy = app_data.fetch('reverse_proxy', @reverse_proxy)
+            @reverse_proxy_ip_address_header = app_data.fetch('reverse_proxy_ip_address_header', @reverse_proxy_ip_address_header)
+
             @host_identifier = @host_identifier || app_data.fetch("host_identifier", @host_identifier)
             @hmac_key ||= app_data["hmac_key"] # if not already set
             @session_cookie_names = app_data["session_cookie_names"]

data/lib/tcell_agent/instrumentation.rb

@@ -9,6 +9,8 @@ require 'cgi'
 
 module TCellAgent
   module Instrumentation
+    TCELL_ID = "tcell.request_data"
+
     class ContextFilter
       attr_accessor :type
       attr_accessor :rule
@@ -58,16 +60,10 @@ module TCellAgent
 
 
     class TCellData
-      attr_accessor :transaction_id
-      attr_accessor :session_id
-      attr_accessor :hmac_session_id
-      attr_accessor :user_id
-      attr_accessor :route_id
-      attr_accessor :uri
-      attr_accessor :context_filters_by_term
-      attr_accessor :database_filters
-      attr_accessor :ip_address
-      attr_accessor :user_agent
+      attr_accessor :transaction_id, :session_id, :hmac_session_id, :user_id, :route_id,
+        :uri, :context_filters_by_term, :database_filters, :ip_address, :user_agent, :request_method,
+        :path_parameters
+
       def self.filterx(sanitize_string, event_flag, replace_flag, term)
         send_event = false
         sanitize_string.gsub!(term) {|m|
@@ -204,6 +200,14 @@ module TCellAgent
         return log_msg
       end
 
+      def to_s
+        "<#{self.class.name} transaction_id: #{transaction_id} session_id: #{session_id} " +
+        "hmac_session_id: #{hmac_session_id} user_id: #{user_id} route_id: #{route_id} " +
+        "uri: #{uri} context_filters_by_term: #{context_filters_by_term} " +
+        "database_filters: #{database_filters} ip_address: #{ip_address} user_agent: #{user_agent} " +
+        "request_method: #{@request_method} path_parameters: #{@path_parameters}>"
+      end
+
     end
 
     def self.instrument_frameworks

data/lib/tcell_agent/logger.rb

@@ -17,7 +17,7 @@ module TCellAgent
     def create_logfile(filename)
       logdev = super
 
-      TCellAgent::Utils::IO.set_owner(filename, TCellAgent.configuration.agent_home_owner)
+      TCellAgent::Utils::IO.set_owner(filename, TCellAgent.configuration.agent_home_owner.to_s)
 
       logdev
     end
@@ -50,16 +50,16 @@ module TCellAgent
       return @payloads_logger
     end
 
-    TCellAgent::Utils::IO.create_directory(
-      File.dirname(TCellAgent.configuration.appfirewall_payloads_log_filename),
-      TCellAgent.configuration.agent_home_owner
-    )
-
-    log_device = TCellLogDevice.new(
-      TCellAgent.configuration.appfirewall_payloads_log_filename,
-      shift_age: 9, shift_size: 5242880
-    )
     if TCellAgent.configuration.allow_unencrypted_appfirewall_payloads_logging
+      TCellAgent::Utils::IO.create_directory(
+        File.dirname(TCellAgent.configuration.appfirewall_payloads_log_filename),
+        TCellAgent.configuration.agent_home_owner.to_s
+      )
+
+      log_device = TCellLogDevice.new(
+        TCellAgent.configuration.appfirewall_payloads_log_filename,
+        shift_age: 9, shift_size: 5242880
+      )
       @payloads_logger = Logger.new(log_device)
       @payloads_logger.level = Logger::INFO
       @payloads_logger.formatter = proc do |severity, datetime, progname, msg|
@@ -68,11 +68,9 @@ module TCellAgent
       end
 
       return @payloads_logger
+    else
+      @null_logger
     end
-
-    logger = Logger.new(log_device)
-    logger.level = Logger::ERROR
-    return logger
   end
 
   def self.logger
@@ -86,13 +84,14 @@ module TCellAgent
     @logger_pid = Process.pid
     logging_options = TCellAgent.configuration.logging_options
 
-    logging_file = TCellAgent.configuration.log_filename
-    logging_directory = File.dirname(logging_file)
-    TCellAgent::Utils::IO.create_directory(logging_directory, TCellAgent.configuration.agent_home_owner)
+    if logging_options && (logging_options[:enabled] || logging_options["enabled"])
+      logging_file = TCellAgent.configuration.log_filename
+      logging_directory = File.dirname(logging_file)
+      TCellAgent::Utils::IO.create_directory(logging_directory, TCellAgent.configuration.agent_home_owner.to_s)
 
-    log_device = TCellLogDevice.new(logging_file, shift_age: 9, shift_size: 5242880)
-    if logging_options && logging_options["enabled"]
-      level = loggingLevelFromString(logging_options["level"])
+      log_device = TCellLogDevice.new(logging_file, shift_age: 9, shift_size: 5242880)
+
+      level = loggingLevelFromString(logging_options[:level] || logging_options["level"])
       # limit the total log file to about 9 * 5 = 45 mb
       @logger = Logger.new(log_device)
       @logger.level = level
@@ -103,11 +102,12 @@ module TCellAgent
       end
 
       return @logger
+
+    else
+      @null_logger
     end
 
-    logger = Logger.new(log_device)
-    logger.level = Logger::ERROR
-    return logger
+    @null_logger
   end
 
   def self.logger=(logger)

data/lib/tcell_agent/policies/appsensor/database_sensor.rb

@@ -0,0 +1,61 @@
+module TCellAgent
+  module Policies
+
+    class DatabaseSensor
+
+      DP_CODE="dbmaxrows"
+
+      attr_accessor :enabled, :max_rows, :excluded_route_ids
+
+      def initialize(policy_json=nil)
+        @enabled = false
+        @max_rows = 1001
+        @excluded_route_ids = {}
+
+        if policy_json
+          @enabled = policy_json.fetch("enabled", false)
+          large_result = policy_json.fetch("large_result", {})
+          @max_rows = large_result.fetch("limit", @max_rows)
+
+          policy_json.fetch("exclude_routes", []).each do |excluded_route|
+            @excluded_route_ids[excluded_route] = true
+          end
+        end
+      end
+
+
+      def check(tcell_data, number_of_records)
+        return unless @enabled
+
+        return if @excluded_route_ids.fetch(tcell_data.route_id, false)
+
+        send_event(tcell_data, number_of_records) if number_of_records > @max_rows
+      end
+
+      def send_event(tcell_data, number_of_records)
+        event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
+          tcell_data.uri,
+          DP_CODE,
+          tcell_data.request_method,
+          tcell_data.ip_address,
+          nil,
+          tcell_data.route_id,
+          {"rows" => number_of_records},
+          tcell_data.transaction_id,
+          tcell_data.session_id,
+          tcell_data.user_id,
+          nil
+        )
+
+        TCellAgent.send_event(event)
+      end
+
+      def to_s
+        "<#{self.class.name} enabled: #{@enabled} max_rows: #{@max_rows} " +
+        "excluded_route_ids: #{@excluded_route_ids}>"
+      end
+
+    end
+
+  end
+end

data/lib/tcell_agent/policies/appsensor/injection_sensor.rb

@@ -9,7 +9,7 @@ module TCellAgent
 
       attr_accessor :enabled, :detection_point, :exclude_headers, :exclude_forms,
         :exclude_cookies, :exclusions, :active_pattern_ids, :v1_compatability_enabled,
-        :rule_manager
+        :rule_manager, :excluded_route_ids
 
 
       def initialize(detection_point, policy_json=nil)
@@ -22,6 +22,7 @@ module TCellAgent
         @active_pattern_ids = {}
         @v1_compatability_enabled = false
         @rule_manager = AppSensorRuleManager.new
+        @excluded_route_ids = {}
 
         if policy_json
           @enabled = policy_json.fetch("enabled", false)
@@ -31,6 +32,10 @@ module TCellAgent
           @v1_compatability_enabled = policy_json.fetch("v1_compatability_enabled", false)
           @rule_manager = policy_json.fetch("rule_manager", AppSensorRuleManager.new)
 
+          policy_json.fetch("exclude_routes", []).each do |excluded_route|
+            @excluded_route_ids[excluded_route] = true
+          end
+
           policy_json.fetch("patterns", []).each do |pattern|
             @active_pattern_ids[pattern] = true
           end
@@ -57,6 +62,8 @@ module TCellAgent
       def check(type_of_param, appsensor_meta, param_name, param_value)
         return false unless @enabled
 
+        return false if @excluded_route_ids.fetch(appsensor_meta.route_id, false)
+
         if @exclude_forms && (GET_PARAM == type_of_param or POST_PARAM == type_of_param or JSON_PARAM == type_of_param)
           return false
         end
@@ -127,7 +134,8 @@ module TCellAgent
         "<#{self.class.name} enabled: #{@enabled} dp: #{@detection_point} " +
         "exclude_headers: #{@exclude_headers} exclude_forms: #{exclude_forms} " +
         "exclude_cookies: #{exclude_cookies} v1_compatability_enabled: #{@v1_compatability_enabled} " +
-        "active_pattern_ids: #{@active_pattern_ids} exclusions: #{exclusions}>"
+        "active_pattern_ids: #{@active_pattern_ids} exclusions: #{exclusions} " +
+        "excluded_route_ids: #{@excluded_route_ids}>"
       end
     end
 

data/lib/tcell_agent/policies/appsensor/misc_sensor.rb

@@ -0,0 +1,66 @@
+module TCellAgent
+  module Policies
+
+    class MiscSensor
+
+      attr_accessor :enabled, :csrf_exception_enabled, :sql_exception_enabled, :excluded_route_ids
+
+      def initialize(policy_json=nil)
+        @enabled = false
+        @csrf_exception_enabled = false
+        @sql_exception_enabled = false
+        @excluded_route_ids = {}
+
+        if policy_json
+          @enabled = policy_json.fetch("enabled", false)
+          @csrf_exception_enabled = policy_json.fetch("csrf_exception_enabled", false)
+          @sql_exception_enabled = policy_json.fetch("sql_exception_enabled", false)
+
+          policy_json.fetch("exclude_routes", []).each do |excluded_route|
+            @excluded_route_ids[excluded_route] = true
+          end
+        end
+      end
+
+      def csrf_rejected(tcell_data)
+        return unless @enabled && @csrf_exception_enabled
+
+        return if tcell_data && @excluded_route_ids.fetch(tcell_data.route_id, false)
+
+        send_event("excsrf", tcell_data)
+      end
+
+      def sql_exception_detected(tcell_data, exception)
+        return unless @enabled && @sql_exception_enabled
+
+        return if tcell_data && @excluded_route_ids.fetch(tcell_data.route_id, false)
+
+        send_event("exsql", tcell_data)
+      end
+
+      def send_event(detection_point, tcell_data)
+        event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
+          tcell_data.uri,
+          detection_point,
+          tcell_data.request_method,
+          tcell_data.ip_address,
+          nil,
+          tcell_data.route_id,
+          nil,
+          tcell_data.transaction_id,
+          tcell_data.session_id,
+          tcell_data.user_id,
+          nil
+        )
+
+        TCellAgent.send_event(event)
+      end
+
+      def to_s
+        "<#{self.class.name} enabled: #{@enabled} csrf_exception_enabled: #{@csrf_exception_enabled} " +
+        "sql_exception_enabled: #{sql_exception_enabled}>"
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/policies/appsensor/response_codes_sensor.rb

@@ -15,22 +15,29 @@ module TCellAgent
         5 => "s5xx"
       }
 
-      attr_accessor :enabled, :series_400_enabled, :series_500_enabled
+      attr_accessor :enabled, :series_400_enabled, :series_500_enabled, :excluded_route_ids
 
       def initialize(policy_json=nil)
         @enabled = false
         @series_400_enabled = false
         @series_500_enabled = false
+        @excluded_route_ids = {}
 
         if policy_json
           @enabled = policy_json.fetch("enabled", false)
           @series_400_enabled = policy_json.fetch("series_400_enabled", false)
           @series_500_enabled = policy_json.fetch("series_500_enabled", false)
+
+          policy_json.fetch("exclude_routes", []).each do |excluded_route|
+            @excluded_route_ids[excluded_route] = true
+          end
         end
       end
 
       def check(appsensor_meta, response_code)
-        return unless self.enabled
+        return unless @enabled
+
+        return if @excluded_route_ids.fetch(appsensor_meta.route_id, false)
 
         return if response_code == 200
         return if !self.series_400_enabled && (response_code >= 400 && response_code < 500)
@@ -48,7 +55,8 @@ module TCellAgent
         end
 
         def to_s
-          "<#{self.class.name} enabled: #{@enabled} series_400_enabled: #{@series_400_enabled} series_500_enabled: #{@series_500_enabled}>"
+          "<#{self.class.name} enabled: #{@enabled} series_400_enabled: #{@series_400_enabled} " +
+          "series_500_enabled: #{@series_500_enabled}>"
         end
       end
 

data/lib/tcell_agent/policies/appsensor/size_sensor.rb

@@ -5,12 +5,12 @@ module TCellAgent
 
     class SizeSensor < Sensor
 
-      attr_accessor :enabled, :limit, :exclude_routes, :dp_code
+      attr_accessor :enabled, :limit, :excluded_route_ids, :dp_code
 
       def initialize(default_limit, dp_code, policy_json)
         @enabled = false
         @limit = default_limit
-        @exclude_routes = {}
+        @excluded_route_ids = {}
         @dp_code = dp_code
 
         if policy_json != nil
@@ -18,13 +18,13 @@ module TCellAgent
           @limit = policy_json.fetch("limit", @limit)
 
           policy_json.fetch("exclude_routes", []).each do |route_id|
-            @exclude_routes[route_id] = true
+            @excluded_route_ids[route_id] = true
           end
         end
       end
 
       def check(appsensor_meta, content_length)
-        if !@enabled || @exclude_routes.fetch(appsensor_meta.route_id, false)
+        if !@enabled || @excluded_route_ids.fetch(appsensor_meta.route_id, false)
           return
         end
 
@@ -34,7 +34,8 @@ module TCellAgent
       end
 
       def to_s
-        "<#{self.class.name} enabled: #{@enabled} limit: #{@limit} dp_code: #{@dp_code} exclude_routes: #{@exclude_routes}>"
+        "<#{self.class.name} enabled: #{@enabled} limit: #{@limit} dp_code: #{@dp_code} " +
+        "excluded_route_ids: #{@excluded_route_ids}>"
       end
 
     end