tcell_agent 0.2.29.rc2 → 0.2.29

data/lib/tcell_agent/configuration.rb

@@ -5,6 +5,8 @@ require 'yaml'
 require 'socket'
 require 'securerandom'
 
+require 'tcell_agent/config/unknown_options'
+
 module TCellAgent
   class ConfigurationException < StandardError
   end
@@ -23,13 +25,11 @@ module TCellAgent
       :tcell_api_url, :tcell_input_url,
       :logging_options,
       :logger,
-      :appfirewall_payloads_logger,
+      :appfirewall_payloads_logger, # appfirewall_payloads_logger can be specified from initializers
       :fetch_policies_from_tcell, :instrument_for_events,
       :preload_policy_filename,
-      :proxy_host, :proxy_port, :proxy_username, :proxy_password,
-      :use_websockets, :host_identifier, :session_cookie_names,
+      :host_identifier,
       :uuid,
-      :company,
       :event_batch_size_limit, :event_time_limit_seconds,
       :base_dir,
       :cache_filename,
@@ -41,13 +41,14 @@ module TCellAgent
       :config_filename,
       :agent_log_dir,
       :max_data_ex_db_records_per_request,
-      :allow_unencrypted_appfirewall_payloads_logging,
       :agent_home_dir,
       :agent_home_owner,
       :reverse_proxy,
       :reverse_proxy_ip_address_header,
       :log_file_name,
-      :log_tag
+      :log_tag,
+      :max_csp_header_bytes,
+      :demomode
 
     attr_accessor :disable_all,
       :enabled,
@@ -149,6 +150,9 @@ module TCellAgent
       @max_data_ex_db_records_per_request = 1000
       @reverse_proxy = true
       @reverse_proxy_ip_address_header = nil
+      @allow_unencrypted_appfirewall_payloads = false
+
+      @max_csp_header_bytes = nil
 
       read_config_using_env
       read_config_from_file(@config_filename)
@@ -157,8 +161,6 @@ module TCellAgent
         puts "tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS is deprecated, please switch to TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS."
       end
 
-      @allow_unencrypted_appfirewall_payloads = false
-
       if (ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"] != nil)
         @allow_unencrypted_appfirewall_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"])
       end
@@ -166,8 +168,6 @@ module TCellAgent
         @allow_unencrypted_appfirewall_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"])
       end
 
-      @allow_unencrypted_appfirewall_payloads_logging = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS_LOGGING"])
-
       @tcell_api_url ||= "https://api.tcell.io/api/v1"
       @tcell_input_url ||= "https://input.tcell.io/api/v1"
       @js_agent_api_base_url ||= nil
@@ -223,6 +223,12 @@ module TCellAgent
         begin
           config_text = File.open(filename).read
           config = JSON.parse(config_text)
+
+          messages = TCellAgent::Config::Validate.get_unknown_options(config)
+          messages.each do |message|
+            puts message
+          end
+
           if (config["version"] == 1)
             # Required
             app_data = config["applications"][0] #Default
@@ -257,12 +263,7 @@ module TCellAgent
             @tcell_api_url = app_data.fetch("tcell_api_url", @tcell_api_url)
             @tcell_input_url = app_data.fetch("tcell_input_url", @tcell_input_url)
 
-            @proxy_host = app_data["proxy_host"]
-            @proxy_port = app_data["proxy_port"]
-            @proxy_username = app_data["proxy_username"]
-            @proxy_password = app_data["proxy_password"]
-
-            @use_websockets = app_data["use_websockets"]
+            @max_csp_header_bytes = app_data.fetch("max_csp_header_bytes", @max_csp_header_bytes)
 
             @allow_unencrypted_appfirewall_payloads =
               app_data.fetch('allow_unencrypted_appsensor_payloads', @allow_unencrypted_appfirewall_payloads)
@@ -279,7 +280,6 @@ module TCellAgent
 
             @host_identifier = @host_identifier || app_data.fetch("host_identifier", @host_identifier)
             @hmac_key ||= app_data["hmac_key"] # if not already set
-            @session_cookie_names = app_data["session_cookie_names"]
             @uuid = SecureRandom.uuid
             if (@uuid == nil)
               @uuid = "secure-random-failed"
@@ -292,9 +292,6 @@ module TCellAgent
               @js_agent_url = app_data["js_agent_url"]
             end
 
-            # Causes old event url to be used
-            @company = app_data["company"]
-
             if @demomode != true
               @demomode = app_data.fetch('demomode', false)
             end

data/lib/tcell_agent/instrumentation.rb

@@ -1,6 +1,5 @@
 # encoding: utf-8
 # See the file "LICENSE" for the full license governing this code.
-require 'rest-client'
 require 'tcell_agent/logger'
 require 'tcell_agent/configuration'
 require 'tcell_agent/version'

data/lib/tcell_agent/logger.rb

@@ -89,27 +89,23 @@ module TCellAgent
       return @payloads_logger
     end
 
-    if TCellAgent.configuration.allow_unencrypted_appfirewall_payloads_logging
-      TCellAgent::Utils::IO.create_directory(
-        File.dirname(TCellAgent.configuration.appfirewall_payloads_log_filename),
-        TCellAgent.configuration.agent_home_owner.to_s
-      )
-
-      log_device = TCellLogDevice.new(
-        TCellAgent.configuration.appfirewall_payloads_log_filename,
-        shift_age: 9, shift_size: 5242880
-      )
-      @payloads_logger = Logger.new(log_device)
-      @payloads_logger.level = Logger::INFO
-      @payloads_logger.formatter = proc do |severity, datetime, progname, msg|
-        date_format = datetime.strftime("%Y-%m-%dT%H:%M:%S.%L%:z")
-         "#{date_format} - #{msg}\n"
-      end
-
-      return @payloads_logger
-    else
-      @null_logger
-    end
+    TCellAgent::Utils::IO.create_directory(
+      File.dirname(TCellAgent.configuration.appfirewall_payloads_log_filename),
+      TCellAgent.configuration.agent_home_owner.to_s
+    )
+
+    log_device = TCellLogDevice.new(
+      TCellAgent.configuration.appfirewall_payloads_log_filename,
+      shift_age: 9, shift_size: 5242880
+    )
+    @payloads_logger = Logger.new(log_device)
+    @payloads_logger.level = Logger::INFO
+    @payloads_logger.formatter = proc do |severity, datetime, progname, msg|
+      date_format = datetime.strftime("%Y-%m-%dT%H:%M:%S.%L%:z")
+       "#{date_format} - #{msg}\n"
+    end
+
+    return @payloads_logger
   end
 
   def self.logger

data/lib/tcell_agent/patches/block_rule.rb

@@ -8,13 +8,15 @@ module TCellAgent
         "block_403s" => 403
       }
 
-      attr_accessor :ips, :rids, :sensors_matcher, :action
+      attr_accessor :ips, :rids, :sensors_matcher, :action, :exact_blocked_paths, :starts_with_blocked_paths
 
-      def initialize(ips, rids, sensors_matcher, action)
+      def initialize(ips, rids, sensors_matcher, action, exact_blocked_paths, starts_with_blocked_paths)
         @ips = ips
         @rids = rids
         @sensors_matcher = sensors_matcher
         @action = action
+        @exact_blocked_paths = exact_blocked_paths
+        @starts_with_blocked_paths = starts_with_blocked_paths
       end
 
       def resp
@@ -22,11 +24,23 @@ module TCellAgent
       end
 
       def block?(meta_data)
-        return false unless @ips.empty? || @ips.include?(meta_data.remote_address)
+        if @exact_blocked_paths.size > 0 || @starts_with_blocked_paths.size > 0
+          if meta_data.path
+            return true if @exact_blocked_paths.include?(meta_data.path)
 
-        return false unless @rids.empty? || @rids.include?(meta_data.route_id)
+            return true if @starts_with_blocked_paths.any? do |blocked_path|
+              meta_data.path.start_with?(blocked_path)
+            end
+          end
+
+          return false
+        else
+          return false unless @ips.empty? || @ips.include?(meta_data.remote_address)
+
+          return false unless @rids.empty? || @rids.include?(meta_data.route_id)
 
-        return @sensors_matcher.any_matches?(meta_data)
+          return @sensors_matcher.any_matches?(meta_data)
+        end
       end
 
       def self.from_json(rule_json)
@@ -36,15 +50,36 @@ module TCellAgent
           ips = Set.new(rule_json.fetch("ips", []))
           rids = Set.new(rule_json.fetch("rids", []))
 
-          if ips.empty? && rids.empty?
-            TCellAgent.logger.error("Patches Policy block rule cannot be global. Specify either ips and/or route ids")
+          exact_blocked_paths = Set.new
+          starts_with_blocked_paths = []
+          rule_json.fetch("paths", []).each do |path_predicate|
+            if path_predicate.fetch("exact", nil)
+              exact_path = TCellAgent::Utils::Strings.remove_trailing_slash(path_predicate["exact"])
+              exact_blocked_paths.add(exact_path)
+              if exact_path.size > 1
+                exact_blocked_paths.add(exact_path + "/")
+              end
+
+            elsif path_predicate.fetch("starts_with", nil)
+              starts_with_blocked_paths.push(path_predicate["starts_with"])
+            end
+          end
+
+          if ips.empty? && rids.empty? && exact_blocked_paths.size == 0 && starts_with_blocked_paths.size == 0
+            TCellAgent.logger.error("Patches Policy block rule cannot be global. Specify either ips and/or route ids or blocked paths")
 
             return nil
           end
 
           sensors_matcher = SensorsMatcher.from_json(rule_json.fetch("sensor_matches", {}))
 
-          return BlockRule.new(ips, rids, sensors_matcher, action)
+          return BlockRule.new(
+            ips,
+            rids,
+            sensors_matcher,
+            action,
+            exact_blocked_paths,
+            starts_with_blocked_paths)
 
         else
           TCellAgent.logger.error("Patches Policy action not supported: #{action}")

data/lib/tcell_agent/patches/meta_data.rb

@@ -16,6 +16,7 @@ module TCellAgent
 
           meta_event.remote_address = TCellAgent::Utils::Rails.better_ip(request)
           meta_event.method = request.request_method
+          meta_event.path = request.path
           meta_event.user_agent = request.env['HTTP_USER_AGENT']
           meta_event.get_dict = request.GET
           meta_event.cookie_dict = request.cookies
@@ -45,7 +46,7 @@ module TCellAgent
         end
       end
 
-      attr_accessor :remote_address, :method, :location, :route_id, :session_id, :user_id, :transaction_id,
+      attr_accessor :remote_address, :method, :path, :route_id, :session_id, :user_id, :transaction_id,
         :request_content_bytes_len, :user_agent
 
       def initialize

data/lib/tcell_agent/patches/sensors_matcher.rb

@@ -21,7 +21,8 @@ module TCellAgent
       end
 
       def self.from_json(sensor_matcher_json)
-        injections_matcher = TCellAgent::AppSensor::InjectionsMatcher.from_json(2, sensor_matcher_json)
+        injections_matcher =
+          TCellAgent::AppSensor::InjectionsMatcher.from_json(2, sensor_matcher_json)
         SensorsMatcher.new(injections_matcher)
       end
     end

data/lib/tcell_agent/policies/appsensor/database_sensor.rb

@@ -7,17 +7,19 @@ module TCellAgent
 
       DP_CODE="dbmaxrows"
 
-      attr_accessor :enabled, :max_rows, :excluded_route_ids
+      attr_accessor :enabled, :max_rows, :excluded_route_ids, :collect_full_uri
 
       def initialize(policy_json=nil)
         @enabled = false
         @max_rows = 1001
         @excluded_route_ids = {}
+        @collect_full_uri = false
 
         if policy_json
           @enabled = policy_json.fetch("enabled", false)
           large_result = policy_json.fetch("large_result", {})
           @max_rows = large_result.fetch("limit", @max_rows)
+          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
 
           policy_json.fetch("exclude_routes", []).each do |excluded_route|
             @excluded_route_ids[excluded_route] = true
@@ -38,7 +40,8 @@ module TCellAgent
             tcell_data,
             DP_CODE,
             param,
-            meta)
+            meta,
+            @collect_full_uri)
         end
       end
 

data/lib/tcell_agent/policies/appsensor/misc_sensor.rb

@@ -5,18 +5,21 @@ module TCellAgent
 
     class MiscSensor
 
-      attr_accessor :enabled, :csrf_exception_enabled, :sql_exception_enabled, :excluded_route_ids
+      attr_accessor :enabled, :csrf_exception_enabled, :sql_exception_enabled,
+                    :excluded_route_ids, :collect_full_uri
 
       def initialize(policy_json=nil)
         @enabled = false
         @csrf_exception_enabled = false
         @sql_exception_enabled = false
         @excluded_route_ids = {}
+        @collect_full_uri = false
 
         if policy_json
           @enabled = policy_json.fetch("enabled", false)
           @csrf_exception_enabled = policy_json.fetch("csrf_exception_enabled", false)
           @sql_exception_enabled = policy_json.fetch("sql_exception_enabled", false)
+          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
 
           policy_json.fetch("exclude_routes", []).each do |excluded_route|
             @excluded_route_ids[excluded_route] = true
@@ -30,7 +33,9 @@ module TCellAgent
         return if tcell_data && @excluded_route_ids.fetch(tcell_data.route_id, false)
 
         meta = nil
-        TCellAgent::AppSensor::Sensor.send_event_from_tcell_data(tcell_data, "excsrf", exception_class.name, meta)
+        TCellAgent::AppSensor::Sensor.send_event_from_tcell_data(
+          tcell_data, "excsrf", exception_class.name, meta, @collect_full_uri
+        )
       end
 
       def sql_exception_detected(tcell_data, exception)
@@ -39,7 +44,9 @@ module TCellAgent
         return if tcell_data && @excluded_route_ids.fetch(tcell_data.route_id, false)
 
         meta = nil
-        TCellAgent::AppSensor::Sensor.send_event_from_tcell_data(tcell_data, "exsql", exception.class.name, meta)
+        TCellAgent::AppSensor::Sensor.send_event_from_tcell_data(
+          tcell_data, "exsql", exception.class.name, meta, @collect_full_uri
+        )
       end
 
       def to_s

data/lib/tcell_agent/policies/appsensor/payloads_policy.rb

@@ -14,11 +14,13 @@ module TCellAgent
       }
 
       attr_accessor :send_payloads, :send_blacklist, :send_whitelist, :use_send_whitelist,
-        :log_payloads, :log_blacklist, :log_whitelist, :use_log_whitelist
+                    :log_payloads, :log_blacklist, :log_whitelist, :use_log_whitelist,
+                    :collect_full_uri
 
       def initialize
         @send_payloads = false
         @log_payloads = false
+        @collect_full_uri = false
 
         @send_blacklist = {}
         @log_blacklist = {}
@@ -66,7 +68,7 @@ module TCellAgent
       end
 
       def log(dp, appsensor_meta, type_of_param, vuln_param, vuln_value, meta, pattern)
-        if @log_payloads && TCellAgent.configuration.allow_unencrypted_appfirewall_payloads_logging
+        if @log_payloads
           blacklisted_locations = @log_blacklist[vuln_param.downcase]
           param_location = PARAM_TYPE_MAP[type_of_param]
 
@@ -90,7 +92,8 @@ module TCellAgent
                 appsensor_meta.session_id,
                 appsensor_meta.user_id,
                 vuln_value,
-                pattern
+                pattern,
+                @collect_full_uri
               )
               event.post_process
               TCellAgent.appfirewall_payloads_logger.info(JSON.dump(event))
@@ -103,6 +106,8 @@ module TCellAgent
         policy = PayloadsPolicy.new
 
         if policy_json
+          policy.collect_full_uri = policy_json.fetch("uri_options", {}).fetch("collect_full_uri", false)
+
           payloads_json = policy_json.fetch("payloads", {})
           policy.send_payloads = payloads_json.fetch("send_payloads", false)
           policy.log_payloads = payloads_json.fetch("log_payloads", false)

data/lib/tcell_agent/policies/appsensor/request_size_sensor.rb

@@ -13,7 +13,7 @@ module TCellAgent
           MAX_NORMAL_REQUEST_BYTES,
           DP_UNUSUAL_REQUEST_SIZE,
           policy_json
-      )
+        )
       end
 
       def get_content_length(appsensor_meta)

data/lib/tcell_agent/policies/appsensor/response_codes_sensor.rb

@@ -15,18 +15,21 @@ module TCellAgent
         5 => "s5xx"
       }
 
-      attr_accessor :enabled, :series_400_enabled, :series_500_enabled, :excluded_route_ids
+      attr_accessor :enabled, :series_400_enabled, :series_500_enabled, :excluded_route_ids,
+                    :collect_full_uri
 
       def initialize(policy_json=nil)
         @enabled = false
         @series_400_enabled = false
         @series_500_enabled = false
         @excluded_route_ids = {}
+        @collect_full_uri = false
 
         if policy_json
           @enabled = policy_json.fetch("enabled", false)
           @series_400_enabled = policy_json.fetch("series_400_enabled", false)
           @series_500_enabled = policy_json.fetch("series_500_enabled", false)
+          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
 
           policy_json.fetch("exclude_routes", []).each do |excluded_route|
             @excluded_route_ids[excluded_route] = true
@@ -53,7 +56,9 @@ module TCellAgent
         if dp
           param = payload = pattern = nil
           meta = { code: response_code }
-          TCellAgent::AppSensor::Sensor.send_event(appsensor_meta, dp, param, meta, payload, pattern)
+          TCellAgent::AppSensor::Sensor.send_event(
+            appsensor_meta, dp, param, meta, payload, pattern, @collect_full_uri
+          )
         end
 
         def to_s

data/lib/tcell_agent/policies/appsensor/size_sensor.rb

@@ -5,17 +5,19 @@ module TCellAgent
 
     class SizeSensor
 
-      attr_accessor :enabled, :limit, :excluded_route_ids, :dp_code
+      attr_accessor :enabled, :limit, :excluded_route_ids, :dp_code, :collect_full_uri
 
       def initialize(default_limit, dp_code, policy_json)
         @enabled = false
         @limit = default_limit
         @excluded_route_ids = {}
         @dp_code = dp_code
+        @collect_full_uri = false
 
-        if policy_json != nil
+        if policy_json
           @enabled = policy_json.fetch("enabled", false)
           @limit = policy_json.fetch("limit", @limit)
+          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
 
           policy_json.fetch("exclude_routes", []).each do |route_id|
             @excluded_route_ids[route_id] = true
@@ -44,7 +46,9 @@ module TCellAgent
             param,
             meta,
             payload,
-            pattern)
+            pattern,
+            @collect_full_uri
+          )
         end
       end