tcell_agent 0.2.21 → 0.2.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94acb227285ba44cdff883596610b0a1acfa7a44
-  data.tar.gz: 3434e3c09598e19d4913c23fa7241fd28ae71316
+  metadata.gz: 0436d6cc489b525b048c92ac39ca316368c5fb90
+  data.tar.gz: 09b162b17a7d373de8eaeba59d192c30d2fbb1c2
 SHA512:
-  metadata.gz: 511965665332196c8d0c5715057251fcad8eeee5118d0e8a6626d14ef031d4c2861d2875e50aac1380c010cd9d110cff95632b19abe7362440ff91fff8b67c88
-  data.tar.gz: 7a551c6c4b09f6a1fc4c59c07b9275aadb3ad44c9de109b19b5d8ee5bd322fa6029d7f1f81f127d5c29fe81f91a05c758c84a31666051d776fa85fd069b2e97c
+  metadata.gz: b244fce3ca335f9a524e286fe214915656485209eb3be501abc5ecf891a31133b9c116b4e8b929123752b56343607918e96252f4b1b0da8b165c8170cc13c266
+  data.tar.gz: 16eebbdcf0a19dbb64c106fc8f2ff43521a95317d2da3bf2414c88f5de22ca5b6e09570f775073c0dcca4f6f7a0d622c6ce11d77778f3395b4dd89fdd9a9d0ea

data/lib/tcell_agent.rb

@@ -16,6 +16,7 @@ require 'tcell_agent/policies/secure_headers_policy'
 require 'tcell_agent/policies/honeytokens_policy'
 require 'tcell_agent/policies/clickjacking_policy'
 require 'tcell_agent/policies/appsensor_policy'
+require 'tcell_agent/policies/patches_policy'
 require 'tcell_agent/policies/login_fraud_policy'
 require 'tcell_agent/policies/dataloss_policy'
 

data/lib/tcell_agent/api.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 # See the file "LICENSE" for the full license governing this code.
+require 'json'
 require 'rest-client'
 require 'tcell_agent/logger'
 require 'tcell_agent/configuration'
@@ -55,7 +56,7 @@ module TCellAgent
       eventset = { "uuid"=>TCellAgent.configuration.uuid,
                    "hostname"=>TCellAgent.configuration.host_identifier,
                    "events"=>events }
-      TCellAgent.logger.debug("Sending #{eventset.to_json}")
+      TCellAgent.logger.debug("Sending #{JSON.dump(eventset)}")
       full_url = TCellAgent.configuration.tcell_input_url + "/app/" + TCellAgent.configuration.app_id + "/server_agent"
 
       TCellAgent.logger.debug("tCell.io SendEvents API Request: " + full_url)
@@ -69,7 +70,7 @@ module TCellAgent
       rescue Exception => e
         TCellAgent.logger.debug("tCell.io Could not add agent string: " + e.message)
       end
-      response = RestClient.post full_url, eventset.to_json, request_headers
+      response = RestClient.post full_url, JSON.dump(eventset), request_headers
       TCellAgent.logger.debug("tCell.io SendEvents API Response: " + response.code.to_s)
       return response.code == 200
     end

data/lib/tcell_agent/appsensor/injections_matcher.rb

@@ -0,0 +1,137 @@
+module TCellAgent
+  module AppSensor
+
+    class InjectionsMatcher
+      GET_PARAM = TCellAgent::Utils::Params::GET_PARAM
+      POST_PARAM = TCellAgent::Utils::Params::POST_PARAM
+      JSON_PARAM = TCellAgent::Utils::Params::JSON_PARAM
+      COOKIE_PARAM = TCellAgent::Utils::Params::COOKIE_PARAM
+      URI_PARAM = TCellAgent::Utils::Params::URI_PARAM
+
+      DETECTION_POINTS_V2 = {
+        "xss" => TCellAgent::Policies::XssSensor,
+        "sqli" => TCellAgent::Policies::SqliSensor,
+        "cmdi" => TCellAgent::Policies::CmdiSensor,
+        "fpt" => TCellAgent::Policies::FptSensor,
+        "nullbyte" => TCellAgent::Policies::NullbyteSensor,
+        "retr" => TCellAgent::Policies::RetrSensor
+      }
+
+      attr_accessor :enabled, :sensors
+
+      def initialize(sensors)
+        @sensors = sensors
+        @enabled = sensors.size > 0
+      end
+
+      def each_injection(meta_data)
+        return unless @enabled
+
+        meta_data.flattened_path_parameters.each do |param_name, param_value|
+          TCellAgent::Instrumentation.safe_block("AppSensor Check Path Params injections") do
+            param_name = param_name[-1]
+            injection_attempt =
+              check_param_for_injections(URI_PARAM, meta_data, param_name, param_value)
+
+            yield(injection_attempt) if injection_attempt
+          end
+        end
+
+        meta_data.flattened_get_dict.each do |param_name, param_value|
+          TCellAgent::Instrumentation.safe_block("AppSensor Check GET var injections") do
+            param_name = param_name[-1]
+            injection_attempt =
+              check_param_for_injections(GET_PARAM, meta_data, param_name, param_value)
+
+            yield(injection_attempt) if injection_attempt
+          end
+        end
+
+        meta_data.flattened_post_dict.each do |param_name, param_value|
+          TCellAgent::Instrumentation.safe_block("AppSensor Check POST var injections") do
+            param_name = param_name[-1]
+            injection_attempt =
+              check_param_for_injections(POST_PARAM, meta_data, param_name, param_value)
+
+            yield(injection_attempt) if injection_attempt
+          end
+        end
+
+        meta_data.flattened_body_dict.each do |param_name, param_value|
+          TCellAgent::Instrumentation.safe_block("AppSensor Check JSON var injections") do
+            param_name = param_name[-1]
+            injection_attempt = check_param_for_injections(JSON_PARAM, meta_data, param_name, param_value)
+
+            yield(injection_attempt) if injection_attempt
+          end
+        end
+
+        meta_data.flattened_cookie_dict.each do |param_name, param_value|
+          TCellAgent::Instrumentation.safe_block("AppSensor Check COOKIE var injections") do
+            param_name = param_name[-1]
+            injection_attempt =
+              check_param_for_injections(COOKIE_PARAM, meta_data, param_name, param_value)
+
+            yield(injection_attempt) if injection_attempt
+          end
+        end
+      end
+
+      def check_param_for_injections(param_type, appsensor_meta, param_name, param_value)
+        @sensors.each do |sensor|
+          next unless sensor.applicable_for_param_type?(param_type)
+
+          injection_attempt = sensor.get_injection_attempt(param_type, appsensor_meta, param_name, param_value)
+          return injection_attempt if injection_attempt
+        end
+
+        return nil
+      end
+
+      def self.from_json(version, sensors_json)
+        sensors_json = sensors_json || {}
+        sensors = []
+
+        if version == 1
+          options_json = sensors_json.fetch("options", {})
+
+          (options_json || {}).each do |sensor_key, enabled|
+            next unless enabled
+
+            if sensor_key == "null"
+              sensor_key = "nullbyte"
+            end
+
+            clazz = DETECTION_POINTS_V2[sensor_key]
+
+            next unless clazz
+
+            if enabled
+              sensors.push(clazz.new(
+                {
+                  "enabled" => enabled,
+                  "v1_compatability_enabled" => true
+                }
+              ))
+            end
+          end
+
+        elsif version == 2
+          sensors_json.each do |sensor_key, settings|
+            clazz = DETECTION_POINTS_V2[sensor_key]
+
+            next unless clazz
+
+            updated_settings = {"enabled" => true}.merge(settings)
+            if updated_settings["enabled"]
+              sensors.push(clazz.new(updated_settings))
+            end
+          end
+        end
+
+        InjectionsMatcher.new(sensors)
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/appsensor/injections_reporter.rb

@@ -0,0 +1,67 @@
+require 'tcell_agent/appsensor/sensor'
+require 'tcell_agent/utils/params'
+
+
+module TCellAgent
+  module AppSensor
+
+    class InjectionsReporter
+      GET_PARAM = TCellAgent::Utils::Params::GET_PARAM
+      POST_PARAM = TCellAgent::Utils::Params::POST_PARAM
+      JSON_PARAM = TCellAgent::Utils::Params::JSON_PARAM
+      COOKIE_PARAM = TCellAgent::Utils::Params::COOKIE_PARAM
+      URI_PARAM = TCellAgent::Utils::Params::URI_PARAM
+
+      PARAM_TYPE_TO_L = {
+        GET_PARAM => 'query',
+        POST_PARAM => 'body',
+        JSON_PARAM => 'body',
+        URI_PARAM => 'uri',
+        COOKIE_PARAM => 'cookie'
+      }
+
+      attr_accessor :injections_matcher, :payloads_policy
+
+      def initialize(injections_matcher, payloads_policy)
+        @injections_matcher = injections_matcher
+        @payloads_policy = payloads_policy
+      end
+
+      def check(appsensor_meta)
+        @injections_matcher.each_injection(appsensor_meta) do |injection_attempt|
+          vuln_param = injection_attempt.param_name
+          type_of_param = injection_attempt.type_of_param
+
+          meta = {"l" => PARAM_TYPE_TO_L[type_of_param]}
+          pattern = injection_attempt.pattern
+
+          payload = @payloads_policy.apply(
+            injection_attempt.detection_point,
+            appsensor_meta,
+            type_of_param,
+            vuln_param,
+            injection_attempt.param_value,
+            meta,
+            pattern
+          )
+
+          TCellAgent::AppSensor::Sensor.send_event(
+            appsensor_meta,
+            injection_attempt.detection_point,
+            vuln_param,
+            meta,
+            payload,
+            pattern)
+        end
+      end
+
+      def self.from_json(version, data_json, payloads_policy)
+        injections_matcher = InjectionsMatcher.from_json(version, data_json)
+
+        InjectionsReporter.new(injections_matcher, payloads_policy)
+      end
+
+    end
+
+  end
+end

data/lib/tcell_agent/appsensor/meta_data.rb

@@ -0,0 +1,71 @@
+require 'tcell_agent/logger'
+require 'tcell_agent/sensor_events/sensor'
+require 'tcell_agent/utils/params'
+
+module TCellAgent
+  module AppSensor
+
+    class MetaData < TCellAgent::SensorEvents::TCellSensorEvent
+
+      attr_accessor :get_dict, :post_dict, :body_dict, :cookie_dict, :path_parameters
+
+      def initialize
+        @send = false
+
+        @body_dict = {}
+        @get_dict = {}
+        @post_dict = {}
+        @cookie_dict = {}
+        @path_parameters = {}
+      end
+
+      def flattened_path_parameters
+        @flattened_path_parameters ||= TCellAgent::Utils::Params.flatten(@path_parameters)
+
+        @flattened_path_parameters
+      end
+
+      def flattened_get_dict
+        @flattened_get_dict ||= TCellAgent::Utils::Params.flatten(@get_dict)
+
+        @flattened_get_dict
+      end
+
+      def flattened_post_dict
+        @flattened_post_dict ||= TCellAgent::Utils::Params.flatten(@post_dict)
+
+        @flattened_post_dict
+      end
+
+      def flattened_body_dict
+        @body_dict
+      end
+
+      def flattened_cookie_dict
+        @flattened_cookie_dict ||= TCellAgent::Utils::Params.flatten(@cookie_dict)
+
+        @flattened_cookie_dict
+      end
+
+      def set_body_dict(request_content_len, request_content_type, request_body)
+        if request_content_len > 2000000
+          @body_dict = {}
+
+        else
+          if request_content_type =~ %r{application/json}i && request_body
+            begin
+              # don't enqueue parameter values of unknown type to avoid any serialization issues
+              @body_dict = TCellAgent::Utils::Params.flatten(JSON.parse(request_body))
+            rescue
+              TCellAgent.logger.debug("JSON body parameter parsing failed")
+              @body_dict = {}
+            end
+          else
+            @body_dict = {}
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/appsensor/rules/appsensor_rule_manager.rb

@@ -1,15 +1,18 @@
+require 'singleton'
 require 'tcell_agent/appsensor/rules/appsensor_rule_set'
 
 module TCellAgent
 
   class AppSensorRuleManager
 
+    include Singleton
+
     attr_accessor :rule_info
 
-    def initialize(filename=nil)
+    def initialize
       @rule_info = {}
 
-      load_rules_file(filename) if filename
+      load_default_rules_file
     end
 
     def load_default_rules_file

data/lib/tcell_agent/appsensor/rules/appsensor_rule_set.rb

@@ -23,7 +23,7 @@ module TCellAgent
       @patterns.each do |pattern|
         next if pattern.nil? || pattern.enabled == false
 
-        if v1_compatability_enabled || active_pattern_ids.fetch(pattern.pattern_id, false)
+        if v1_compatability_enabled || active_pattern_ids.include?(pattern.pattern_id)
           pattern_result = param_value.match(pattern.pattern_regex)
 
           if pattern_result

data/lib/tcell_agent/appsensor/sensor.rb

@@ -0,0 +1,48 @@
+require 'tcell_agent/sensor_events/appsensor_event'
+
+module TCellAgent
+  module AppSensor
+
+    class Sensor
+      class << self
+        def send_event(appsensor_meta, detection_point, parameter, meta, payload, pattern)
+          event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
+            appsensor_meta.location,
+            detection_point,
+            appsensor_meta.method,
+            appsensor_meta.remote_address,
+            parameter,
+            appsensor_meta.route_id,
+            meta,
+            appsensor_meta.session_id,
+            appsensor_meta.user_id,
+            payload,
+            pattern
+          )
+
+          TCellAgent.send_event(event)
+        end
+
+        def send_event_from_tcell_data(tcell_data, detection_point, parameter, meta)
+          payload = pattern = nil
+          event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
+            tcell_data.uri,
+            detection_point,
+            tcell_data.request_method,
+            tcell_data.ip_address,
+            parameter,
+            tcell_data.route_id,
+            meta,
+            tcell_data.session_id,
+            tcell_data.user_id,
+            payload,
+            pattern
+          )
+
+          TCellAgent.send_event(event)
+        end
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/configuration.rb

@@ -19,6 +19,8 @@ module TCellAgent
     attr_accessor :version, :app_id, :api_key, :hmac_key,
       :tcell_api_url, :tcell_input_url,
       :logging_options,
+      :logger,
+      :appfirewall_payloads_logger,
       :fetch_policies_from_tcell, :instrument_for_events,
       :preload_policy_filename,
       :proxy_host, :proxy_port, :proxy_username, :proxy_password,
@@ -30,6 +32,7 @@ module TCellAgent
       :cache_filename,
       :js_agent_api_base_url,
       :js_agent_url,
+      :startup_js_agent_url,
       :raise_exceptions,
       :allow_unencrypted_appfirewall_payloads,
       :config_filename,
@@ -39,7 +42,8 @@ module TCellAgent
       :agent_home_dir,
       :agent_home_owner,
       :reverse_proxy,
-      :reverse_proxy_ip_address_header
+      :reverse_proxy_ip_address_header,
+      :log_file_name
 
     attr_accessor :disable_all,
       :enabled,
@@ -77,6 +81,9 @@ module TCellAgent
       @cache_filename = nil
       @agent_log_dir = nil
 
+      @logger = nil
+      @appfirewall_payloads_logger = nil
+
       @version = 0
       @exp_config_settings = true
       @demomode = false
@@ -96,6 +103,7 @@ module TCellAgent
       @agent_home_dir = File.join(Dir.getwd, "tcell")
       @config_filename = File.join(Dir.getwd, filename)
 
+      @log_file_name = "tcell_agent.log"
 
       @event_batch_size_limit = 50
       @event_time_limit_seconds = 15
@@ -113,6 +121,8 @@ module TCellAgent
         puts "tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS is deprecated, please switch to TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS."
       end
 
+      @allow_unencrypted_appfirewall_payloads = false
+
       # Because ENV can override this one
       env_unencrypted_firewall = 
       if (ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"] != nil)
@@ -128,6 +138,7 @@ module TCellAgent
       @tcell_input_url ||= "https://input.tcell.io/api/v1"
       @js_agent_api_base_url ||= nil
       @js_agent_url ||= "https://api.tcell.io/tcellagent.min.js"
+      @startup_js_agent_url = @js_agent_url
 
       if (@host_identifier == nil)
         begin
@@ -203,6 +214,8 @@ module TCellAgent
             @agent_home_owner = app_data.fetch("agent_home_owner",@agent_home_owner)
 
             @logging_options = app_data.fetch("logging_options", {})
+            @agent_log_dir = @logging_options.fetch("log_dir", @agent_log_dir)
+            @log_file_name = @logging_options.fetch("filename", @log_file_name)
 
             @tcell_api_url = app_data.fetch("tcell_api_url", @tcell_api_url)
             @tcell_input_url = app_data.fetch("tcell_input_url", @tcell_input_url)
@@ -268,7 +281,7 @@ module TCellAgent
 
     def log_filename
       @agent_log_dir ||= File.join(@agent_home_dir, "logs")
-      File.join(@agent_log_dir, "tcell_agent.log")
+      File.join(@agent_log_dir, @log_file_name)
     end
 
     def appfirewall_payloads_log_filename