tcell_agent 0.2.29 → 0.4.0

data/lib/tcell_agent/policies/appsensor_policy.rb

@@ -1,231 +1,47 @@
 require 'tcell_agent/instrumentation'
 require 'tcell_agent/appsensor/injections_reporter'
-require 'tcell_agent/policies/appsensor/cmdi_sensor'
-require 'tcell_agent/policies/appsensor/database_sensor'
-require 'tcell_agent/policies/appsensor/fpt_sensor'
-require 'tcell_agent/policies/appsensor/misc_sensor'
-require 'tcell_agent/policies/appsensor/nullbyte_sensor'
-require 'tcell_agent/policies/appsensor/payloads_policy'
-require 'tcell_agent/policies/appsensor/request_size_sensor'
-require 'tcell_agent/policies/appsensor/response_codes_sensor'
-require 'tcell_agent/policies/appsensor/response_size_sensor'
-require 'tcell_agent/policies/appsensor/retr_sensor'
-require 'tcell_agent/policies/appsensor/sqli_sensor'
-require 'tcell_agent/policies/appsensor/user_agent_sensor'
-require 'tcell_agent/policies/appsensor/xss_sensor'
-require 'tcell_agent/utils/params'
+require 'tcell_agent/rust/models'
+require 'tcell_agent/rust/whisperer'
+require 'tcell_agent/policies/policy'
 
 
 module TCellAgent
   module Policies
 
-    class AppSensorPolicy
-      DETECTION_POINTS_V1 = [
-        "req_res_size",
-        "resp_codes",
-        "ua",
-        "errors",
-        "database"]
+    class AppSensorPolicy < Policy
+      attr_reader :appfirewall_enabled, :appfirewall_ptr
 
-      DETECTION_POINTS_V2 = {
-          "req_size" => RequestSizeSensor,
-          "resp_size" => ResponseSizeSensor,
-          "resp_codes" => ResponseCodesSensor,
-          "xss" => XssSensor,
-          "sqli" => SqliSensor,
-          "cmdi" => CmdiSensor,
-          "fpt" => FptSensor,
-          "nullbyte" => NullbyteSensor,
-          "retr" => RetrSensor,
-          "ua" => UserAgentSensor,
-          "errors" => MiscSensor,
-          "database" => DatabaseSensor
-      }
-
-      DETECTION_POINTS_V2_NON_INJECTION = {
-          "req_size" => RequestSizeSensor,
-          "resp_size" => ResponseSizeSensor,
-          "resp_codes" => ResponseCodesSensor,
-          "ua" => UserAgentSensor,
-          "errors" => MiscSensor,
-          "database" => DatabaseSensor
-      }
-
-      attr_accessor :policy_id, :options, :enabled, :injections_reporter
-
-      def initialize
-        @policy_id = nil
-        @options = Hash.new
-        @enabled = false
-        @injections_reporter =
-          TCellAgent::AppSensor::InjectionsReporter.from_json(0, {}, PayloadsPolicy.new)
+      def initialize(appfirewall_enabled=false, appfirewall_ptr=nil)
+        @appfirewall_ptr = appfirewall_ptr
+        @appfirewall_enabled = appfirewall_enabled
       end
 
       def process_meta_event(appsensor_meta)
-        return unless @enabled
-
-        check_user_agent(appsensor_meta)
-        check_request_size(appsensor_meta)
-        check_response_size(appsensor_meta)
-        check_response_code(appsensor_meta)
-
-        @injections_reporter.check(appsensor_meta)
-      end
-
-      def process_db_rows(tcell_data, number_of_records)
-        return unless @enabled
-
-        TCellAgent::Instrumentation.safe_block("AppSensor Testing Number of DB Rows") do
-          if self.options.has_key?("database")
-            self.options["database"].check(tcell_data, number_of_records)
-          end
-        end
-      end
-
-      def check_user_agent(appsensor_meta)
-        TCellAgent::Instrumentation.safe_block("AppSensor Checking User Agent") do
-          if self.options.has_key?("ua")
-            self.options["ua"].check(appsensor_meta)
-          end
-        end
-      end
-
-      def check_request_size(appsensor_meta)
-        TCellAgent::Instrumentation.safe_block("AppSensor Testing Response Size") do
-          if self.options.has_key?("req_size")
-            self.options["req_size"].check(appsensor_meta)
-          end
-        end
-      end
-
-      def check_response_size(appsensor_meta)
-        return unless @enabled
-
-        TCellAgent::Instrumentation.safe_block("AppSensor Testing Response Size") do
-          if self.options.has_key?("resp_size")
-            self.options["resp_size"].check(appsensor_meta)
-          end
-        end
-      end
-
-      def check_response_code(appsensor_meta)
-        TCellAgent::Instrumentation.safe_block("AppSensor Testing Response Code") do
-          if self.options.has_key?("resp_codes")
-            self.options["resp_codes"].check(appsensor_meta, appsensor_meta.response_code)
-          end
-        end
-      end
+        return unless @appfirewall_enabled && @appfirewall_ptr
 
-      def csrf_rejected(tcell_data, exception_class)
-        TCellAgent::Instrumentation.safe_block("AppSensor CSRF Exception processing") do
-          if self.options.has_key?("errors")
-            self.options["errors"].csrf_rejected(tcell_data, exception_class)
-          end
+        TCellAgent::Instrumentation.safe_block("AppSensor inspection") do
+          request_response = TCellAgent::Rust::Models.create_request_response(appsensor_meta)
+          whisper = TCellAgent::Rust::Whisperer.apply_appfirewall(@appfirewall_ptr, request_response)
+          TCellAgent::AppSensor::InjectionsReporter.report_and_log(whisper["apply_response"])
         end
       end
 
-      def sql_exception_detected(tcell_data, exception)
-        TCellAgent::Instrumentation.safe_block("AppSensor SQL Exception processing") do
-          if self.options.has_key?("errors")
-            self.options["errors"].sql_exception_detected(tcell_data, exception)
-          end
-        end
+      def free_native_memory
+        TCellAgent::Rust::Whisperer.free_appfirewall(@appfirewall_ptr) if @appfirewall_ptr
       end
 
       def self.from_json(policy_json)
         return nil unless policy_json
-        policy_json = policy_json.deep_dup
 
-        sensor_policy = AppSensorPolicy.new
-        if policy_json.has_key?("policy_id")
-          sensor_policy.policy_id = policy_json["policy_id"]
+        whisper = TCellAgent::Rust::Whisperer.init_appfirewall(
+          policy_json, TCellAgent.configuration.allow_payloads
+        )
+        if whisper["error"]
+          TCellAgent.logger.debug("Error initializing AppFirewall Policy: #{whisper['error']}")
+          return AppSensorPolicy.new
         else
-          raise "Policy ID missing"
-        end
-
-        if policy_json.has_key?("data")
-          data_json = policy_json["data"]
-
-          if policy_json["version"] && policy_json["version"] == 2
-            if data_json
-              sensors_json = data_json.fetch("sensors", {})
-
-              if sensors_json.empty?
-                sensor_policy.enabled = false
-
-              else
-                sensor_policy.enabled = true
-
-                options_hash = data_json.fetch("options", {})
-                collect_full_uri = options_hash.fetch("uri_options", {}).fetch("collect_full_uri", false)
-
-                DETECTION_POINTS_V2_NON_INJECTION.each do |sensor_name, sensor_class|
-                  settings = sensors_json.fetch(sensor_name, {})
-                  updated_settings = {
-                    "enabled" => sensors_json.has_key?(sensor_name),
-                    "collect_full_uri" => collect_full_uri
-                  }.merge(settings)
-
-                  sensor_policy.options[sensor_name] =
-                    sensor_class.new(updated_settings)
-                end
-
-                payloads_policy = PayloadsPolicy.from_json(options_hash)
-                sensor_policy.injections_reporter =
-                  TCellAgent::AppSensor::InjectionsReporter.from_json(
-                    2, sensors_json, payloads_policy, collect_full_uri)
-              end
-            end
-
-          else
-            if data_json
-              options_json = data_json.fetch("options", {})
-
-              if options_json.empty?
-                sensor_policy.enabled = false
-
-              else
-                sensor_policy.enabled = true
-
-                payloads_policy = PayloadsPolicy.from_json({
-                  "payloads" => {
-                    "send_payloads" => true,
-                    "log_payloads" => true
-                  }
-                })
-
-                sensor_policy.injections_reporter =
-                  TCellAgent::AppSensor::InjectionsReporter.from_json(1, data_json, payloads_policy)
-
-                enabled = options_json.fetch("req_res_size", false)
-                sensor_policy.options["req_size"] = RequestSizeSensor.new({"enabled" => enabled})
-                sensor_policy.options["resp_size"] = ResponseSizeSensor.new({"enabled" => enabled})
-
-                enabled = options_json.fetch("resp_codes", false)
-                sensor_policy.options["resp_codes"] = ResponseCodesSensor.new({
-                    "enabled" => enabled,
-                    "series_400_enabled" => true,
-                    "series_500_enabled" => true})
-
-                sensor_policy.options["ua"] = UserAgentSensor.new({
-                  "enabled" => false, "empty_enabled" => false
-                })
-
-                sensor_policy.options["errors"] = MiscSensor.new({
-                  "enabled" => false,
-                  "csrf_exception_enabled" => false,
-                  "sql_exception_enabled" => false
-                })
-
-                sensor_policy.options["database"] = DatabaseSensor.new({
-                  "enabled" => false
-                })
-              end
-            end
-          end
+          return AppSensorPolicy.new(whisper["enabled"], whisper["policy_ptr"])
         end
-
-        return sensor_policy
       end
     end
 

data/lib/tcell_agent/policies/clickjacking_policy.rb

@@ -2,10 +2,11 @@
 # See the file "LICENSE" for the full license governing this code.
 
 require 'uri'
+require 'tcell_agent/policies/policy'
 
 module TCellAgent
     module Policies
-        class ClickjackingPolicy
+        class ClickjackingPolicy < Policy
             class ContentSecurityPolicyHeader
                 @@approved_headers = [
                     "csp"
@@ -71,8 +72,9 @@ module TCellAgent
                 end
                 result.each(&block)
             end
+
             def self.from_json(policy_json)
-                if (!policy_json) 
+                if (!policy_json)
                     return nil
                 end
                 csp = ClickjackingPolicy.new

data/lib/tcell_agent/policies/command_injection_policy.rb

@@ -0,0 +1,196 @@
+require 'tcell_agent/rust/whisperer'
+require 'tcell_agent/sensor_events/command_injection'
+require 'tcell_agent/policies/policy'
+
+
+module TCellAgent
+  module Policies
+
+    class CommandRule
+      IGNORE = "ignore"
+      REPORT = "report"
+      BLOCK = "block"
+
+      attr_reader :rule_id, :action, :command
+
+      def initialize(policy_json)
+        @rule_id = nil
+        @action = nil
+        @command = nil
+
+        if policy_json
+          @rule_id = policy_json["rule_id"]
+          @action = policy_json["action"]
+          @command = policy_json["command"]
+        end
+      end
+
+      def ignore?
+        @action == IGNORE
+      end
+
+      def report?
+        @action == REPORT
+      end
+
+      def block?
+        @action == BLOCK
+      end
+
+      def valid?
+        !!@rule_id && [IGNORE, REPORT, BLOCK].include?(@action)
+      end
+    end
+
+    class CommandInjectionPolicy < Policy
+      attr_accessor :policy_id, :version, :enabled, :overall_action, :command_rules, :compound_statement_rule,
+        :collect_full_commandline
+
+      def initialize
+        @enabled = false
+        @version = nil
+        @policy_id = nil
+        @overall_action = nil
+        @command_rules = {}
+        @compound_statement_rule = nil
+        @collect_full_commandline = false
+      end
+
+      def self.from_json(policy_json)
+        return nil unless policy_json
+        policy_json = policy_json.deep_dup
+
+        policy_id = policy_json["policy_id"]
+
+        raise "Policy ID missing" unless policy_id
+
+        command_injection_policy = CommandInjectionPolicy.new
+        command_injection_policy.policy_id = policy_id
+        command_injection_policy.version = policy_json["version"]
+
+        if 1 != command_injection_policy.version
+          TCellAgent.logger.error("Command Injection not supported: #{command_injection_policy.version}")
+          return command_injection_policy
+        end
+
+        policy_data = policy_json["data"]
+        command_rules = {}
+        overall_action = nil
+        compound_statement_rule = nil
+        if policy_data
+          command_injection_policy.collect_full_commandline = !!policy_data["collect_full_commandline"]
+
+          (policy_data["command_rules"] or []).each do |command_rule_policy|
+            command_rule = CommandRule.new(command_rule_policy)
+            if command_rule.valid?
+              command = command_rule.command
+              if command
+                if command_rules.has_key?(command)
+                  TCellAgent.logger.warn(
+                    "CommandInjectionPolicy multiple rules for one " +
+                    "command (dropping rule): #{command} #{command_rule.action}"
+                  )
+                else
+                  command_rules[command] = command_rule
+                end
+              elsif !command_rule.ignore?
+                overall_action = command_rule
+              end
+            end
+          end
+
+          compound_statement_rules = policy_data["compound_statement_rules"]
+          if compound_statement_rules && compound_statement_rules.size > 0
+            compound_statement_rule = CommandRule.new(compound_statement_rules[0])
+            if !compound_statement_rule.valid? || compound_statement_rule.ignore?
+              compound_statement_rule = nil
+            end
+          end
+
+          command_injection_policy.command_rules = command_rules
+          command_injection_policy.overall_action = overall_action
+          command_injection_policy.compound_statement_rule = compound_statement_rule
+          command_injection_policy.enabled = !overall_action.nil? || !compound_statement_rule.nil? || command_rules.size > 0
+        end
+
+        command_injection_policy
+      end
+
+      def block?(cmd, tcell_context)
+        return false unless @enabled
+
+        commands = parse_cmd(cmd).fetch('commands', [])
+
+        command_injection_match_events = []
+        block_command = false
+
+        if commands.size > 1 && !!@compound_statement_rule
+          if !@compound_statement_rule.ignore?
+            command_injection_match_events.push(
+              TCellAgent::SensorEvents::CommandInjectionMatchEvent.new(
+                @compound_statement_rule.rule_id, @compound_statement_rule.command
+              )
+            )
+            block_command = block_command || @compound_statement_rule.block?
+          end
+        end
+
+        commands.each do |command_info|
+          command = command_info["command"]
+          command_rule = command_rules[command] || @overall_action
+          if command_rule && !command_rule.ignore?
+            command_injection_match_events.push(
+              TCellAgent::SensorEvents::CommandInjectionMatchEvent.new(
+                command_rule.rule_id,
+                command
+              )
+            )
+            block_command = block_command || command_rule.block?
+          end
+        end
+
+        if command_injection_match_events.size > 0
+          method, remote_address, route_id, session_id, user_id, full_commandline = nil
+          if tcell_context
+            method = tcell_context.request_method
+            remote_address = tcell_context.ip_address
+            route_id = tcell_context.route_id
+            session_id = tcell_context.hmac_session_id
+            user_id = tcell_context.user_id
+          end
+
+          if @collect_full_commandline
+            full_commandline = cmd
+          end
+
+          TCellAgent.send_event(
+            TCellAgent::SensorEvents::CommandInjectionEvent.new(
+              commands,
+              block_command,
+              command_injection_match_events,
+              method=method,
+              remote_address=remote_address,
+              route_id=route_id,
+              session_id=session_id,
+              user_id=user_id,
+              full_commandline=full_commandline
+            )
+          )
+        end
+
+        block_command
+      end
+
+      private
+        def parse_cmd(cmd)
+          TCellAgent::Instrumentation.safe_block("Call Rust Parse Command") do
+            require "tcell_agent/rust/whisperer"
+
+            return TCellAgent::Rust::Whisperer.parse_cmd(cmd)
+          end
+
+          return {}
+        end
+    end
+  end
+end

data/lib/tcell_agent/policies/content_security_policy.rb

@@ -2,13 +2,14 @@
 # See the file "LICENSE" for the full license governing this code.
 
 require 'uri'
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/configuration'
+require 'tcell_agent/policies/policy'
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 
 module TCellAgent
   module Policies
 
-    class ContentSecurityPolicy
+    class ContentSecurityPolicy < Policy
       class ContentSecurityPolicyHeader
         @@approved_headers = [
           "csp",

data/lib/tcell_agent/policies/dataloss_policy.rb

@@ -1,8 +1,10 @@
 require 'set'
+require 'tcell_agent/policies/policy'
+
 
 module TCellAgent
     module Policies
-        class DataLossPolicy
+        class DataLossPolicy < Policy
             class FilterActions
                 attr_accessor :body_event
                 attr_accessor :body_redact

data/lib/tcell_agent/policies/honeytokens_policy.rb

@@ -10,10 +10,12 @@
 #+"}";
 require 'pbkdf2'
 require 'openssl'
+require 'tcell_agent/policies/policy'
+
 
 module TCellAgent
     module Policies
-        class HoneytokensPolicy
+        class HoneytokensPolicy < Policy
             attr_accessor :policy_id
             attr_accessor :token_salt
             attr_accessor :cred_tokens

data/lib/tcell_agent/policies/http_redirect_policy.rb

@@ -1,58 +1,59 @@
 # See the file "LICENSE" for the full license governing this code.
 require 'uri'
-require 'tcell_agent/logger'
-require 'tcell_agent/sensor_events/util/redirect_utils'
-# See the file "LICENSE" for the full license governing this code.
 
-#{}"http-tx": {
-#        "policy_id":"afh023",
-#        "types": {
-#               "firehose": { enabled: true },
-#{}"auth_framework_only": {enabled: true},
-#{}"{}structure": {enabled: true },
-#{}"fingerprint": {enabled: true }
-#}
-#},
+require 'tcell_agent/policies/policy'
+require 'tcell_agent/logger'
 
 module TCellAgent
   module Policies
-    class HttpRedirectPolicy
-      attr_accessor :policy_id
-      attr_accessor :enabled
-      attr_accessor :whitelist
-      attr_accessor :block
+
+    class HttpRedirectPolicy < Policy
+      attr_accessor :policy_id, :enabled, :whitelist, :block, :data_scheme_allowed
+
       def initialize
         @policy_id = nil
         @enabled = false
         @whitelist = []
         @block = false
+        @data_scheme_allowed = false
       end
-      def check(host, current_host)
+
+      def suspicious_redirect?(host, current_host)
         if (!(host) || host == "" || host == current_host)
           # local redirect
           return false
         end
-        if (whitelist)
-          whitelist.each do |domain|
-            if (TCellAgent::SensorEvents::Util.wildcardMatch(host, domain))
-              return false
-            end
+
+        whitelist.each do |whitelist_regex|
+          if (host =~ whitelist_regex) || ("www.#{host}" =~ whitelist_regex)
+            return false
           end
         end
-        return true
+
+        true
       end
-      def enforce(target_url, current_host, current_path, method, route_id, status_code, remote_addr, hmac_session_id=nil)
-        if @enabled == false
-          return nil
-        end
-        uri = URI.parse(target_url)
-        host = uri.host
-        if self.check(host, current_host) == false
-          return nil
+
+      def enforce(target_uri, request_uri, current_path, method, route_id, status_code, remote_addr, hmac_session_id=nil)
+        return nil unless @enabled
+
+        current_host = URI.parse(request_uri).host
+        if target_uri.downcase.start_with?("data:")
+          if @data_scheme_allowed
+            return nil
+          end
+
+          target_host = target_uri.split(",")[0]
+
+        else
+          target_host = URI.parse(target_uri).host
+          if !self.suspicious_redirect?(target_host, current_host)
+            return nil
+          end
         end
+
         begin
           event = TCellAgent::SensorEvents::TCellRedirectSensorEvent.new(
-            host,
+            target_host,
             current_host,
             current_path,
             method,
@@ -66,29 +67,42 @@ module TCellAgent
         rescue Exception => ie
           TCellAgent.logger.error("uncaught exception while creating redirect event: #{ie.message}")
         end
-        if @block == true
+
+        if @block
           return "/"
+        else
+          return nil
         end
-        return nil
       end
+
       def self.from_json(policy_json)
-        if (!policy_json) 
+        if (!policy_json)
           return nil
         end
+
         http_redirect_policy = HttpRedirectPolicy.new
         if policy_json.has_key?("policy_id")
           http_redirect_policy.policy_id = policy_json["policy_id"]
         else
           raise "Policy ID missing"
         end
+
         if policy_json.has_key?("data")
           policy_data_json = policy_json["data"]
           http_redirect_policy.enabled = policy_data_json.fetch("enabled", false)
-          http_redirect_policy.whitelist = policy_data_json.fetch("whitelist", [])
           http_redirect_policy.block = policy_data_json.fetch("block", false)
+          http_redirect_policy.data_scheme_allowed = policy_data_json.fetch("dataSchemeAllowed", false)
+
+          http_redirect_policy.whitelist = []
+          policy_data_json.fetch("whitelist", []).each do |regex_pattern|
+            escaped = Regexp.escape(regex_pattern).gsub('\*','.*?')
+            http_redirect_policy.whitelist.push(Regexp.new("^#{escaped}$", Regexp::IGNORECASE))
+          end
         end
+
         return http_redirect_policy
       end
     end
+
   end
 end

data/lib/tcell_agent/policies/http_tx_policy.rb

@@ -8,9 +8,12 @@
 #}
 #},
 
+require 'tcell_agent/policies/policy'
+
+
 module TCellAgent
     module Policies
-        class HttpTxPolicy
+        class HttpTxPolicy < Policy
             attr_accessor :policy_id
             attr_accessor :firehose
             attr_accessor :auth_framework
@@ -23,6 +26,7 @@ module TCellAgent
                 @profile = {"enabled"=>false }
                 @fingerprint = {"enabled"=>false, "hmacUserAgent"=>false, "hmacUserId"=>false, "sampling"=>nil }
             end
+
             def self.from_json(policy_json)
                 if (!policy_json) 
                     return nil