tcell_agent 0.2.29 → 0.4.0

data/lib/tcell_agent/sensor_events/appsensor_meta_event.rb

@@ -21,30 +21,37 @@ module TCellAgent
 
       class << self
         def build(request, response_content_length, response_code, response_headers)
-          meta_event = AppSensorMetaEvent.new
+          tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
+          meta_event = AppSensorMetaEvent.new(
+            request.request_method,
+            TCellAgent::Utils::Rails.better_ip(request),
+            tcell_context.route_id,
+            tcell_context.hmac_session_id,
+            tcell_context.user_id,
+            tcell_context.transaction_id
+          )
 
-          meta_event.remote_address = TCellAgent::Utils::Rails.better_ip(request)
-          meta_event.method = request.request_method
-          meta_event.location = "#{request.base_url}#{request.fullpath}"
-          meta_event.request_headers = request.env
+          meta_event.csrf_exception_name = tcell_context.csrf_exception_name
           meta_event.user_agent = request.env['HTTP_USER_AGENT']
           meta_event.request_content_bytes_len = (request.content_length || 0).to_i
           meta_event.response_content_bytes_len = response_content_length
 
+          meta_event.location = "#{request.base_url}#{request.fullpath}"
+          meta_event.path = request.path
+
           meta_event.get_dict = request.GET
           meta_event.cookie_dict = request.cookies
+          meta_event.set_headers_dict(request.env)
 
           # don't enqueue parameter values of unknown type to avoid any serialization issues
           meta_event.post_dict = TCellAgent::Utils::Params.flatten(request.POST)
 
+          meta_event.path_parameters = request.env[TCellAgent::Instrumentation::TCELL_ID].path_parameters
           meta_event.response_code = response_code
           meta_event.response_headers = response_headers
 
-          meta_event.path_parameters = request.env[TCellAgent::Instrumentation::TCELL_ID].path_parameters
-          meta_event.route_id = request.env[TCellAgent::Instrumentation::TCELL_ID].route_id
-          meta_event.transaction_id = request.env[TCellAgent::Instrumentation::TCELL_ID].transaction_id
-          meta_event.session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
-          meta_event.user_id = request.env[TCellAgent::Instrumentation::TCELL_ID].user_id
+          meta_event.sql_exceptions = tcell_context.sql_exceptions
+          meta_event.database_result_sizes = tcell_context.database_result_sizes
 
           # Positions strio to the beginning of input, resetting lineno to zero.
           # rails 4.1 seems to read the stringIO directly and so body.gets is empty
@@ -61,28 +68,28 @@ module TCellAgent
         end
 
         def build_basic(appsensor_meta)
-          meta_event = AppSensorMetaEvent.new
+          meta_event = AppSensorMetaEvent.new(
+            appsensor_meta.method,
+            appsensor_meta.remote_address,
+            appsensor_meta.route_id,
+            appsensor_meta.session_id,
+            appsensor_meta.user_id,
+            appsensor_meta.transaction_id
+          )
+
           meta_event.location = appsensor_meta.location
-          meta_event.method =appsensor_meta.method
-          meta_event.remote_address = appsensor_meta.remote_address
-          meta_event.route_id = appsensor_meta.route_id
-          meta_event.session_id = appsensor_meta.session_id
-          meta_event.user_id = appsensor_meta.user_id
-          meta_event.route_id = appsensor_meta.route_id
 
           meta_event
         end
       end
 
 
-      attr_accessor :remote_address, :method, :location, :route_id, :session_id, :user_id, :transaction_id,
-        :request_content_bytes_len, :get_dict, :post_dict, :body_dict, :cookie_dict, :response_content_bytes_len, :response_code,
-        :user_agent, :path_parameters
-
-      attr_accessor :request_headers, :response_headers
+      attr_accessor :location, :request_content_bytes_len, :response_content_bytes_len,
+        :response_code, :user_agent, :response_headers, :csrf_exception_name, :path,
+        :sql_exceptions, :database_result_sizes
 
-      def initialize
-        super
+      def initialize(method, remote_address, route_id, session_id, user_id, transaction_id)
+        super(method, remote_address, route_id, session_id, user_id, transaction_id)
 
         @request_content_bytes_len = 0
         @response_content_bytes_len = 0

data/lib/tcell_agent/sensor_events/command_injection.rb

@@ -0,0 +1,58 @@
+require 'tcell_agent/sensor_events/sensor'
+
+module TCellAgent
+  module SensorEvents
+
+    class CommandInjectionMatchEvent < Hash
+      def initialize(rule_id, command)
+        self["rule_id"] = rule_id
+        if command
+          self["command"] = command
+        end
+      end
+    end
+
+    class CommandInjectionEvent < TCellSensorEvent
+      def initialize(commands,
+                     blocked,
+                     matches,
+                     method=nil,
+                     remote_address=nil,
+                     route_id=nil,
+                     session_id=nil,
+                     user_id=nil,
+                     full_commandline=nil)
+        super("cmdi")
+
+        self["commands"] = commands
+        self["blocked"] = blocked
+        self["matches"] = matches
+
+        if method
+          self["method"] = method
+        end
+
+        if remote_address
+          self["remote_address"] = remote_address
+        end
+
+        if route_id
+          self["route_id"] = route_id
+        end
+
+        if session_id
+          self["session_id"] = session_id
+        end
+
+        if user_id
+          self["user_id"] = user_id
+        end
+
+        if full_commandline
+          self["full_commandline"] = full_commandline
+        end
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/sensor_events/discovery.rb

@@ -1,4 +1,4 @@
-require 'tcell_agent/sensor_events/sensor'        
+require 'tcell_agent/sensor_events/sensor'
 
 module TCellAgent
     module SensorEvents

data/lib/tcell_agent/sensor_events/login_fraud.rb

@@ -2,7 +2,6 @@
 
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 
 module TCellAgent
   module SensorEvents
@@ -14,26 +13,17 @@ module TCellAgent
         self["header_keys"] = header_keys
 
         self["user_agent"] = tcell_data.user_agent.to_s if tcell_data.user_agent
-        self["referrer"] = tcell_data.referrer.to_s if tcell_data.referrer
+        self["referrer"] = TCellAgent::SensorEvents::Util.strip_uri_values(tcell_data.referrer) if tcell_data.referrer
         self["remote_addr"] = tcell_data.ip_address.to_s if tcell_data.ip_address
         self["user_id"] = user_id.to_s if user_id
         self["document_uri"] = TCellAgent::SensorEvents::Util.strip_uri_values(tcell_data.path) if tcell_data.path
         self["session"] = tcell_data.hmac_session_id if tcell_data.hmac_session_id
       end
-
-      protected
-        def clean_header_keys(request_env_or_header_keys)
-          if request_env_or_header_keys.is_a?(Hash)
-            request_env_or_header_keys.select {|k,v| k.start_with? 'HTTP_'}.collect {|k,v| k.sub(/^HTTP_/, '') }
-          else
-            request_env_or_header_keys.map { |k| k.sub(/^HTTP_/, '') }
-          end
-        end
     end
 
     class LoginFailure < LoginEvent
       def initialize(request_env_or_header_keys, tcell_data, user_id, user_valid=nil)
-        header_keys = clean_header_keys(request_env_or_header_keys)
+        header_keys = Util.clean_header_keys(request_env_or_header_keys)
 
         super(header_keys, tcell_data, user_id, user_valid)
 
@@ -43,7 +33,7 @@ module TCellAgent
 
     class LoginSuccess < LoginEvent
       def initialize(request_env_or_header_keys, tcell_data, user_id, user_valid=nil)
-        header_keys = clean_header_keys(request_env_or_header_keys)
+        header_keys = Util.clean_header_keys(request_env_or_header_keys)
 
         super(header_keys, tcell_data, user_id, user_valid)
 

data/lib/tcell_agent/sensor_events/sensor.rb

@@ -5,86 +5,90 @@ require 'tcell_agent/logger'
 require 'uri'
 
 module TCellAgent
-    module SensorEvents
-        class TCellSensorEvent < Hash
-            attr_accessor :send
-            attr_accessor :flush
-            attr_accessor :ensure
-            def initialize(event_type)
-                @send = true
-                @flush = false
-                @ensure = false
-                @timestamp = DateTime.now.to_time.to_i
-                self["event_type"] = event_type
-            end
-            def calculateOffset(from_timestamp)
-                self["offset"] = from_timestamp - @timestamp
-            end
-            def post_process
-                # This is called in the background thread, so any
-                #   santization, analysis, etc doesn't get in the way
-            end
-            def bucket_key
-                return nil
-            end
-        end
-        class TCellHttpTxSensorEvent < TCellSensorEvent
-            def initialize(request, response)
-                super("http_tx")
-                @raw_request = request
-                @raw_response = response
-            end
-            def post_process
-                if defined?@raw_request
-                    self["request"] = Util.request_sanitized_json(@raw_request)
-                end
-                if defined?@raw_response
-                    self["response"] = Util.response_sanitized_json(@raw_response)
-                end
-            end
+  module SensorEvents
+
+    class TCellSensorEvent < Hash
+      attr_accessor :send, :flush, :ensure
+
+      def initialize(event_type)
+        @send = true
+        @flush = false
+        @ensure = false
+        @timestamp = DateTime.now.to_time.to_i
+        self["event_type"] = event_type
+      end
+
+      def calculateOffset(from_timestamp)
+        self["offset"] = from_timestamp - @timestamp
+      end
+
+      def post_process
+        # This is called in the background thread, so any
+        #   santization, analysis, etc doesn't get in the way
+      end
+
+      def bucket_key
+        return nil
+      end
+    end
+
+    class TCellHttpTxSensorEvent < TCellSensorEvent
+      def initialize(request, response)
+        super("http_tx")
+        @raw_request = request
+        @raw_response = response
+      end
+      def post_process
+        if defined?@raw_request
+          self["request"] = Util.request_sanitized_json(@raw_request)
         end
-        class TCellRedirectSensorEvent < TCellSensorEvent
-            def initialize(redirect_domain, original_domain, original_url, method, route_id, status_code, remote_addr, hmac_session_id=nil, user_id=nil)
-                super("redirect")
-                @raw_original_url = original_url
-                self["method"] = method
-                self["from_domain"] = original_domain
-                self["status_code"] = status_code
-                self["remote_addr"] = remote_addr
-                if route_id
-                    self["rid"] = route_id
-                end
-                @raw_redirect_domain = redirect_domain
-                @user_id = user_id
-                @hmac_session_id = hmac_session_id
-            end
-            def post_process
-                self["from"] = Util.strip_uri_values(@raw_original_url)
-                self["to"] = @raw_redirect_domain
-                if @hmac_session_id
-                  self["sid"] = @hmac_session_id
-                end
-            end
+        if defined?@raw_response
+          self["response"] = Util.response_sanitized_json(@raw_response)
         end
+      end
+    end
 
-        class TCellFingerprintSensorEvent < TCellSensorEvent
-            def initialize(request, hmac_session_id, user_id=nil)
-                super("fingerprint")
-                @raw_request = request
-                @hmac_session_id = hmac_session_id
-                @user_id = user_id
-            end
-            def post_process
-                if !(@raw_request.headers.key?("HTTP_USER_AGENT"))
-                    raise "User Agent not Found!"
-                end
-                self["ua"] = @raw_request.headers["HTTP_USER_AGENT"]
-                self["ip"] = @raw_request.remote_ip
-                self["sid"] = @hmac_session_id
-                if @user_id
-                    self["uid"] = @user_id
-                end
-            end
+    class TCellRedirectSensorEvent < TCellSensorEvent
+      def initialize(redirect_domain,
+                     original_domain,
+                     original_url,
+                     method,
+                     route_id,
+                     status_code,
+                     remote_addr,
+                     hmac_session_id=nil,
+                     user_id=nil)
+        super("redirect")
+        self["method"] = method
+        self["from_domain"] = original_domain
+        self["status_code"] = status_code
+        self["remote_addr"] = remote_addr
+        self["to"] = redirect_domain
+        self["uid"] = user_id.to_s if user_id
+        self["from"] = Util.strip_uri_values(original_url)
+        self["rid"] = route_id if route_id
+        self["sid"] = hmac_session_id if hmac_session_id
+      end
+    end
+
+    class TCellFingerprintSensorEvent < TCellSensorEvent
+      def initialize(request, hmac_session_id, user_id=nil)
+        super("fingerprint")
+        @raw_request = request
+        @hmac_session_id = hmac_session_id
+        @user_id = user_id
+      end
+      def post_process
+        if !(@raw_request.headers.key?("HTTP_USER_AGENT"))
+          raise "User Agent not Found!"
+        end
+        self["ua"] = @raw_request.headers["HTTP_USER_AGENT"]
+        self["ip"] = @raw_request.remote_ip
+        self["sid"] = @hmac_session_id
+        if @user_id
+          self["uid"] = @user_id
         end
+      end
     end
+  end
 end

data/lib/tcell_agent/sensor_events/util/sanitizer_utilities.rb

@@ -157,6 +157,14 @@ module TCellAgent
         end
         return "tcell_hmac_key"
       end
+
+      def self.clean_header_keys(request_env_or_header_keys)
+        if request_env_or_header_keys.is_a?(Hash)
+          request_env_or_header_keys.select {|k,v| k.start_with? 'HTTP_'}.collect {|k,v| k.sub(/^HTTP_/, '') }
+        else
+          request_env_or_header_keys.map { |k| k.sub(/^HTTP_/, '') }
+        end
+      end
     end
   end
 end

data/lib/tcell_agent/start_background_thread.rb

@@ -8,9 +8,11 @@ if (TCellAgent.configuration.disable_all == false)
   module TCellAgent
     #require 'tcell_agent/sinatra' if defined?(Sinatra)
     require 'tcell_agent/rails' if defined?(Rails)
+    require 'tcell_agent/cmdi'
 
     def self.run_instrumentation(server_name, send_startup_events=true)
 
+      require 'tcell_agent/hooks/login_fraud'
       require 'tcell_agent/rails/on_start' if defined?(Rails)
 
       TCellAgent::Instrumentation.safe_block("Starting thread agent") do
@@ -19,7 +21,6 @@ if (TCellAgent.configuration.disable_all == false)
       end
 
       if send_startup_events && TCellAgent.configuration.should_instrument?
-        Thread.abort_on_exception = TCellAgent.configuration.raise_exceptions
         Thread.new do
 
           TCellAgent::Instrumentation.safe_block("Instrumenting Agent Details") do
@@ -33,10 +34,18 @@ if (TCellAgent.configuration.disable_all == false)
           end
 
           TCellAgent::Instrumentation.safe_block("Instrumenting Initial Config") do
+            require 'tcell_agent/rust/whisperer'
+
+            TCellAgent.send_event(
+              TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
+              "native_lib_loaded",
+              TCellAgent::Rust::Wrapper.common_lib_available?.to_s)
+            )
+
             TCellAgent.send_event(
               TCellAgent::SensorEvents::TCellAgentSettingEvent.new(
-              "allow_unencrypted_appfirewall_payloads",
-              (!!TCellAgent.configuration.allow_unencrypted_appfirewall_payloads).to_s)
+              "allow_payloads",
+              (!!TCellAgent.configuration.allow_payloads).to_s)
             )
 
             TCellAgent.send_event(

data/lib/tcell_agent/utils/io.rb

@@ -1,4 +1,7 @@
-require 'pathname'
+require "pathname"
+require "tcell_agent/instrumentation"
+require "tcell_agent/utils/strings"
+
 
 module TCellAgent
   module Utils

data/lib/tcell_agent/utils/params.rb

@@ -6,6 +6,7 @@ module TCellAgent
       JSON_PARAM = "json"
       URI_PARAM = "uri"
       COOKIE_PARAM = "cookies"
+      HEADER_PARAM = "header"
 
       def self.flatten(param_dict, namespace=nil)
         flattened = {}

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "0.2.29"
+  VERSION = "0.4.0"
 end

data/lib/tcell_agent.rb

@@ -22,7 +22,6 @@ require 'tcell_agent/policies/dataloss_policy'
 
 require 'tcell_agent/sensor_events/dlp'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
-require 'tcell_agent/sensor_events/util/redirect_utils'
 
 require 'tcell_agent/instrumentation'
 require 'tcell_agent/start_background_thread'

data/spec/lib/tcell_agent/appsensor/injections_matcher_spec.rb

@@ -363,14 +363,22 @@ module TCellAgent
       end
 
       describe "#check_param_for_injections" do
+        before(:each) do
+          @meta_data = TCellAgent::AppSensor::MetaData.new(
+            "get",
+            "remote_address",
+            "route_id",
+            "session_id",
+            "user_id",
+            "transaction_id")
+        end
+
         context "with no sensors" do
           it "should not find any injections" do
             injection_matcher = InjectionsMatcher.new([])
 
-            meta_data = TCellAgent::AppSensor::MetaData.new
-
             result = injection_matcher.check_param_for_injections(
-              InjectionsMatcher::URI_PARAM, meta_data, "dirty", "<script></script>"
+              InjectionsMatcher::URI_PARAM, @meta_data, "dirty", "<script></script>"
             )
 
             expect(result).to eq(nil)
@@ -383,17 +391,15 @@ module TCellAgent
 
             injection_matcher = InjectionsMatcher.new([fake_sensor])
 
-            meta_data = TCellAgent::AppSensor::MetaData.new
-
             expect(fake_sensor).to receive(:applicable_for_param_type?).with(
               InjectionsMatcher::URI_PARAM
             ).and_return(true)
             expect(fake_sensor).to receive(:get_injection_attempt).with(
-              InjectionsMatcher::URI_PARAM, meta_data, "dirty", "<script></script>"
+              InjectionsMatcher::URI_PARAM, @meta_data, "dirty", "<script></script>"
             ).and_return({"injection" => true})
 
             result = injection_matcher.check_param_for_injections(
-              InjectionsMatcher::URI_PARAM, meta_data, "dirty", "<script></script>"
+              InjectionsMatcher::URI_PARAM, @meta_data, "dirty", "<script></script>"
             )
 
             expect(result).to eq({"injection" => true})
@@ -405,7 +411,13 @@ module TCellAgent
         context "with appsensor meta data" do
           context "with one param of each type" do
             it "should call check_param_for_injections once for each param" do
-              meta_data = TCellAgent::SensorEvents::AppSensorMetaEvent.new
+              meta_data = TCellAgent::SensorEvents::AppSensorMetaEvent.new(
+                "get",
+                "remote_address",
+                "route_id",
+                "session_id",
+                "user_id",
+                "transaction_id")
               meta_data.get_dict = {"get_param" => "get_value"}
               # post dict for appsensor meta data gets flatten before being enqueued
               meta_data.post_dict = TCellAgent::Utils::Params.flatten({"post_param" => "post_value"})
@@ -453,7 +465,13 @@ module TCellAgent
         context "with patches meta data" do
           context "with one param of each type" do
             it "should call check_param_for_injections once for each param" do
-              meta_data = TCellAgent::Patches::MetaData.new
+              meta_data = TCellAgent::Patches::MetaData.new(
+                "get",
+                "remote_address",
+                "route_id",
+                "session_id",
+                "user_id",
+                "transaction_id")
               meta_data.get_dict = {"get_param" => "get_value"}
               meta_data.post_dict = {"post_param" => "post_value"}
               meta_data.body_dict = TCellAgent::Utils::Params.flatten({"body_param" => "body_value"})