tcell_agent 0.2.18 → 0.2.19

data/lib/tcell_agent/rails/middleware/body_filter_middleware.rb

@@ -30,7 +30,7 @@ module TCellAgent
             if TCellAgent.configuration.should_intercept_requests?
               response_time = (Time.now.to_f * 1000).to_i - orig
               TCellAgent::Instrumentation.safe_block("Handling Route Time") {
-                route_id = env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].route_id
+                route_id = env[TCellAgent::Instrumentation::TCELL_ID].route_id
                 if route_id
                   TCellAgent.increment_route(route_id, response_time)
                 else
@@ -40,14 +40,14 @@ module TCellAgent
               TCellAgent::Instrumentation.safe_block("Handling Sessions Info") {
                 login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
                 if (login_fraud_policy && login_fraud_policy.session_hijacking_metrics)
-                  hmac_session_id = env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].hmac_session_id
-                  user_id = env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].user_id
+                  hmac_session_id = env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
+                  user_id = env[TCellAgent::Instrumentation::TCELL_ID].user_id
                   if user_id && hmac_session_id
                     TCellAgent.increment_session_info(
                       hmac_session_id,
                       user_id,
-                      env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].ip_address,
-                      env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].user_agent,
+                      env[TCellAgent::Instrumentation::TCELL_ID].ip_address,
+                      env[TCellAgent::Instrumentation::TCELL_ID].user_agent,
                     )
                   end
                 end

data/lib/tcell_agent/rails/middleware/context_middleware.rb

@@ -20,7 +20,6 @@ module TCellAgent
   module Instrumentation
     module Rails
       module Middleware
-        TCELL_ID = "tcell.request_data"
 
         class ContextMiddleware
           THREADS = {}
@@ -30,11 +29,14 @@ module TCellAgent
 
           def call(env)
             if TCellAgent.configuration.should_intercept_requests?
-              env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID] = TCellAgent::Instrumentation::TCellData.new
+              env[TCellAgent::Instrumentation::TCELL_ID] = TCellAgent::Instrumentation::TCellData.new
               TCellAgent::Instrumentation.safe_block("Setting transaction_id") {
-                env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].transaction_id = SecureRandom.uuid
+                env[TCellAgent::Instrumentation::TCELL_ID].transaction_id = SecureRandom.uuid
                 request = Rack::Request.new(env)
-                env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].uri = request.fullpath
+                env[TCellAgent::Instrumentation::TCELL_ID].uri = request.fullpath
+                if request.request_method
+                  env[TCellAgent::Instrumentation::TCELL_ID].request_method = request.request_method.downcase
+                end
               }
               env["filter_body_set"] = Set.new
               ContextMiddleware::THREADS[Thread.current.object_id] = env

data/lib/tcell_agent/rails/middleware/global_middleware.rb

@@ -30,24 +30,24 @@ module TCellAgent
               request = Rack::Request.new(env)
               TCellAgent::Instrumentation.safe_block("Setting session_id & user_id") {
                 if request.session
-                  env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].session_id =
+                  env[TCellAgent::Instrumentation::TCELL_ID].session_id =
                     request.session["session_id"]
-                  env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].user_id =
+                  env[TCellAgent::Instrumentation::TCELL_ID].user_id =
                     TCellAgent::UserInformation.getUserFromRequest(request)
                 end
               }
 
               TCellAgent::Instrumentation.safe_block("Setting hmac_session_id") {
                 hmac_key = TCellAgent::SensorEvents::Util.getHmacKey()
-                if request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].session_id
-                  env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].hmac_session_id =
-                    TCellAgent::SensorEvents::Util.hmac(request.env["tcell.request_data"].session_id, hmac_key)
+                if request.env[TCellAgent::Instrumentation::TCELL_ID].session_id
+                  env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id =
+                    TCellAgent::SensorEvents::Util.hmac(request.env[TCellAgent::Instrumentation::TCELL_ID].session_id, hmac_key)
                 end
               }
 
               TCellAgent::Instrumentation.safe_block("Setting and ip address user agent") {
-                env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].ip_address = request.ip
-                env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].user_agent = request.user_agent
+                env[TCellAgent::Instrumentation::TCELL_ID].ip_address = TCellAgent::Utils::Rails.better_ip(request)
+                env[TCellAgent::Instrumentation::TCELL_ID].user_agent = request.user_agent
               }
             end
 

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -33,7 +33,7 @@ module TCellAgent
               TCellAgent::Instrumentation.safe_block("Checking for blocked ips") do
                 patches_policy = TCellAgent.policy(TCellAgent::PolicyTypes::Patches)
                 if patches_policy
-                  if patches_policy.block_ip?(request)
+                  if patches_policy.block_ip?(TCellAgent::Utils::Rails.better_ip(request))
                     return [403, {"Content-Type" => "text/plain"}, ["Forbidden based on referer"]]
                   end
                 end
@@ -66,10 +66,10 @@ module TCellAgent
 
               if content_security_policy
                 content_security_policy.each(
-                  request.env["tcell.request_data"].transaction_id, 
-                  request.env["tcell.request_data"].route_id,
-                  request.env["tcell.request_data"].hmac_session_id, 
-                  request.env["tcell.request_data"].user_id) do | header_pair | 
+                  request.env[TCellAgent::Instrumentation::TCELL_ID].transaction_id,
+                  request.env[TCellAgent::Instrumentation::TCELL_ID].route_id,
+                  request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id,
+                  request.env[TCellAgent::Instrumentation::TCELL_ID].user_id) do | header_pair |
                   headers[header_pair["name"]] = header_pair["value"]
                 end
               end
@@ -85,9 +85,9 @@ module TCellAgent
 
               if clickjacking_policy
                   clickjacking_policy.each(
-                    request.env["tcell.request_data"].transaction_id, 
-                    request.env["tcell.request_data"].hmac_session_id, 
-                    request.env["tcell.request_data"].user_id) do | header_pair | 
+                    request.env[TCellAgent::Instrumentation::TCELL_ID].transaction_id,
+                    request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id,
+                    request.env[TCellAgent::Instrumentation::TCELL_ID].user_id) do | header_pair |
                   header_name = header_pair["name"]
                   header_value = header_pair["value"]
                   if (headers.has_key?header_name)
@@ -124,16 +124,16 @@ module TCellAgent
               http_redirect_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HttpRedirect)
               if http_redirect_policy && headers.has_key?("Location")
                 local_uri = URI.parse(request.url)
-                route_id = request.env["tcell.request_data"].route_id
-                session_id = request.env["tcell.request_data"].session_id
+                route_id = request.env[TCellAgent::Instrumentation::TCELL_ID].route_id
+                session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].session_id
                 new_location = http_redirect_policy.enforce(
-                  headers["Location"], 
-                  local_uri.host, 
-                  request.fullpath, 
+                  headers["Location"],
+                  local_uri.host,
+                  request.fullpath,
                   request.request_method,
                   route_id,
-                  status, 
-                  request.ip,
+                  status,
+                  TCellAgent::Utils::Rails.better_ip(request),
                   session_id)
                 # Enforcement
                 if (new_location)

data/lib/tcell_agent/rails/path_parameters_setter.rb

@@ -0,0 +1,43 @@
+module TCellAgent
+
+  ActionDispatch::Routing::RouteSet::Dispatcher.class_eval do
+    if (::Rails::VERSION::MAJOR == 3)
+      alias_method :tcell_dispatch, :dispatch
+      def dispatch(controller, action, env)
+        result = tcell_dispatch(controller, action, env)
+
+        TCellAgent::Instrumentation.safe_block("Set path_parameters in TCellData") do
+          if TCellAgent.configuration.should_intercept_requests?
+            request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+            tcell_data = request_env[TCellAgent::Instrumentation::TCELL_ID]
+            if tcell_data
+              tcell_data.path_parameters = env[ActionDispatch::Routing::RouteSet::PARAMETERS_KEY]
+            end
+          end
+        end
+
+        result
+      end
+    end
+
+    if (::Rails::VERSION::MAJOR == 4)
+      alias_method :tcell_serve, :serve
+      def serve(req)
+        result = tcell_serve(req)
+
+        TCellAgent::Instrumentation.safe_block("Set path_parameters in TCellData") do
+          if TCellAgent.configuration.should_intercept_requests?
+            request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+            tcell_data = request_env[TCellAgent::Instrumentation::TCELL_ID]
+            if tcell_data
+              tcell_data.path_parameters = req.path_parameters
+            end
+          end
+        end
+
+        result
+      end
+    end
+  end
+
+end

data/lib/tcell_agent/rails/routes.rb

@@ -26,7 +26,7 @@ module TCellAgent
       end
       def self._handle_dataexpsure_forms(request)
         dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-        tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+        tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
         if tcell_context && dataex_policy && dataex_policy.has_actions_for_form_parameter?
           for_params(request) { |method, param_name, param_value|
             actions = dataex_policy.get_actions_for_form_parameter(param_name, tcell_context.route_id)
@@ -43,7 +43,7 @@ module TCellAgent
       }
       def self._handle_dataexpsure_headers(request)
         dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-        tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+        tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
         if tcell_context && dataex_policy && dataex_policy.has_actions_for_headers?
           headers = request.env.select {|k,v| k.start_with? 'HTTP_'}
           headers.each { |header_name, header_value|
@@ -62,7 +62,7 @@ module TCellAgent
       }
       def self._handler_dataexposure_cookies(request)
         dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-        tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]               
+        tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
         if tcell_context && dataex_policy && dataex_policy.has_actions_for_cookie?
           request.cookies.each { |cookie_name, cookie_value|
             actions = dataex_policy.get_actions_for_cookie(cookie_name)
@@ -92,7 +92,7 @@ module TCellAgent
               route = Rails.application.routes.router.recognize(request) { |r, _| r }.first
               if route
                 route_path = route[2].path.spec
-                tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+                tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
                 if tcell_context
                   tcell_context.route_id = TCellAgent::SensorEvents::Util.calculateRouteId(request.method.downcase, route_path)
                 end

data/lib/tcell_agent/sensor_events/appsensor_meta_event.rb

@@ -20,10 +20,11 @@ module TCellAgent
         def build(request, rack_response, response_code, response_headers)
           meta_event = AppSensorMetaEvent.new
 
-          meta_event.remote_address = request.ip
+          meta_event.remote_address = TCellAgent::Utils::Rails.better_ip(request)
           meta_event.method = request.request_method
           meta_event.location = "#{request.base_url}#{request.fullpath}"
           meta_event.request_headers = request.env
+          meta_event.user_agent = request.env['HTTP_USER_AGENT']
           meta_event.request_content_len = (request.content_length || "0").to_i
           meta_event.response_content_len = (rack_response.length || "0").to_i
           meta_event.get_dict = request.GET
@@ -33,10 +34,11 @@ module TCellAgent
           meta_event.response_code = response_code
           meta_event.response_headers = response_headers
 
-          meta_event.route_id = request.env["tcell.request_data"].route_id
-          meta_event.transaction_id = request.env["tcell.request_data"].transaction_id
-          meta_event.session_id = request.env["tcell.request_data"].hmac_session_id
-          meta_event.user_id = request.env["tcell.request_data"].user_id
+          meta_event.path_parameters = request.env[TCellAgent::Instrumentation::TCELL_ID].path_parameters
+          meta_event.route_id = request.env[TCellAgent::Instrumentation::TCELL_ID].route_id
+          meta_event.transaction_id = request.env[TCellAgent::Instrumentation::TCELL_ID].transaction_id
+          meta_event.session_id = request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id
+          meta_event.user_id = request.env[TCellAgent::Instrumentation::TCELL_ID].user_id
 
           meta_event.set_body_dict(
             meta_event.request_content_len,
@@ -50,7 +52,8 @@ module TCellAgent
 
 
       attr_accessor :remote_address, :method, :location, :route_id, :session_id, :user_id, :transaction_id,
-        :request_content_len, :get_dict, :post_dict, :body_dict, :cookie_dict, :response_content_len, :response_code
+        :request_content_len, :get_dict, :post_dict, :body_dict, :cookie_dict, :response_content_len, :response_code,
+        :user_agent, :path_parameters
 
       attr_accessor :request_headers, :response_headers
 
@@ -62,6 +65,8 @@ module TCellAgent
         @get_dict = {}
         @post_dict = {}
         @cookie_dict = {}
+        @user_agent = nil
+        @path_parameters = {}
       end
 
       def set_body_dict(request_content_len, request_content_type, request_body)

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "0.2.18"
+  VERSION = "0.2.19"
 end

data/spec/lib/tcell_agent/policies/appsensor/database_sensor_spec.rb

@@ -0,0 +1,165 @@
+require 'spec_helper'
+
+module TCellAgent
+  module Policies
+
+    describe DatabaseSensor do
+
+      context "#initialize" do
+        context "default sensor" do
+          it "should have properties set to defaults" do
+            sensor = DatabaseSensor.new
+            expect(sensor.enabled).to eq(false)
+            expect(sensor.max_rows).to eq(1001)
+            expect(sensor.excluded_route_ids).to eq({})
+          end
+        end
+
+        context "setting enabled on sensor" do
+          it "should have properties set to defaults" do
+            sensor = DatabaseSensor.new({"enabled" => true})
+            expect(sensor.enabled).to eq(true)
+            expect(sensor.max_rows).to eq(1001)
+            expect(sensor.excluded_route_ids).to eq({})
+          end
+        end
+
+        context "setting max_rows on sensor" do
+          it "should have properties set to defaults" do
+            sensor = DatabaseSensor.new({"large_result" => {"limit" => 1}})
+            expect(sensor.enabled).to eq(false)
+            expect(sensor.max_rows).to eq(1)
+            expect(sensor.excluded_route_ids).to eq({})
+          end
+        end
+
+        context "setting excluded routes on sensor" do
+          it "should have properties set to defaults" do
+            sensor = DatabaseSensor.new({"exclude_routes" => ["1", "10", "20"]})
+            expect(sensor.enabled).to eq(false)
+            expect(sensor.max_rows).to eq(1001)
+            expect(sensor.excluded_route_ids).to eq({"1"=>true, "10"=>true, "20"=>true})
+          end
+        end
+      end
+
+      context "#check" do
+        context "with disabled sensor" do
+          it "should not send event" do
+            sensor = DatabaseSensor.new({"enabled" => false})
+
+            expect(TCellAgent).to_not receive(:send_event)
+            sensor.check({}, 10)
+          end
+        end
+
+        context "with enabled sensor" do
+          context "records is less than limit" do
+            it "should not send event" do
+              sensor = RequestSizeSensor.new({
+                "enabled" => true,
+                "large_result" => { "limit" => 10},
+                "exclude_routes" => []
+              })
+
+              tcell_data = TCellAgent::Instrumentation::TCellData.new
+
+              expect(TCellAgent).to_not receive(:send_event)
+              sensor.check(tcell_data, 1)
+            end
+          end
+
+          context "records are more than limit" do
+            it "should send event" do
+              sensor = DatabaseSensor.new({
+                "enabled" => true,
+                "large_result" => { "limit" => 10},
+                "exclude_routes" => []
+              })
+
+              tcell_data = TCellAgent::Instrumentation::TCellData.new
+              tcell_data.ip_address = "ip_address"
+              tcell_data.request_method = "get"
+              tcell_data.uri = "location"
+              tcell_data.route_id = "route_id"
+              tcell_data.session_id = "session_id"
+              tcell_data.user_id = "user_id"
+              tcell_data.transaction_id = "transaction_id"
+
+              expect(TCellAgent).to receive(:send_event).with(
+                {
+                  "event_type" => "as",
+                  "dp" => DatabaseSensor::DP_CODE,
+                  "param" => nil,
+                  "remote_addr" => "ip_address",
+                  "rou" => "route_id",
+                  "m" => "get"
+                }
+              )
+              sensor.check(tcell_data, 11)
+            end
+          end
+
+          context "records are more than limit" do
+            context "route_id is excluded" do
+              it "should not send event" do
+                sensor = DatabaseSensor.new({
+                  "enabled" => true,
+                  "large_result" => { "limit" => 10},
+                  "exclude_routes" => ["route_id"]
+                })
+
+                tcell_data = TCellAgent::Instrumentation::TCellData.new
+                tcell_data.ip_address = "ip_address"
+                tcell_data.request_method = "get"
+                tcell_data.uri = "location"
+                tcell_data.route_id = "route_id"
+                tcell_data.session_id = "session_id"
+                tcell_data.user_id = "user_id"
+                tcell_data.transaction_id = "transaction_id"
+
+                expect(TCellAgent).to_not receive(:send_event)
+
+                sensor.check(tcell_data, 11)
+              end
+            end
+
+            context "route is not excluded" do
+              it "should send event" do
+                sensor = DatabaseSensor.new({
+                  "enabled" => true,
+                  "large_result" => { "limit" => 10},
+                  "exclude_routes" => ["nonmatching_route_id"]
+                })
+
+                tcell_data = TCellAgent::Instrumentation::TCellData.new
+                tcell_data.ip_address = "ip_address"
+                tcell_data.request_method = "get"
+                tcell_data.uri = "location"
+                tcell_data.route_id = "route_id"
+                tcell_data.session_id = "session_id"
+                tcell_data.user_id = "user_id"
+                tcell_data.transaction_id = "transaction_id"
+
+                expect(TCellAgent).to receive(:send_event).with(
+                  {
+                    "event_type" => "as",
+                    "dp" => DatabaseSensor::DP_CODE,
+                    "param" => nil,
+                    "remote_addr" => "ip_address",
+                    "rou" => "route_id",
+                    "m" => "get"
+                  }
+                )
+                sensor.check(tcell_data, 11)
+              end
+            end
+          end
+
+        end
+      end
+
+    end
+
+  end
+end