tcell_agent 0.2.9 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffa24423fa841f2a9bdb3b19d44b1250ba2bc4ff
-  data.tar.gz: ae260430c31ff45f822cf49efd0735664a181a8c
+  metadata.gz: 4acf90271f7e52d11433efda8b0345eac7c10267
+  data.tar.gz: 9b409662d96c032d94aa4f4fc1dc02cea5cbd752
 SHA512:
-  metadata.gz: bfd43837abf1c1ee7297c09d6960f1bb37673ad0b3dd805b93980bed531f216e57e54384f532e74f3c1f5d435564b177925070d0ab24354fc68dfc06122f0f18
-  data.tar.gz: 0b745e26beb71868ede7f5703a7292462d07c830a3374a0e2a98a3ddedb655695a7b2320a0bbf222926bd3bbeba7dc8462621cd52d465e3295e25a69d25a66e6
+  metadata.gz: 41a77673225c1970dddfb9341db54b2a7834984f5d48eb2242125ffa4d32e1869bac719027496c2aa142f09c76a450a3cf0fd252784c8dbd5c710331068e0881
+  data.tar.gz: c6e6d7b2c23da1d4866181521a5c803589a3753873740e7bc31c9b994a41ac3982ee36283a0810d8098091c06218f52703e9363eef626edca0cad596e939067d

data/bin/tcell_agent

@@ -42,6 +42,11 @@ CONFIG_FILE = 'config/tcell_agent.config'
 
 global = OptionParser.new do |opts|
   opts.banner = "Usage: tcell_agent [options] [subcommand [options]]"
+  opts.on("--version", "Print version") do |v|
+    require 'tcell_agent/version'
+    puts "TCell.io Ruby Agent (Version #{TCellAgent::VERSION})"
+    Kernel.exit(1)
+  end
   opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
     options[:verbose] = v
   end

data/lib/tcell_agent/agent.rb

@@ -54,6 +54,7 @@ module TCellAgent
     attr_accessor :event_dispatch_monitor
 
     attr_accessor :stop_agent
+    attr_accessor :complete_policy_cache
 
 
     def initialize(start_pid=Process.pid)
@@ -90,6 +91,8 @@ module TCellAgent
     end
 
     def initialize_processor_variables
+      @complete_policy_cache = {}
+
       @metricsLock = Monitor.new
       @stop_agent = false
       @route_table = TCellAgent::Routes::RouteTable.new
@@ -98,8 +101,11 @@ module TCellAgent
       @event_dispatch_monitor = Monitor.new
       @mutex = Monitor.new
 
+
       @response_time_table = {}
       @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
+      @sessions_metrics_mutex = Monitor.new
+
       @dispatchEvents = []
       @eventQueue = BoundedQueue.new(200)
 

data/lib/tcell_agent/agent/event_processor.rb

@@ -58,6 +58,38 @@ module TCellAgent
       end
     end
 
+    def send_dispatch_events(tapi)
+        events_to_send = []
+        @event_dispatch_monitor.synchronize {
+          events_to_send = @dispatchEvents
+          @dispatchEvents = []
+        }
+        if (@response_time_table.size > 0)
+          metrics_event = TCellAgent::SensorEvents::MetricsEvent.new
+          metrics_event.set_route_count_table(@response_time_table)
+          events_to_send.push( metrics_event )
+          @mutex.synchronize do
+            @response_time_table = {}
+          end
+        end
+        if @sessions_metrics.has_sessions?
+          sessions_to_send = []
+          @sessions_metrics_mutex.synchronize do
+            sessions_to_send = @sessions_metrics
+            @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
+          end
+          events_to_send.push( sessions_to_send )
+        end
+        success = tapi.sendEventSet(events_to_send)
+        if ( success == false )
+          ensured_events = events_to_send.find_all{|item| item.ensure == true }
+          @event_dispatch_monitor.synchronize {
+            @dispatchEvents.push(*ensured_events)
+          }
+        end
+    end
+
+
     def start_event_processor(send_empties=true)
       return if TCellAgent.configuration.should_start_event_manager? == false
 
@@ -79,46 +111,11 @@ module TCellAgent
                 now = Time.now
                 wait_for = @dispatchEventsTimeout - (now - last_run_time).to_i.abs
                 event = @eventQueue.pop([wait_for, 1].max)
-
                 if event == nil
+                  if (@events_send_empties || @dispatchEvents.length > 0)
                     last_run_time = Time.now
-                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ 
-                      if (@events_send_empties || @dispatchEvents.length > 0)
-                        if (@response_time_table.size > 0)
-                          metrics_event = TCellAgent::SensorEvents::MetricsEvent.new
-                          metrics_event.set_route_count_table(@response_time_table)
-                          @event_dispatch_monitor.synchronize {
-                            @dispatchEvents.push( metrics_event )
-                          }
-                          @mutex.synchronize do
-                            @response_time_table = {}
-                          end
-                        end
-                        if @sessions_metrics.has_sessions?
-                          sessions_to_send = nil
-                          @mutex.synchronize do
-                            sessions_to_send = @sessions_metrics
-                            @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
-                          end
-                          @event_dispatch_monitor.synchronize {
-                            @dispatchEvents.push( sessions_to_send )
-                          }
-                        end
-                        @event_dispatch_monitor.synchronize {
-                          events_to_send = @dispatchEvents
-                          result = tapi.sendEventSet(events_to_send)
-                          if ( result )
-                            @event_dispatch_monitor.synchronize {
-                              @dispatchEvents = []
-                            }
-                          else
-                            @event_dispatch_monitor.synchronize {
-                              @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
-                            }
-                          end
-                        }
-                      end
-                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
+                    self.send_dispatch_events(tapi)
+                  end
                 else
                   event.post_process
                   if event.send == true
@@ -128,39 +125,7 @@ module TCellAgent
                   end
                   if (event.flush or @dispatchEvents.length >= @dispatchEventsLimit or wait_for < 0)
                     last_run_time = Time.now
-                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
-                      if (@events_send_empties || @dispatchEvents.length > 0)
-                        if (@response_time_table.size > 0)
-                          metrics_event = TCellAgent::SensorEvents::MetricsEvent.new
-                          metrics_event.set_route_count_table(@response_time_table)
-                          @event_dispatch_monitor.synchronize {
-                            @dispatchEvents.push( metrics_event )
-                          }
-                          @mutex.synchronize do
-                            @response_time_table = {}
-                          end
-                        end
-                        if @sessions_metrics.has_sessions?
-                          sessions_to_send = nil
-                          @mutex.synchronize do
-                            sessions_to_send = @sessions_metrics
-                            @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
-                          end
-                          @event_dispatch_monitor.synchronize {
-                            @dispatchEvents.push( sessions_to_send )
-                          }
-                        end
-                        @event_dispatch_monitor.synchronize {
-                          events_to_send = @dispatchEvents
-                          result = tapi.sendEventSet(events_to_send)
-                          if ( result )
-                            @dispatchEvents = []
-                          else
-                            @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
-                          end
-                        }
-                      end
-                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
+                    self.send_dispatch_events(tapi)
                   end
                 end
               rescue ThreadError
@@ -169,7 +134,6 @@ module TCellAgent
                   @dispatchEvents = []
                 }
               end
-
             rescue Exception => e
               last_run_time = Time.now
               TCellAgent.logger.error("Exception while processing events: #{e.message}")
@@ -333,7 +297,7 @@ module TCellAgent
           return
         end
 
-        @mutex.synchronize do
+        @sessions_metrics_mutex.synchronize do
           @sessions_metrics.add_session_info(hmac_session_id, user_id, ip_address, user_agent)
         end
 
@@ -354,7 +318,6 @@ module TCellAgent
           })
           return
         end
-
         @mutex.synchronize do
           if (route_id == nil || route_id == "")
             route_id = "?"

data/lib/tcell_agent/agent/policy_manager.rb

@@ -127,23 +127,32 @@ module TCellAgent
         policy_cache = {}
         existing_policy = f1.read
 
-        if !existing_policy.nil? && existing_policy != ""
-          policy_jsons = JSON.parse(existing_policy)
-          if policy_jsons
-            if policy_jsons.key?("result")
-              policy_cache = policy_jsons["result"]
-            else
-              policy_cache = policy_jsons
+        begin
+          if !existing_policy.nil? && existing_policy != ""
+            policy_jsons = JSON.parse(existing_policy)
+            if policy_jsons
+              if policy_jsons.key?("result")
+                policy_cache = policy_jsons["result"]
+              else
+                policy_cache = policy_jsons
+              end
             end
           end
+          policy_cache[policy_name] = policy
+          @complete_policy_cache = policy_cache
+        rescue Exception => e
+          TCellAgent.logger.warn(e.message)
+          if @complete_policy_cache
+            policy_cache = @complete_policy_cache
+          end
         end
-        policy_cache[policy_name] = policy
 
         f1.rewind
-        f1.puts( JSON.dump(policy_cache) )
-
+        f1.write( JSON.dump(policy_cache) )
+        f1.flush
+        f1.truncate(f1.pos)
       rescue Exception => e
-        TCellAgent.logger.error(e.message)
+        TCellAgent.logger.warn(e.message)
 
       ensure
         f1.close unless f1.nil?
@@ -169,10 +178,11 @@ module TCellAgent
         if policy_jsons.key?("result")
           return policy_jsons["result"]
         end
+        @complete_policy_cache = policy_jsons
         return policy_jsons
 
       rescue Exception => e
-        TCellAgent.logger.error(e.message)
+        TCellAgent.logger.warn(e.message)
       end
 
       return nil

data/lib/tcell_agent/api.rb

@@ -64,10 +64,13 @@ module TCellAgent
         raise "Config Information Not Found, can't send events"
       end
       current_time = DateTime.now.to_time.to_i
-      if (events)
-        events.each { |event| event.calculateOffset(current_time) }
-      else
-        events = []
+      #if (events)
+      #  events.each { |event| event.calculateOffset(current_time) }
+      #else
+      #  events = []
+      #end
+      if (events == nil)
+        return false
       end
       eventset = { "uuid"=>TCellAgent.configuration.uuid,
                    "hostname"=>TCellAgent.configuration.host_identifier,

data/lib/tcell_agent/configuration.rb

@@ -33,7 +33,8 @@ module TCellAgent
       :raise_exceptions,
       :allow_unencrypted_appsensor_payloads
 
-    attr_accessor :enabled,
+    attr_accessor :disable_all,
+      :enabled,
       :enable_event_manager,       # false = Do not start the even manager
       :enable_event_consumer,      # false = Do not consume events, drop them
       :enable_policy_polling,      # false = Do not poll for policies
@@ -67,11 +68,10 @@ module TCellAgent
       @exp_config_settings = true
       @demomode = false
 
-      @allow_unencrypted_appsensor_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"])
-
       @fetch_policies_from_tcell = true
       @instrument_for_events = true
 
+      @disable_all = false
       @enabled = true
       @enable_event_manager = true
       @enable_event_consumer = true
@@ -90,6 +90,15 @@ module TCellAgent
       read_config_using_env
       read_config_from_file(filename)
 
+      # Because ENV can override this one
+      env_unencrypted_firewall = 
+      if (ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"] != nil)
+        @allow_unencrypted_appsensor_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"])
+      end
+      if (ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"] != nil)
+        @allow_unencrypted_appsensor_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"])
+      end
+
       @tcell_api_url ||= "https://api.tcell.io/api/v1"
       @tcell_input_url ||= "https://input.tcell.io/api/v1"
       @js_agent_api_base_url ||= nil
@@ -155,6 +164,7 @@ module TCellAgent
             # Optional
             @preload_policy_filename = app_data.fetch("preload_policy_filename", nil)
 
+            @disable_all = app_data.fetch("disable_all", @disable_all)
             @enabled = app_data.fetch("enabled", @enabled)
             @enable_event_manager = app_data.fetch("enable_event_manager", @enable_event_manager)
             @enable_event_consumer = app_data.fetch("enable_event_consumer", @enable_event_consumer)
@@ -176,6 +186,10 @@ module TCellAgent
 
             @use_websockets = app_data["use_websockets"]
 
+            @allow_unencrypted_appsensor_payloads = app_data.fetch('allow_unencrypted_appsensor_payloads', @allow_unencrypted_appsensor_payloads) 
+            @allow_unencrypted_appsensor_payloads = app_data.fetch('allow_unencrypted_appfirewall_payloads', @allow_unencrypted_appsensor_payloads) 
+
+
             @host_identifier = @host_identifier || app_data.fetch("host_identifier", @host_identifier)
             if (@host_identifier == nil)
               @host_identifier = (Socket.gethostname() || "localhost")

data/lib/tcell_agent/instrumentation.rb

@@ -122,7 +122,8 @@ module TCellAgent
                 TCellAgent::SensorEvents::DlpEvent.new(
                   self.route_id, 
                   self.uri, 
-                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY,
+                  session_id_actions.action_id
                   ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
               )
             end
@@ -166,7 +167,8 @@ module TCellAgent
                 TCellAgent::SensorEvents::DlpEvent.new(
                   self.route_id, 
                   self.uri, 
-                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG,
+                  session_id_actions.action_id
                   ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
               )
             end

data/lib/tcell_agent/policies/dataloss_policy.rb

@@ -205,10 +205,12 @@ module TCellAgent
                         data_discovery_json = data_json["data_discovery"]
                         policy.database_discovery_enabled = data_discovery_json.fetch('database_enabled', false)
                     end
-                    if data_json.has_key?("session_id_protection")
-                        session_id_protection = data_json["session_id_protection"]
+                    if data_json.has_key?("session_id_protections")
+                        session_id_protection = data_json["session_id_protections"]
+                        rule_id = session_id_protection.fetch("id",nil)
                         filter_actions = DataLossPolicy.actions_from_json(session_id_protection)
                         if filter_actions != nil
+                            filter_actions.action_id = rule_id
                             policy.session_id_filter_actions = filter_actions
                         end
                     end
@@ -217,6 +219,7 @@ module TCellAgent
                             context = protection.fetch('variable_context', nil)
                             variables = protection.fetch('variables', nil)
                             scope = protection.fetch('scope', "global")
+                            rule_id = protection.fetch("id",nil)
                             options = protection.fetch('actions', nil)
                             route_ids = []
                             if (scope == "global")
@@ -229,6 +232,7 @@ module TCellAgent
                             if context && policy.request_filter_actions.has_key?(context) && variables && options
                                 filter_actions = DataLossPolicy.actions_from_json(options)
                                 if filter_actions != nil
+                                    filter_actions.action_id = rule_id
                                     variables.each do |variable|
                                         route_ids.each  do |route_id|
                                             if (context == RequestProtectionManager::COOKIE)
@@ -252,7 +256,7 @@ module TCellAgent
                                 _schemas = protection_json.fetch("schemas",["*"])
                                 _tables = protection_json.fetch("tables",["*"])
                                 _fields = protection_json.fetch("fields",nil)
-                                rule_id = protection_json.fetch("rule_id",nil)
+                                rule_id = protection_json.fetch("id",nil)
                                 actions = protection_json.fetch("actions",{})
                                 filter_actions = DataLossPolicy.actions_from_json(actions)
                                 _route_ids = ["*"]

data/lib/tcell_agent/rails/auth/devise.rb

@@ -66,22 +66,6 @@ module TCellAgent
                 TCellAgent.send_event(event)
               end
             end
-            appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
-            if (appsensor_policy && appsensor_policy.enabled && appsensor_policy.option_enabled?("login_failure"))
-              hmac_session_id = request.env["tcell.request_data"].hmac_session_id
-              event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
-                request.fullpath,
-                TCellAgent::Policies::AppSensorPolicy::DP_LOGIN_FAILURE,
-                request.request_method,
-                request.remote_ip,
-                tcell_username,
-                request.env["tcell.request_data"].route_id,
-                data=nil,
-                transaction_id=nil,
-                session_id=hmac_session_id,
-                user_id=nil)
-              TCellAgent.send_event(event)
-            end
           }
         end
       end

data/lib/tcell_agent/rails/dlp.rb

@@ -203,7 +203,6 @@ module TCellAgent
       around_filter :global_request_logging
       def global_request_logging
         begin
-
           yield
 
           if TCellAgent.configuration.enabled &&

data/lib/tcell_agent/rails/routes.rb

@@ -1,6 +1,85 @@
 require 'tcell_agent/configuration'
 
 module TCellAgent
+  module AroundFilters
+    def self.handle_request_dlp_parameters(request)
+      def self.loop_params_hash(method, param_hash, prefix, &block)
+        param_hash.each do |param_name, param_value|
+          if param_value && param_value.is_a?(Hash)
+            loop_params_hash(method, param_value, 'hash', &block)
+          elsif !param_value || !param_value.instance_of?(String) || param_value == ""
+            next
+          else
+            block.call(method, param_name, param_value)
+          end
+        end
+      end
+      def self.for_params(request, &block)
+        get_params = request.GET
+        if get_params
+          self.loop_params_hash('get', get_params, nil, &block)
+        end
+        post_params = request.POST
+        if post_params
+          self.loop_params_hash('post', post_params, nil, &block)
+        end
+      end
+      def self._handle_dataexpsure_forms(request)
+        dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+        tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+        if tcell_context && dataex_policy && dataex_policy.has_actions_for_form_parameter?
+          for_params(request) { |method, param_name, param_value|
+            actions = dataex_policy.get_actions_for_form_parameter(param_name, tcell_context.route_id)
+            if actions
+              actions.each { |action|
+                tcell_context.add_filter_for_request_parameter(param_value, action, param_name)
+              }
+            end
+          }
+        end
+      end
+      TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request forms)") {
+        _handle_dataexpsure_forms(request)
+      }
+      def self._handle_dataexpsure_headers(request)
+        dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+        tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+        if tcell_context && dataex_policy && dataex_policy.has_actions_for_headers?
+          headers = request.env.select {|k,v| k.start_with? 'HTTP_'}
+          headers.each { |header_name, header_value|
+            header_name = header_name.sub(/^HTTP_/, '').gsub('_','-')
+            actions = dataex_policy.get_actions_for_header(header_name)
+            if actions
+              actions.each { |action|
+                tcell_context.add_filter_for_header_value(header_value, action, header_name)
+              }
+            end
+          }
+        end
+      end
+      TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request headers)") {
+        _handle_dataexpsure_headers(request)
+      }
+      def self._handler_dataexposure_cookies(request)
+        dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+        tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]               
+        if tcell_context && dataex_policy && dataex_policy.has_actions_for_cookie?
+          request.cookies.each { |cookie_name, cookie_value|
+            actions = dataex_policy.get_actions_for_cookie(cookie_name)
+            if actions
+              actions.each { |action|
+                tcell_context.add_filter_for_cookie_value(cookie_value, action, cookie_name)
+              }
+            end
+          }
+        end
+      end
+      TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request cookies)") {
+        _handler_dataexposure_cookies(request)
+      }
+    end
+  end
+
   ActiveSupport.on_load(:action_controller) do
     ActionController::Base.class_eval do
 
@@ -19,82 +98,8 @@ module TCellAgent
                 end
               end
             }
-            def loop_params_hash(method, param_hash, prefix, &block)
-              param_hash.each do |param_name, param_value|
-                if param_value && param_value.is_a?(Hash)
-                  loop_params_hash(method, param_value, 'hash', &block)
-                elsif !param_value || !param_value.instance_of?(String) || param_value == ""
-                  next
-                else
-                  block.call(method, param_name, param_value)
-                end
-              end
-            end
-            def for_params(request, &block)
-              get_params = request.GET
-              if get_params
-                self.loop_params_hash('get', get_params, nil, &block)
-              end
-              post_params = request.POST
-              if post_params
-                self.loop_params_hash('post', post_params, nil, &block)
-              end
-            end
-            def _handle_dataexpsure_forms(request)
-              dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-              tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
-              if tcell_context && dataex_policy && dataex_policy.has_actions_for_form_parameter?
-                for_params(request) { |method, param_name, param_value|
-                  actions = dataex_policy.get_actions_for_request("form",param_name)
-                  if actions
-                    actions.each { |action|
-                      tcell_context.add_filter_for_request_parameter(param_value, action, param_name)
-                    }
-                  end
-                }
-              end
-            end
-            TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request forms)") {
-              _handle_dataexpsure_forms(request)
-            }
-            def _handle_dataexpsure_headers(request)
-              dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-              tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
-              if tcell_context && dataex_policy && dataex_policy.has_actions_for_headers?
-                headers = request.env.select {|k,v| k.start_with? 'HTTP_'}
-                headers.each { |header_name, header_value|
-                  header_name = header_name.sub(/^HTTP_/, '').gsub('_','-')
-                  actions = dataex_policy.get_actions_for_header(header_name)
-                  if actions
-                    actions.each { |action|
-                      tcell_context.add_filter_for_header_value(header_value, action, header_name)
-                    }
-                  end
-                }
-              end
-            end
-            TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request headers)") {
-              _handle_dataexpsure_headers(request)
-            }
-            def _handler_dataexposure_cookies(request)
-              dataex_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
-              tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]               
-              if tcell_context && dataex_policy && dataex_policy.has_actions_for_cookie?
-                request.cookies.each { |cookie_name, cookie_value|
-                  actions = dataex_policy.get_actions_for_cookie(cookie_name)
-                  if actions
-                    actions.each { |action|
-                      tcell_context.add_filter_for_cookie_value(cookie_value, action, cookie_name)
-                    }
-                  end
-                }
-              end
-            end
-            TCellAgent::Instrumentation.safe_block("Handling Dataexposure (request cookies)") {
-              _handler_dataexposure_cookies(request)
-            }
+            TCellAgent::AroundFilters.handle_request_dlp_parameters(request)
           end
-
           yield
         end
       end

data/lib/tcell_agent/sensor_events/dlp.rb

@@ -30,7 +30,7 @@ module TCellAgent
                     self["uid"] = user_id
                 end
                 if id
-                    self["id"] = id
+                    self["rule"] = id
                 end
             end
             def for_database(database, schema, table, field)

data/lib/tcell_agent/sensor_events/metrics.rb

@@ -54,7 +54,6 @@ module TCellAgent
               @object_counter.add_object
               ips.push(ip_address)
             end
-
           else
             @object_counter.add_object
             @user_agents[truncated_agent] = [truncated_agent, [ip_address]]
@@ -73,7 +72,6 @@ module TCellAgent
           if @user_ids.has_key?(user_id)
             user_id_info = @user_ids[user_id]
             user_id_info.add_user_agent_ip(truncated_agent, ip_address)
-
           else
             @object_counter.add_object
 

data/lib/tcell_agent/start_background_thread.rb

@@ -1,82 +1,83 @@
 # See the file "LICENSE" for the full license governing this code.
-
-require 'tcell_agent/logger'
-require 'tcell_agent/agent'
 require 'tcell_agent/configuration'
-require 'thread'
 
-module TCellAgent
-  #require 'tcell_agent/sinatra' if defined?(Sinatra)
-  require 'tcell_agent/rails' if defined?(Rails)
+if (TCellAgent.configuration.disable_all == false)
+  require 'tcell_agent/logger'
+  require 'tcell_agent/agent'
+  require 'thread'
+  module TCellAgent
+    #require 'tcell_agent/sinatra' if defined?(Sinatra)
+    require 'tcell_agent/rails' if defined?(Rails)
 
-  def self.run_instrumentation(server_name)
+    def self.run_instrumentation(server_name)
 
-    require 'tcell_agent/rails/on_start' if defined?(Rails)
+      require 'tcell_agent/rails/on_start' if defined?(Rails)
 
-    begin
-      TCellAgent.logger.debug("Instrumenting: #{server_name}")
-      TCellAgent.thread_agent.start
-    rescue Exception => e
-      TCellAgent.logger.error("Could not start thread agent. #{e.message}")
-    end
+      begin
+        TCellAgent.logger.debug("Instrumenting: #{server_name}")
+        TCellAgent.thread_agent.start
+      rescue Exception => e
+        TCellAgent.logger.error("Could not start thread agent. #{e.message}")
+      end
 
-    if TCellAgent.configuration.should_instrument?
-      Thread.abort_on_exception = TCellAgent.configuration.raise_exceptions
-      Thread.new do
+      if TCellAgent.configuration.should_instrument?
+        Thread.abort_on_exception = TCellAgent.configuration.raise_exceptions
+        Thread.new do
 
-        TCellAgent::Instrumentation.safe_block("Instrumenting Agent Details") do
-          event = TCellAgent::SensorEvents::ServerAgentDetailsSensorEvent.new
-          TCellAgent.send_event(event)
-        end
+          TCellAgent::Instrumentation.safe_block("Instrumenting Agent Details") do
+            event = TCellAgent::SensorEvents::ServerAgentDetailsSensorEvent.new
+            TCellAgent.send_event(event)
+          end
 
-        TCellAgent::Instrumentation.safe_block("Instrumenting Server Packages") do
-          event = TCellAgent::SensorEvents::ServerAgentPackagesSensorEvent.new
-          TCellAgent.send_event(event)
-        end
+          TCellAgent::Instrumentation.safe_block("Instrumenting Server Packages") do
+            event = TCellAgent::SensorEvents::ServerAgentPackagesSensorEvent.new
+            TCellAgent.send_event(event)
+          end
 
-        if defined?(Rails)
-          TCellAgent::Instrumentation.safe_block("Instrumenting routes") do
-            TCellAgent::Instrumentation::Rails.instrument_routes
+          if defined?(Rails)
+            TCellAgent::Instrumentation.safe_block("Instrumenting routes") do
+              TCellAgent::Instrumentation::Rails.instrument_routes
+            end
           end
         end
+
       end
 
     end
 
   end
 
-end
-
-tcell_server = ENV["TCELL_AGENT_SERVER"]
+  tcell_server = ENV["TCELL_AGENT_SERVER"]
 
-if TCellAgent.configuration.should_instrument?
-  if (!(tcell_server && tcell_server == "mock"))
+  if TCellAgent.configuration.should_instrument?
+    if (!(tcell_server && tcell_server == "mock"))
 
-    if (tcell_server && tcell_server == "webrick") || defined?(Rails::Server)
-      require("tcell_agent/servers/rails_server")
+      if (tcell_server && tcell_server == "webrick") || defined?(Rails::Server)
+        require("tcell_agent/servers/rails_server")
 
-    elsif (tcell_server && tcell_server == "thin") || defined?(Thin)
-      require("tcell_agent/servers/thin")
+      elsif (tcell_server && tcell_server == "thin") || defined?(Thin)
+        require("tcell_agent/servers/thin")
 
-    elsif (tcell_server && tcell_server == "puma") || defined?(Puma)
-      require("tcell_agent/servers/puma")
+      elsif (tcell_server && tcell_server == "puma") || defined?(Puma)
+        require("tcell_agent/servers/puma")
 
-    elsif (tcell_server && tcell_server == "unicorn") || defined?(Unicorn)
-      require("tcell_agent/servers/unicorn")
+      elsif (tcell_server && tcell_server == "unicorn") || defined?(Unicorn)
+        require("tcell_agent/servers/unicorn")
 
-    else
-      puts "[tCell.io] **********************************************************************"
-      puts "[tCell.io] Server used to launch rails not recognized."
-      puts "[tCell.io] You can override this with the env variable"
-      puts "[tCell.io]       TCELL_AGENT_SERVER=thin|puma|unicorn"
-      puts "[tCell.io] **********************************************************************"
+      else
+        puts "[tCell.io] **********************************************************************"
+        puts "[tCell.io] Server used to launch rails not recognized."
+        puts "[tCell.io] You can override this with the env variable"
+        puts "[tCell.io]       TCELL_AGENT_SERVER=thin|puma|unicorn"
+        puts "[tCell.io] **********************************************************************"
+      end
     end
-  end
 
-else
+  else
 
-  # unicorn is always instrumented to support rolling restarts
-  if (tcell_server && tcell_server == "unicorn") || defined?(Unicorn)
-    require("tcell_agent/servers/unicorn")
+    # unicorn is always instrumented to support rolling restarts
+    if (tcell_server && tcell_server == "unicorn") || defined?(Unicorn)
+      require("tcell_agent/servers/unicorn")
+    end
   end
-end
+end

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "0.2.9"
+  VERSION = "0.2.10"
 end

data/spec/lib/tcell_agent/agent/policy_manager_spec.rb

@@ -52,7 +52,7 @@ module TCellAgent
 
                 logger = double("logger")
                 expect(TCellAgent).to receive(:logger).and_return(logger)
-                expect(logger).to receive(:error).with("execution expired")
+                expect(logger).to receive(:warn).with("execution expired")
 
                 TCellAgent.thread_agent.cache(
                   "http-redirect",
@@ -141,7 +141,7 @@ module TCellAgent
 
               logger = double("logger")
               expect(TCellAgent).to receive(:logger).and_return(logger)
-              expect(logger).to receive(:error).with("A JSON text must at least contain two octets!")
+              expect(logger).to receive(:warn).with("A JSON text must at least contain two octets!")
 
               expect_any_instance_of(TCellAgent::Agent).to_not receive(:processPolicyJson)
 
@@ -163,7 +163,7 @@ module TCellAgent
 
                 logger = double("logger")
                 expect(TCellAgent).to receive(:logger).and_return(logger)
-                expect(logger).to receive(:error).with("757: unexpected token at 'bad_json'")
+                expect(logger).to receive(:warn).with("757: unexpected token at 'bad_json'")
                 expect_any_instance_of(TCellAgent::Agent).to_not receive(:processPolicyJson)
 
                 agent = TCellAgent::Agent.new(Process.pid)

data/spec/lib/tcell_agent/instrumentation_spec.rb

@@ -17,7 +17,7 @@ module TCellAgent
           policy_json_two = {
             "policy_id"=>"x1a1",
             "data"=>{
-              "session_id_protection"=>{"body"=>["redact"], "log"=>["event"]}
+              "session_id_protections"=>{"body"=>["redact"], "log"=>["event"]}
             }
           }
           session_id_policy = TCellAgent::Policies::DataLossPolicy.fromJson(policy_json_two)
@@ -42,7 +42,7 @@ module TCellAgent
           policy_json_two = {
             "policy_id"=>"x1a1",
             "data"=>{
-              "session_id_protection"=>{"body"=>["event"], "log"=>["redact"]}
+              "session_id_protections"=>{"body"=>["event"], "log"=>["redact"]}
             }
           }
           session_id_policy = TCellAgent::Policies::DataLossPolicy.fromJson(policy_json_two)
@@ -69,7 +69,7 @@ module TCellAgent
           policy_json_two = {
             "policy_id"=>"x1a1",
             "data"=>{
-              "session_id_protection"=>{"body"=>["redact"], "log"=>["redact"]}
+              "session_id_protections"=>{"body"=>["redact"], "log"=>["redact"]}
             }
           }
           session_id_policy = TCellAgent::Policies::DataLossPolicy.fromJson(policy_json_two)
@@ -94,7 +94,7 @@ module TCellAgent
           policy_json_two = {
             "policy_id"=>"x1a1",
             "data"=>{
-              "session_id_protection"=>{"body"=>["redact"], "log"=>["event"]}
+              "session_id_protections"=>{"body"=>["redact"], "log"=>["event"]}
             }
           }
           session_id_policy = TCellAgent::Policies::DataLossPolicy.fromJson(policy_json_two)

data/spec/lib/tcell_agent/policies/dataloss_policy_spec.rb

@@ -27,7 +27,7 @@ module TCellAgent
       policy_json_two = {
         "policy_id"=>"x1a1",
         "data"=>{
-          "session_id_protection"=>{"body"=>["redact"], "log"=>["event"]}
+          "session_id_protections"=>{"body"=>["redact"], "log"=>["event"]}
         }
       }
       policy_two = DataLossPolicy.fromJson(policy_json_two)

data/spec/lib/tcell_agent/rails/middleware/dlp_middleware_spec.rb

@@ -0,0 +1,170 @@
+require 'spec_helper'
+require 'rack/test'
+require 'rack'
+
+module TCellAgent
+  module Instrumentation
+    module Rails
+      module Middleware
+
+
+        class MockDLPRackApp
+
+          attr_reader :request_body
+
+          def initialize(body="OK", route_id=nil, session_id=nil)
+            @route_id = route_id
+            @session_id = session_id
+            @request_headers = {}
+            @body = body
+          end
+
+          def loop_params_hash(method, param_hash, prefix, &block)
+            param_hash.each do |param_name, param_value|
+              if param_value && param_value.is_a?(Hash)
+                loop_params_hash(method, param_value, 'hash', &block)
+              elsif !param_value || !param_value.instance_of?(String) || param_value == ""
+                next
+              else
+                block.call(method, param_name, param_value)
+              end
+            end
+          end
+
+          def for_params(request, &block)
+            get_params = request.GET
+            if get_params
+              self.loop_params_hash('get', get_params, nil, &block)
+            end
+            post_params = request.POST
+            if post_params
+              self.loop_params_hash('post', post_params, nil, &block)
+            end
+          end   
+
+          def call(env)
+            @env = env
+            rack_request = Rack::Request.new(env)
+            response_headers = {'Content-Type' => 'text/html'}
+            env["tcell.request_data"].transaction_id = "a-b-c-d-e-f"
+            env["tcell.request_data"].session_id = @session_id
+            env["tcell.request_data"].route_id = @route_id
+            tcell_context = env["tcell.request_data"]
+            dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+            if dlp_policy
+              action_objs = dlp_policy.get_actions_for_table("*", "*", "tablex", "columnb", tcell_context.route_id)
+              if action_objs
+                action_objs.each do |action_obj|
+                  tcell_context.add_response_db_filter("secretvalue", action_obj, "databx", "*", "tablex", "columnb")
+                end
+              end
+              TCellAgent::AroundFilters.handle_request_dlp_parameters(rack_request)
+              #if tcell_context && dlp_policy && dlp_policy.has_actions_for_form_parameter?
+              #  for_params(rack_request) { |method, param_name, param_value|
+              #    actions = dlp_policy.get_actions_for_form_parameter(param_name, tcell_context.route_id)
+              #    if actions
+              #      actions.each { |action|
+              #        puts action.action_id
+              #        tcell_context.add_filter_for_request_parameter(param_value, action, param_name)
+              #      }
+              #    end
+              #  }
+              #end
+            end
+            tcell_context.filter_body(@body)
+            [200, response_headers, [@body]]
+          end
+
+          def [](key)
+            @env[key]
+          end
+
+        end
+
+        describe HeadersMiddleware do
+
+          let(:app) { MockDLPRackApp.new }
+          let(:app2) { MockDLPRackApp.new(body="My secretvalue othervalue test", route_id="myrouteid", session_id="plainsessionid") }
+
+          subject { withTCellMiddleware( app ) }
+
+          context "DLP Middleware" do
+            before(:each) do
+              TCellAgent.configuration = TCellAgent::Configuration.new
+              TCellAgent.configuration.read_config_from_file(get_test_resource_path("normal_config.json"))
+            end
+            let(:request) { Rack::MockRequest.new(subject) }
+            let(:request2) { Rack::MockRequest.new( withTCellMiddleware( app2 )) }
+            let(:agent) { ::TCellAgent::Agent.new }
+            context "Event" do
+              before(:each) do
+                TCellAgent.thread_agent.processPolicyJson({"dlp" => {
+                    "policy_id"=>"x1a1",
+                    "data"=>{
+                      "db_protections"=>[
+                        {
+                          "scope"=>"route",
+                          "route_ids"=>["myrouteid"],
+                          "databases"=>["*"],
+                          "schemas"=>["*"],
+                          "tables"=>["tablex"],
+                          "fields"=>["columnb"],
+                          "id"=>"323213",
+                          "actions"=>{
+                            "log"=>["redact"],
+                            "body"=>["redact"]
+                          }
+                        }
+                      ]
+                    }
+                  }}, cache=false) 
+                TCellAgent.empty_event_queue
+              end
+              it "redacts body" do
+                response = request2.get("/some/path2?x=abc", 'CONTENT_TYPE' => 'text/html', 'REMOTE_ADDR' => '1.3.3.4,3.4.5.6')
+                expect(response.body).to eq("My [redacted] othervalue test")
+                #expect(response['Location']).to eq("https://www.google.com")
+                expected_as = {"event_type" => "dlp", "rid" => "myrouteid", "found_in" => "body", "rule" => "323213", "type" => "db", "db" => "databx", "schema" => "*", "table" => "tablex", "field" => "columnb", "uri" => "/some/path2?x="}
+                expect(TCellAgent.event_queue).to include(expected_as)
+              end
+            end #/conext
+
+
+            context "Event for request dlp" do
+              before(:each) do
+                TCellAgent.thread_agent.processPolicyJson({"dlp" => {
+                  "policy_id"=>"x1a1",
+                  "data"=>{
+                    "request_protections"=>[
+                      {
+                        "variable_context"=>"form",
+                        "scope"=>"route",
+                        "route_ids"=>["myrouteid"],
+                        "variables"=>["test333"],
+                        "id"=>"08080808",
+                        "actions"=>{
+                          "log"=>["redact"],
+                          "body"=>["event"]
+                        }
+                      }
+                    ]
+                  }
+                  }}, cache=false) 
+                TCellAgent.empty_event_queue
+              end
+              it "redacts body" do
+                response = request2.get("/some/path2?test333=othervalue", 'CONTENT_TYPE' => 'text/html', 'REMOTE_ADDR' => '1.3.3.4,3.4.5.6')
+                expect(response.body).to eq("My secretvalue othervalue test")
+                expected_as = {"event_type" => "dlp", "rid" => "myrouteid", "found_in" => "body", "rule" => "08080808", "type" => "req", "context" => "form", "variable" => "test333", "uri" => "/some/path2?test333="}
+                expect(TCellAgent.event_queue).to include(expected_as)
+              end
+            end #/conext
+            
+          end #/context
+        end #/describe
+
+
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -14,6 +14,7 @@ def get_test_resource_path(name)
 end
 
 require 'tcell_agent/agent'
+require 'tcell_agent/rails/routes'
 
 if TCellAgent.configuration.raise_exceptions
   puts "[tCell.io] ******WARNING*************WARNING**************WARNING****************"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tcell_agent
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.2.10
 platform: ruby
 authors:
 - Garrett
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-01 00:00:00.000000000 Z
+date: 2016-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -246,6 +246,7 @@ files:
 - spec/lib/tcell_agent/policies/secure_headers_policy_spec.rb
 - spec/lib/tcell_agent/rails/logger_spec.rb
 - spec/lib/tcell_agent/rails/middleware/appsensor_middleware_spec.rb
+- spec/lib/tcell_agent/rails/middleware/dlp_middleware_spec.rb
 - spec/lib/tcell_agent/rails/middleware/global_middleware_spec.rb
 - spec/lib/tcell_agent/rails/middleware/redirect_middleware_spec.rb
 - spec/lib/tcell_agent/rails_spec.rb
@@ -349,6 +350,7 @@ test_files:
 - spec/lib/tcell_agent/policies/secure_headers_policy_spec.rb
 - spec/lib/tcell_agent/rails/logger_spec.rb
 - spec/lib/tcell_agent/rails/middleware/appsensor_middleware_spec.rb
+- spec/lib/tcell_agent/rails/middleware/dlp_middleware_spec.rb
 - spec/lib/tcell_agent/rails/middleware/global_middleware_spec.rb
 - spec/lib/tcell_agent/rails/middleware/redirect_middleware_spec.rb
 - spec/lib/tcell_agent/rails_spec.rb