tcell_agent 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3e9ec10853e07ac321bca036627e9e98a9e8545f
-  data.tar.gz: b1dce50c2950629b10397802e5a6f16fa1beabc1
+  metadata.gz: 8a2e9af45884f3f879ebdac3c3df5c1443704b46
+  data.tar.gz: 4ebb0cf3c1dc1082581b098d60d86a1ab629dd2e
 SHA512:
-  metadata.gz: 1506c43edb7ae723f675b4cb1e8e5d0703505941c00cde7c399d3b7c64810ce3f8f0b53744b9eb61719d79272d8676519b0183c4af124f89702d576793b2f342
-  data.tar.gz: fca2c0f960400a55a7f67c7e0d93465174d785b299f703271cc1ce0befc2430093f0556e570469c0a9251ac6da7aff06dba8180499d284b4a6989d0b11ab7980
+  metadata.gz: 34f96472deced27c678bb902dbd70bf3340debe5eed76506ff949ce08b647b5893b005806b82920f75582e371ae5ee7278ea4f3f77bc2bb6751fcb00750e7fee
+  data.tar.gz: d99da7567bae448c909d51c3fd720ddb29429993a60cd499c195460e1d1d052e6c3f9d8481af93dc07209f0460a42ad10a2e8a8ae011923dd17cd23f124faf26

data/lib/tcell_agent/agent.rb

@@ -99,6 +99,7 @@ module TCellAgent
       @mutex = Monitor.new
 
       @response_time_table = {}
+      @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
       @dispatchEvents = []
       @eventQueue = BoundedQueue.new(200)
 
@@ -118,7 +119,7 @@ module TCellAgent
           TCellAgent.configuration.app_id == nil) 
         puts " ********* ********* ********* *********"
         puts "* tCell.io                               *"
-        puts "* Confiuration info is missing, you may  *"
+        puts "* Configuration info is missing, you may  *"
         puts "* need to download config file and place *"
         puts "* it in the config/ directory            *"
         puts " ********* ********* ********* *********"

data/lib/tcell_agent/agent/event_processor.rb

@@ -84,6 +84,16 @@ module TCellAgent
                             @response_time_table = {}
                           end
                         end
+                        if @sessions_metrics.has_sessions?
+                          sessions_to_send = nil
+                          @mutex.synchronize do
+                            sessions_to_send = @sessions_metrics
+                            @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
+                          end
+                          @event_dispatch_monitor.synchronize {
+                            @dispatchEvents.push( sessions_to_send )
+                          }
+                        end
                         @event_dispatch_monitor.synchronize {
                           events_to_send = @dispatchEvents
                           result = tapi.sendEventSet(events_to_send)
@@ -98,8 +108,8 @@ module TCellAgent
                           end
                         }
                       end
-                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ  
-                else  
+                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
+                else
                   event.post_process
                   if event.send == true
                     @event_dispatch_monitor.synchronize {
@@ -108,7 +118,7 @@ module TCellAgent
                   end
                   if (event.flush or @dispatchEvents.length >= @dispatchEventsLimit or wait_for < 0)
                     last_run_time = Time.now
-                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ 
+                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
                       if (@events_send_empties || @dispatchEvents.length > 0)
                         if (@response_time_table.size > 0)
                           metrics_event = TCellAgent::SensorEvents::MetricsEvent.new
@@ -120,6 +130,16 @@ module TCellAgent
                             @response_time_table = {}
                           end
                         end
+                        if @sessions_metrics.has_sessions?
+                          sessions_to_send = nil
+                          @mutex.synchronize do
+                            sessions_to_send = @sessions_metrics
+                            @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
+                          end
+                          @event_dispatch_monitor.synchronize {
+                            @dispatchEvents.push( sessions_to_send )
+                          }
+                        end
                         @event_dispatch_monitor.synchronize {
                           events_to_send = @dispatchEvents
                           result = tapi.sendEventSet(events_to_send)
@@ -130,10 +150,10 @@ module TCellAgent
                           end
                         }
                       end
-                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ  
+                    # JJJJJJJJ JJJJJJJ JJJJJJJ JJJJJJJJ JJJJJJJJJ
                   end
                 end
-              rescue ThreadError => te
+              rescue ThreadError
                 last_run_time = Time.now
                 @event_dispatch_monitor.synchronize {
                   @dispatchEvents = []
@@ -278,12 +298,42 @@ module TCellAgent
       end
     end
 
+    def increment_session_info(hmac_session_id, user_id, ip_address, user_agent)
+      TCellAgent::Instrumentation.safe_block("Incrementing session info") do
+
+        if TCellAgent::Agent.is_parent_process? == false
+          TCellAgent.queue_metric({
+            "_type"=>"increment_session_info",
+            "hmac_session_id" => hmac_session_id,
+            "user_id" => user_id,
+            "ip_address" => ip_address,
+            "user_agent" => user_agent
+          })
+          return
+        end
+
+        @mutex.synchronize do
+          @sessions_metrics.add_session_info(hmac_session_id, user_id, ip_address, user_agent)
+        end
+
+        if @sessions_metrics.flush
+          TCellAgent.send_event(TCellAgent::SensorEvents::FlushDummyEvent.new)
+        end
+      end
+    end
+
     def increment_route(route_id, response_time)
-      begin
+      TCellAgent::Instrumentation.safe_block("Incrementing route") do
+
         if TCellAgent::Agent.is_parent_process? == false
-          TCellAgent.queue_metric({"_type"=>"increment_route", "route_id"=>route_id, "response_time"=>response_time})
+          TCellAgent.queue_metric({
+            "_type"=>"increment_route",
+            "route_id"=>route_id,
+            "response_time"=>response_time
+          })
           return
         end
+
         @mutex.synchronize do
           if (route_id == nil || route_id == "")
             route_id = "?"
@@ -294,8 +344,7 @@ module TCellAgent
           @response_time_table[route_id]["mn"] = [@response_time_table[route_id].fetch("mn",response_time), response_time].min
           @response_time_table[route_id]["t"] = ((@response_time_table[route_id].fetch("t",0)*(@response_time_table[route_id]["c"]-1)) + response_time) / @response_time_table[route_id]["c"]
         end
-      rescue Exception => inc_exception
-        TCellAgent.logger.debug("Could not add incrememnt exception #{inc_exception.message}")
+
       end
     end
 

data/lib/tcell_agent/agent/fork_pipe_manager.rb

@@ -20,7 +20,7 @@ module TCellAgent
               self.start_listener(&block)
           end
         rescue Exception=>init_exception
-          TCellAgent.logger.error("Could not start listener for pipe to forks")          
+          TCellAgent.logger.error("Could not start listener for pipe to forks")
           TCellAgent.logger.error(init_exception.message)
           TCellAgent.logger.debug(init_exception.backtrace)
         end
@@ -44,14 +44,14 @@ module TCellAgent
                   TCellAgent.logger.error(block_exception.message)
                   TCellAgent.logger.debug(block_exception.backtrace)
                   sleep 0.5
-                end                                                                                                                                                                                                                                 
+                end
             end
         }
       end
       def send_to_parent(event)
         if is_parent?
             #puts "Sending in pipe the wrong way"
-            return 
+            return
         else
             begin
               packed_event = Marshal.dump(event)
@@ -61,7 +61,7 @@ module TCellAgent
               TCellAgent.logger.error("Could not write to pipe")
               TCellAgent.logger.error(block_exception.message)
               TCellAgent.logger.debug(block_exception.backtrace)
-            end   
+            end
         end
       end
     end
@@ -76,7 +76,7 @@ module TCellAgent
       end
     }
     @@metrics_pipe_manager = ForkPipeManager.new { |val|
-      begin
+      TCellAgent::Instrumentation.safe_block("Handling metrics_pipe_block") do
         switch_on = val.fetch("_type","")
         case switch_on
         when "increment_route"
@@ -86,20 +86,23 @@ module TCellAgent
           )
         when "discover_database_fields"
           TCellAgent.discover_database_fields(
-            val.fetch("route_id",nil), 
-            val.fetch("database",nil), 
-            val.fetch("schema",nil), 
-            val.fetch("table",nil), 
+            val.fetch("route_id",nil),
+            val.fetch("database",nil),
+            val.fetch("schema",nil),
+            val.fetch("table",nil),
             val.fetch("fields",nil)
           )
+        when "increment_session_info"
+          TCellAgent.increment_session_info(
+            val.fetch("hmac_session_id", nil),
+            val.fetch("user_id", nil),
+            val.fetch("ip_address", nil),
+            val.fetch("user_agent", nil)
+          )
         else
-          # Log this?
+          raise Exception.new("Metrics Pipe Manager received unknown metric: #{val.fetch("_type","")}")
         end
-      rescue Exception=>block_exception
-        TCellAgent.logger.error("Could handle metrics_pipe_block")
-        TCellAgent.logger.error(block_exception.message)
-        TCellAgent.logger.debug(block_exception.backtrace)
-      end  
+      end
     }
     def self.is_parent_process?
       @@event_pipe_manager.is_parent?

data/lib/tcell_agent/agent/static_agent.rb

@@ -42,6 +42,9 @@ module TCellAgent
   def self.policy(policy_type)
     self.thread_agent.policies.fetch(policy_type, nil)
   end
+  def self.increment_session_info(hmac_session_id, user_id, ip_address, user_agent)
+    self.thread_agent.increment_session_info(hmac_session_id, user_id, ip_address, user_agent)
+  end
   def self.increment_route(route_id, response_time)
     self.thread_agent.increment_route(route_id, response_time)
   end
@@ -57,4 +60,4 @@ module TCellAgent
   def self.ensure_event_processor_running
     self.thread_agent.ensure_event_processor_running
   end
-end
+end

data/lib/tcell_agent/configuration.rb

@@ -30,24 +30,26 @@ module TCellAgent
       :cache_filename,
       :js_agent_api_base_url,
       :js_agent_url,
-      :raise_exceptions
+      :raise_exceptions,
+      :allow_unencrypted_appsensor_payloads
 
     attr_accessor :exp_config_settings
 
-
     def initialize(filename="config/tcell_agent.config", useapp=nil)
       @version = 0
       @exp_config_settings = true
       @demomode = false
 
+      @allow_unencrypted_appsensor_payloads = [true, "true", "yes", "1"].include?(ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"])
+
       @fetch_policies_from_tcell = true
       @instrument_for_events = true
       @enabled = true
       @cache_filename = "tcell/cache/tcell_agent.cache"
       @default_log_filename = "tcell/logs/tcell_agent.log"
 
-      @event_batch_size_limit = 25
-      @event_time_limit_seconds = 55
+      @event_batch_size_limit = 50
+      @event_time_limit_seconds = 15
 
       @raise_exceptions = false
 

data/lib/tcell_agent/devise.rb

@@ -39,45 +39,45 @@ module TCellAgent
         end
       end
     end
-    Devise::Strategies::DatabaseAuthenticatable.class_eval do
-      alias_method :original_authenticate!, :authenticate!
-      def authenticate!
-        begin
-          tcell_background_worker = TCellAgent.thread_agent
-          # if (tcell_background_worker && tcell_background_worker.honeytokens_policy)
-          #   credstring = ""
-          #   authentication_keys.each do |authentication_key|
-          #     credstring = credstring + authentication_hash[authentication_key] + "::"
-          #   end
-          #   credstring = credstring + password[1..-3] # Chop off first and last 2 characters
-          #   token_id = tcell_background_worker.honeytokens_policy.id_for_credentialstring(credstring)
-          #   if token_id
-          #     begin
-          #       event = TCellAgent::SensorEvents::HoneytokensSensorEvent.new(request, token_id)
-          #       TCellAgent.send_event(event)
-          #     rescue
-          #       #pass
-          #     end
-          #     return
-          #   end
-          # end
-        rescue Exception => e
-          TCellAgent.logger.error("uncaught exception while processing honeytokens: #{e.message}")
-        end
-        original_authenticate!
-      end
-    end
-  end
-end
-module Devise
-  module Models
-    module Recoverable
-      extend ActiveSupport::Concern
-      alias_method :original_send_reset_password_instructions, :send_reset_password_instructions
-      def send_reset_password_instructions
-        x = original_send_reset_password_instructions
-        x
-      end
-    end
+    # Devise::Strategies::DatabaseAuthenticatable.class_eval do
+    #   alias_method :original_authenticate!, :authenticate!
+    #   def authenticate!
+    #     begin
+    #       tcell_background_worker = TCellAgent.thread_agent
+    #       # if (tcell_background_worker && tcell_background_worker.honeytokens_policy)
+    #       #   credstring = ""
+    #       #   authentication_keys.each do |authentication_key|
+    #       #     credstring = credstring + authentication_hash[authentication_key] + "::"
+    #       #   end
+    #       #   credstring = credstring + password[1..-3] # Chop off first and last 2 characters
+    #       #   token_id = tcell_background_worker.honeytokens_policy.id_for_credentialstring(credstring)
+    #       #   if token_id
+    #       #     begin
+    #       #       event = TCellAgent::SensorEvents::HoneytokensSensorEvent.new(request, token_id)
+    #       #       TCellAgent.send_event(event)
+    #       #     rescue
+    #       #       #pass
+    #       #     end
+    #       #     return
+    #       #   end
+    #       # end
+    #     rescue Exception => e
+    #       TCellAgent.logger.error("uncaught exception while processing honeytokens: #{e.message}")
+    #     end
+    #     original_authenticate!
+    #   end
+    # end
   end
 end
+# module Devise
+#   module Models
+#     module Recoverable
+#       extend ActiveSupport::Concern
+#       alias_method :original_send_reset_password_instructions, :send_reset_password_instructions
+#       def send_reset_password_instructions
+#         x = original_send_reset_password_instructions
+#         x
+#       end
+#     end
+#   end
+# end

data/lib/tcell_agent/instrumentation.rb

@@ -8,6 +8,53 @@ require 'date'
 
 module TCellAgent
   module Instrumentation
+    class ContextFilter
+      attr_accessor :type
+      attr_accessor :rule
+      attr_accessor :context
+      attr_accessor :parameter
+      attr_accessor :database
+      attr_accessor :schema
+      attr_accessor :table
+      attr_accessor :field
+
+      DATABASE = "db"
+      REQUEST = "request"
+
+      def for_request(context, parameter, rule)
+        self.type = ContextFilter::REQUEST
+        self.context = context
+        self.parameter = parameter
+        self.rule = rule
+        return self
+      end
+      def create_hash_value
+          "#{self.type}#{self.context}#{self.parameter}#{self.database}#{self.schema}#{self.table}#{self.field}#{self.rule}".hash
+      end 
+      def eql?(other_key)
+        self.hash == other_key.hash
+      end
+      def hash
+        self.create_hash_value
+      end
+      def for_database(database, schema, table, field, rule)
+          self.type = ContextFilter::DATABASE
+          self.database = database
+          self.schema = schema
+          self.table = table
+          self.field = field
+          self.rule = rule
+          return self
+      end
+      def for_request(context, parameter, rule)
+          self.type = ContextFilter::REQUEST
+          self.context = context
+          self.parameter = parameter
+          self.rule = rule
+          return self
+      end
+    end
+
 
     class TCellData
       attr_accessor :transaction_id
@@ -16,7 +63,10 @@ module TCellAgent
       attr_accessor :user_id
       attr_accessor :route_id
       attr_accessor :uri
+      attr_accessor :context_filters_by_term
       attr_accessor :database_filters
+      attr_accessor :ip_address
+      attr_accessor :user_agent
       def self.filterx(sanitize_string, event_flag, replace_flag, term)
         send_event = false
         sanitize_string.gsub!(term) {|m|
@@ -32,13 +82,35 @@ module TCellAgent
         return send_event
       end
       def initialize
-        @database_filters = Hash.new{|h,k| h[k] = Set.new}
+        @context_filters_by_term = Hash.new{|h,k| h[k] = Set.new}
+      end
+      def is_valid_term?(term)
+        if term != nil && term != '' and term.to_s.length >= 5
+          return true
+        end
+        return false
       end
       def add_response_db_filter(term, action_obj, database, schema, table, field)
-        if (term != nil && term != '' && term.to_s.length >= 5)
-          self.database_filters[term.to_s].add([database, schema, table, field, action_obj])
+        if is_valid_term?(term)
+            self.context_filters_by_term[term.to_s].add((ContextFilter.new).for_database(database, schema, table, field, action_obj))
         end
       end
+      def add_filter_for_request_parameter(term, rule, parameter_name)
+        if is_valid_term?(term)
+            self.context_filters_by_term[term.to_s].add((ContextFilter.new).for_request("form", parameter_name, rule))
+        end
+      end
+      def add_filter_for_header_value(term, rule, header_name)
+        if is_valid_term?(term)
+            self.context_filters_by_term[term.to_s].add((ContextFilter.new).for_request("header", header_name, rule))
+        end
+      end 
+      def add_filter_for_cookie_value(term, rule, cookie_name)
+        if is_valid_term?(term)
+            self.context_filters_by_term[term.to_s].add((ContextFilter.new).for_request("cookie", cookie_name, rule))
+        end
+      end
+
       def filter_body(body)
         dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
         if dlp_policy and self.session_id
@@ -56,34 +128,28 @@ module TCellAgent
             end
           end
         end
-        self.database_filters.sort_by {|term,properties| -term.length }.each do |term, properties|
-          action_infos = self.database_filters[term]
-          replace_actions = (action_infos.select {|action_info| action_info[4].body_redact == true })
-          event_actions = (action_infos.select {|action_info| (action_info[4].body_redact != true && action_info[4].body_event == true) })
-          send_flag = TCellData.filterx(body, event_actions.length > 0, replace_actions.length > 0, term)
+        self.context_filters_by_term.sort_by {|term,context_filters| -term.length }.each do |term, context_filters|
+          replace_filters = (context_filters.select {|context_filter| context_filter.rule.body_redact == true })
+          event_filters = (context_filters.select {|context_filter| (context_filter.rule.body_redact != true && context_filter.rule.body_event == true) })
+          send_flag = TCellData.filterx(body, event_filters.length > 0, replace_filters.length > 0, term)
           if send_flag
-            replace_actions.each do |action_info|
-              database, schema, table, field, action_obj = action_info
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::DlpEvent.new(
-                  self.route_id, 
-                  self.uri, 
-                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY,
-                  action_obj.action_id
-                  ).for_database(database, schema, table, field)
-              )
-            end
-            event_actions.each do |action_info|
-              database, schema, table, field, action_obj = action_info
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::DlpEvent.new(
-                  self.route_id, 
-                  self.uri, 
-                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY,
-                  action_obj.action_id
-                  ).for_database(database, schema, table, field)
+            (replace_filters + event_filters).each { |filter|
+              base_event = TCellAgent::SensorEvents::DlpEvent.new(
+                    self.route_id, 
+                    self.uri, 
+                    TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY,
+                    filter.rule.action_id
               )
-            end
+              if filter.type == ContextFilter::DATABASE
+                TCellAgent.send_event(
+                    base_event.for_database(filter.database, filter.schema, filter.table, filter.field)
+                )
+              elsif filter.type == ContextFilter::REQUEST
+                TCellAgent.send_event(
+                    base_event.for_request(filter.context, filter.parameter)
+                )
+              end
+            }
           end
         end
         return body
@@ -106,35 +172,29 @@ module TCellAgent
             end
           end
         end
-        self.database_filters.sort_by {|term,properties| -term.length }.each do |term, properties|
-          action_infos = self.database_filters[term]
-          replace_actions = (action_infos.select {|action_info| action_info[4].log_redact == true })
-          event_actions = (action_infos.select {|action_info| (action_info[4].log_redact != true && action_info[4].log_event == true) })
-          send_flag = TCellData.filterx(log_msg, event_actions.length > 0, replace_actions.length > 0, term)
+        self.context_filters_by_term.sort_by {|term,context_filters| -term.length }.each do |term, context_filters|
+          replace_filters = (context_filters.select {|context_filter| context_filter.rule.log_redact == true })
+          event_filters = (context_filters.select {|context_filter| (context_filter.rule.log_redact != true && context_filter.rule.log_event == true) })
+          send_flag = TCellData.filterx(log_msg, event_filters.length > 0, replace_filters.length > 0, term)
           if send_flag
-            replace_actions.each do |action_info|
-              database, schema, table, field, action_obj = action_info
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::DlpEvent.new(
-                  self.route_id, 
-                  self.uri, 
-                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG,
-                  action_obj.action_id
-                  ).for_database(database, schema, table, field)
-              )
-            end
-            event_actions.each do |action_info|
-              database, schema, table, field, action_obj = action_info
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::DlpEvent.new(
-                  self.route_id, 
-                  self.uri, 
-                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG,
-                  action_obj.action_id
-                  ).for_database(database, schema, table, field)
+            (replace_filters + event_filters).each { |filter|
+              base_event = TCellAgent::SensorEvents::DlpEvent.new(
+                    self.route_id, 
+                    self.uri, 
+                    TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG,
+                    filter.rule.action_id
               )
-            end
-          end        
+              if filter.type == ContextFilter::DATABASE
+                TCellAgent.send_event(
+                    base_event.for_database(filter.database, filter.schema, filter.table, filter.field)
+                )
+              elsif filter.type == ContextFilter::REQUEST
+                TCellAgent.send_event(
+                    base_event.for_request(filter.context, filter.parameter)
+                )
+              end
+            }
+          end
         end
         return log_msg
       end
@@ -151,16 +211,22 @@ module TCellAgent
     def self.safe_block(message, &block)
       begin
         block.call()
+
       rescue Exception => ex
-        TCellAgent.logger.debug "Exception in safe_block #{message}: #{ex.class} happened, message is #{ex.message}"
-        TCellAgent.logger.debug(ex.backtrace)
+        if TCellAgent.configuration.raise_exceptions
+          raise ex
+
+        else
+          TCellAgent.logger.debug "Exception in safe_block #{message}: #{ex.class} happened, message is #{ex.message}"
+          TCellAgent.logger.debug(ex.backtrace)
+        end
       end
     end
 
     def self.safe_block_no_log(message, &block)
       begin
         block.call()
-      rescue Exception => ex
+      rescue Exception
       end
     end
   end