tcell_agent 0.2.2 → 0.2.4

data/lib/tcell_agent/agent/fork_pipe_manager.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+
+require 'thread'
+require "tcell_agent/logger"
+
+module TCellAgent
+  class Agent
+    class ForkPipeManager
+      attr_accessor :readp
+      attr_accessor :writep
+
+      @@parent_id = Process.pid
+      def initialize(&block)
+        begin
+          @readp, @writep = IO.pipe('ASCII-8BIT', 'ASCII-8BIT', binmode: true)
+          if defined?(::Encoding::ASCII_8BIT)
+            @writep.set_encoding(::Encoding::ASCII_8BIT)
+          end
+          if is_parent?
+              self.start_listener(&block)
+          end
+        rescue Exception=>init_exception
+          TCellAgent.logger.error("Could not start listener for pipe to forks")          
+          TCellAgent.logger.error(init_exception.message)
+          TCellAgent.logger.debug(init_exception.backtrace)
+        end
+      end
+      def is_parent?
+        @@parent_id == Process.pid
+      end
+      def start_listener(&block)
+        Thread.new {
+            while true do
+                begin
+                    packed_bytes = @readp.read(4)
+                    event_length = packed_bytes.unpack("L>").first
+                    packed_event = @readp.read(event_length)
+                    event = Marshal.load(packed_event)
+                    if block
+                      block.call(event)
+                    end
+                rescue Exception=>block_exception
+                  TCellAgent.logger.error("Could not decode block")
+                  TCellAgent.logger.error(block_exception.message)
+                  TCellAgent.logger.debug(block_exception.backtrace)
+                  sleep 0.5
+                end                                                                                                                                                                                                                                 
+            end
+        }
+      end
+      def send_to_parent(event)
+        if is_parent?
+            #puts "Sending in pipe the wrong way"
+            return 
+        else
+            begin
+              packed_event = Marshal.dump(event)
+              packed_bytes = [packed_event.bytesize].pack("L>")
+              @writep.write(packed_bytes+packed_event)
+            rescue Exception=>block_exception
+              TCellAgent.logger.error("Could not write to pipe")
+              TCellAgent.logger.error(block_exception.message)
+              TCellAgent.logger.debug(block_exception.backtrace)
+            end   
+        end
+      end
+    end
+    @@event_pipe_manager = ForkPipeManager.new { |event|
+      begin
+        TCellAgent.send_event(event)
+      rescue Exception=>block_exception
+        TCellAgent.logger.error("Could handle send_event_block")
+        TCellAgent.logger.error(block_exception.message)
+        TCellAgent.logger.debug(block_exception.backtrace)
+      end   
+    }
+    @@metrics_pipe_manager = ForkPipeManager.new { |val|
+      begin
+        switch_on = val.fetch("_type","")
+        case switch_on
+        when "increment_route"
+          TCellAgent.increment_route(
+            val.fetch("route_id",nil),
+            val.fetch("response_time",nil)
+          )
+        when "discover_database_fields"
+          TCellAgent.discover_database_fields(
+            val.fetch("route_id",nil), 
+            val.fetch("database",nil), 
+            val.fetch("schema",nil), 
+            val.fetch("table",nil), 
+            val.fetch("fields",nil)
+          )
+        else
+          # Log this?
+        end
+      rescue Exception=>block_exception
+        TCellAgent.logger.error("Could handle metrics_pipe_block")
+        TCellAgent.logger.error(block_exception.message)
+        TCellAgent.logger.debug(block_exception.backtrace)
+      end  
+    }
+    def self.is_parent_process?
+      @@event_pipe_manager.is_parent?
+    end
+    def self.send_to_metrics_pipe(hash_value)
+      @@metrics_pipe_manager.send_to_parent(hash_value)
+    end
+    def self.send_to_event_pipe(event)
+      @@event_pipe_manager.send_to_parent(event)
+    end
+
+  end
+end

data/lib/tcell_agent/agent/policy_manager.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # See the file "LICENSE" for the full license governing this code.
 
 require "tcell_agent/logger"
@@ -14,7 +16,6 @@ require "tcell_agent/policies/http_redirect_policy"
 require "tcell_agent/policies/secure_headers_policy"
 require "tcell_agent/policies/honeytokens_policy"
 require "tcell_agent/policies/appsensor_policy"
-require "tcell_agent/policies/add_script_tag_policy"
 
 require "tcell_agent/sensor_events/server_agent"
 
@@ -28,21 +29,32 @@ require 'json'
 module TCellAgent
   class Agent
 
+    def ensure_policy_polling_running
+      return if policy_polling_running?
+      @policy_polling_worker_mutex.synchronize do
+        return if policy_polling_running?
+        start_policy_polling
+      end
+    end
+
+    def policy_polling_running?
+      @policy_polling_thread && @policy_polling_thread.alive?
+    end
+
     def start_policy_polling
       if TCellAgent.configuration.fetch_policies_from_tcell == true
         TCellAgent.logger.debug("starting background polling thread")
-        @thread = Thread.new do
+        @policy_polling_thread = Thread.new do
           last_poll_time = 0
-          tapi = TCellApi.new
           last_run = Time.now
           loop do
             begin
-              if (TCellAgent.configuration.version == 0)
-                policy_jsons = tapi.pollOldAPI(last_poll_time)
-              else
-                policy_jsons = tapi.pollAPI(last_poll_time)
-              end
-              if policy_jsons.key?("last_timestamp")
+              policy_jsons = @@policy_tapi.pollAPI(last_poll_time)
+              if policy_jsons == nil
+                TCellAgent.logger.error("Policy was nil")
+                sleep(10.0)
+                next
+              elsif policy_jsons.key?("last_timestamp")
                 if policy_jsons["last_timestamp"] != 0
                   last_poll_time = policy_jsons["last_timestamp"]
                 end
@@ -53,20 +65,19 @@ module TCellAgent
               end
               processPolicyJson(policy_jsons)
             rescue Exception => e 
-              TCellAgent.logger.error("uncaught exception while handling connection: #{e.message}")
+              TCellAgent.logger.error("exception while handling connection: #{e.message}")
               TCellAgent.logger.debug(e.backtrace)
-              TCellAgent.logger.debug("Sleeping 60 seconds because the tCell.io request failed...")
-              sleep(60) #wait a minute before trying again
+              TCellAgent.logger.debug("Sleeping 30 seconds because the tCell.io request failed...")
+              sleep(30) #wait a minute before trying again
             end
-            # A little rate limiting
             if (Time.now - last_run) < 1
-              TCellAgent.logger.debug("Rate limiting: sleeping 30 seconds")
-              sleep(30)
+              TCellAgent.logger.debug("Rate limiting: sleeping 10 seconds")
+              sleep(10)
             end
             last_run = Time.now
           end
         end
-      end   
+      end  
     end
 
     def processPolicyJson(policy_jsons, cache_the_policy=true)
@@ -97,6 +108,7 @@ module TCellAgent
 
     def cache(policy_name, policy)
       cache_filename = TCellAgent.configuration.cache_filename
+      FileUtils.mkdir_p(File.dirname(cache_filename))
       if TCellAgent.configuration.app_id
         cache_filename = cache_filename + '.' + TCellAgent.configuration.app_id
       end

data/lib/tcell_agent/agent/policy_types.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 # See the file "LICENSE" for the full license governing this code.
 
 require "tcell_agent/policies/content_security_policy"
@@ -8,7 +10,6 @@ require "tcell_agent/policies/http_redirect_policy"
 require "tcell_agent/policies/secure_headers_policy"
 require "tcell_agent/policies/honeytokens_policy"
 require "tcell_agent/policies/appsensor_policy"
-require "tcell_agent/policies/add_script_tag_policy"
 require "tcell_agent/policies/login_fraud_policy"
 require "tcell_agent/policies/dataloss_policy"
 
@@ -20,9 +21,8 @@ module TCellAgent
     HttpTx = "http-tx"
     HttpRedirect = "http-redirect"
     AppSensor = "appsensor"
-    AddScriptTag = "add-jsagent-script-tag"
     HoneyTokens = "exp-honeytokens"
-    LoginFraud = "exp-login-fraud"
+    LoginFraud = "login"
     DataLoss = "dlp"
 
     ClassMap = {
@@ -32,7 +32,6 @@ module TCellAgent
       HttpTx=>TCellAgent::Policies::HttpTxPolicy,
       HttpRedirect=>TCellAgent::Policies::HttpRedirectPolicy,
       AppSensor=>TCellAgent::Policies::AppSensorPolicy,
-      AddScriptTag=>TCellAgent::Policies::AddScriptTagPolicy,
       HoneyTokens=>TCellAgent::Policies::HoneytokensPolicy,
       LoginFraud=>TCellAgent::Policies::LoginFraudPolicy,
       DataLoss=>TCellAgent::Policies::DataLossPolicy

data/lib/tcell_agent/agent/route_manager.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+# See the file "LICENSE" for the full license governing this code.
+
+require "tcell_agent/logger"
+require "tcell_agent/version"
+require "tcell_agent/api"
+require "tcell_agent/configuration"
+
+require "tcell_agent/routes/table"
+require "tcell_agent/sensor_events/discovery"
+require "tcell_agent"
+
+module TCellAgent
+  class Agent
+  	def discover_database_field(route_id, database, schema, table, field)
+  		if (@route_table.routes[route_id].database[database][schema][table][field].discovered != true)
+  			@route_table.routes[route_id].database[database][schema][table][field].discovered = true
+  			event = (TCellAgent::SensorEvents::DiscoveryEvent.new(route_id)).for_database(database, schema, table, field)
+  			TCellAgent.send_event(event)
+  		end
+  	end
+  	def discover_database_fields(route_id, database, schema, table, fields)
+      if TCellAgent::Agent.is_parent_process? == false
+        TCellAgent.queue_metric({"_type"=>"discover_database_fields", 
+          "route_id"=>route_id, 
+          "database"=>database, 
+          "schema"=>schema, 
+          "table"=>table, 
+          "fields"=>fields})
+        return
+      end
+  		discovered_fields = fields.select { |field| 
+  				@route_table.routes[route_id].database[database][schema][table][field].discovered != true
+  		}
+  		if (discovered_fields.length > 0)
+  			discovered_fields.each { |field|
+				@route_table.routes[route_id].database[database][schema][table][field].discovered = true
+  			}
+  			event = (TCellAgent::SensorEvents::DiscoveryEvent.new(route_id)).for_database_fields(database, schema, table, fields)
+  			TCellAgent.send_event(event)
+  		end
+   	end 
+  end
+end

data/lib/tcell_agent/agent/static_agent.rb

@@ -1,22 +1,60 @@
+# encoding: utf-8
+
 # See the file "LICENSE" for the full license governing this code.
 require 'tcell_agent/sensor_events/metrics'
+require 'monitor'
 
 module TCellAgent
-  class << self
-    attr_accessor :thread_agent
+  @@instance_lock = Mutex.new
+  @@my_thread_agent = nil
+
+  def self.thread_agent
+    if(self.thread_agent_defined? == false)
+      @@instance_lock.synchronize do
+        if(self.thread_agent_defined? == false)
+          @@my_thread_agent= TCellAgent::Agent.new(Process.pid)
+        end
+      end
+    end
+    @@my_thread_agent
   end
-  def self.send_event(event)
-    if (self.thread_agent != nil)
-        self.thread_agent.queueSensorEvent(event)
+
+  def self.thread_agent_defined?
+    @@my_thread_agent != nil
+  end
+
+  # setter
+  def self.thread_agent=some_agent
+    @@instance_lock.synchronize do
+      @@my_thread_agent = some_agent
     end
   end
+
+  #class << self
+  #  attr_accessor :thread_agent
+  #end
+  def self.send_event(event)
+    self.thread_agent.queueSensorEvent(event)
+  end
+  def self.queue_metric(event)
+    self.thread_agent._queue_metric(event)
+  end
   def self.policy(policy_type)
-    if (self.thread_agent != nil)
-        self.thread_agent.policies.fetch(policy_type, nil)
-    end
+    self.thread_agent.policies.fetch(policy_type, nil)
   end
   def self.increment_route(route_id, response_time)
-    event = TCellAgent::SensorEvents::RequestRouteTimer.new(route_id, response_time)
-    self.send_event(event)
+    self.thread_agent.increment_route(route_id, response_time)
+  end
+  def self.discover_database_field(route_id, database, schema, table, field)
+    self.thread_agent.discover_database_field(route_id, database, schema, table, field)
+  end
+  def self.discover_database_fields(route_id, database, schema, table, fields)
+    self.thread_agent.discover_database_fields(route_id, database, schema, table, fields)
+  end
+  def self.stop_agent
+    self.thread_agent.stop_agent = true
+  end
+  def self.ensure_event_processor_running
+    self.thread_agent.ensure_event_processor_running
   end
 end

data/lib/tcell_agent/api.rb

@@ -30,7 +30,6 @@ module TCellAgent
         TCellAgent.logger.debug("tCell.io Could not add agent string: " + e.message)
       end
       response = RestClient.get full_url,request_headers
-      new_timestamp = last_timestamp
       TCellAgent.logger.debug "tCell.io API Response: " + response
       response_json = JSON.parse(response)
       if (response_json && response_json.has_key?("result"))
@@ -50,7 +49,6 @@ module TCellAgent
       if (last_timestamp && last_timestamp != "")
         full_url = full_url + "?last_timestamp=" + last_timestamp.to_s
       end
-      TCellAgent.logger.debug("tCell.io Old API Request: " + full_url)
       response = RestClient.get full_url
       TCellAgent.logger.debug "tCell.io API Response: " + response
       response_json = JSON.parse(response)

data/lib/tcell_agent/appsensor/path_traversal.rb

@@ -11,7 +11,7 @@ require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 module TCellAgent
     class AppSensor
         PATH_TRAVERSAL_REGEXES = [
-            Regexp.new("(?:(?:\\\/|\\\\)?\\.+(\\\/|\\\\)(?:\\.+)?)|(?:\\w+\\.exe\\??\\s)|(?:;\\s*\\w+\\s*\\\/[\\w*-]+\\\/)|(?:\\d\\.\\dx\\|)|(?:%(?:c0\\.|af\\.|5c\\.))|(?:\\\/(?:%2e){2})"),
+            Regexp.new("(?:(?:\\\/|\\\\)?\\.+(\\\/|\\\\)(?:\\.*))|(?:\\w+\\.exe\\??\\s)|(?:;\\s*\\w+\\s*\\\/[\\w*-]+\\\/)|(?:\\d\\.\\dx\\|)|(?:%(?:c0\\.|af\\.|5c\\.))|(?:\\\/(?:%2e){2})"),
             Regexp.new("(?:%c0%ae\\\/)|(?:(?:\\\/|\\\\)(home|conf|usr|etc|proc|opt|s?bin|local|dev|tmp|kern|[br]oot|sys|system|windows|winnt|program|%[a-z_-]{3,}%)(?:\\\/|\\\\))|(?:(?:\\\/|\\\\)inetpub|localstart\\.asp|boot\\.ini)"),
             Regexp.new("(?:etc\\\/\\W*passwd)")
         ]

data/lib/tcell_agent/configuration.rb

@@ -37,13 +37,17 @@ module TCellAgent
     def initialize(filename="config/tcell_agent.config", useapp=nil)
       @version = 0
       @exp_config_settings = true
-
+      @demomode = false
 
       @fetch_policies_from_tcell = true
       @instrument_for_events = true
       @enabled = true
-      @cache_filename = "tcell/tcell_agent.cache"
-      @default_log_filename = "tcell/tcell_agent.log"
+      @cache_filename = "tcell/cache/tcell_agent.cache"
+      @default_log_filename = "tcell/logs/tcell_agent.log"
+
+      @event_batch_size_limit = 25
+      @event_time_limit_seconds = 55
+
       read_config_using_env
       read_config_from_file(filename)
 
@@ -51,8 +55,6 @@ module TCellAgent
       @tcell_input_url ||= "https://input.tcell.io/api/v1"
       @js_agent_api_base_url ||= nil
       @js_agent_url ||= "https://api.tcell.io/tcellagent.min.js"
-      @event_batch_size_limit = 25
-      @event_time_limit_seconds = 30
 
       begin
         @host_identifier = (Socket.gethostname() || "localhost")
@@ -74,6 +76,11 @@ module TCellAgent
       @hmac_key = ENV["TCELL_HMAC_KEY"]
       @tcell_api_url = ENV["TCELL_API_URL"]
       @tcell_input_url = ENV["TCELL_INPUT_URL"]
+      @demomode = ENV["TCELL_DEMOMODE"] || @demomode
+      if @demomode
+        @event_batch_size_limit = 2
+        @event_time_limit_seconds = 5
+      end
     end
     def read_config_from_file(filename)
       if File.file?(filename)
@@ -127,7 +134,13 @@ module TCellAgent
             # Causes old event url to be used
             @company = app_data["company"]
 
-
+            if @demomode != true
+              @demomode = app_data.fetch('demomode', false)
+            end
+            if @demomode
+              @event_batch_size_limit = 2
+              @event_time_limit_seconds = 5
+            end
           else            
             puts " ********* ********* ********* *********"
             puts "* tCell.io                               *"

data/lib/tcell_agent/instrumentation.rb

@@ -16,6 +16,129 @@ module TCellAgent
       attr_accessor :user_id
       attr_accessor :route_id
       attr_accessor :uri
+      attr_accessor :database_filters
+      def self.filterx(sanitize_string, event_flag, replace_flag, term)
+        send_event = false
+        sanitize_string.gsub!(term) {|m|
+          if replace_flag
+            m = "[redacted]"
+            send_event = true
+          elsif event_flag
+            # m = "[hash]"
+            send_event = true
+          end
+          m
+        }
+        return send_event
+      end
+      def initialize
+        @database_filters = Hash.new{|h,k| h[k] = Set.new}
+      end
+      def add_response_db_filter(term, action_obj, database, schema, table, field)
+        if (term != nil and term != '')
+            self.database_filters[term].add([database, schema, table, field, action_obj])
+        end
+      end
+      def filter_body(body)
+        dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+        if dlp_policy and self.session_id
+          session_id_actions = dlp_policy.get_actions_for_session_id
+          if session_id_actions
+            send_flag = TCellData.filterx(body, session_id_actions.body_event, session_id_actions.body_redact, self.session_id)
+            if send_flag
+              TCellAgent.send_event(
+                TCellAgent::SensorEvents::DlpEvent.new(
+                  self.route_id, 
+                  self.uri, 
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY
+                  ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
+              )
+            end
+          end
+        end
+        self.database_filters.sort_by {|term,properties| -term.length }.each do |term, properties|
+          action_infos = self.database_filters[term]
+          replace_actions = (action_infos.select {|action_info| action_info[4].body_redact == true })
+          event_actions = (action_infos.select {|action_info| (action_info[4].body_redact != true && action_info[4].body_event == true) })
+          send_flag = TCellData.filterx(body, event_actions.length > 0, replace_actions.length > 0, term)
+          if send_flag
+            replace_actions.each do |action_info|
+              database, schema, table, field, action_obj = action_info
+              TCellAgent.send_event(
+                TCellAgent::SensorEvents::DlpEvent.new(
+                  self.route_id, 
+                  self.uri, 
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY,
+                  action_obj.action_id
+                  ).for_database(database, schema, table, field)
+              )
+            end
+            event_actions.each do |action_info|
+              database, schema, table, field, action_obj = action_info
+              TCellAgent.send_event(
+                TCellAgent::SensorEvents::DlpEvent.new(
+                  self.route_id, 
+                  self.uri, 
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY,
+                  action_obj.action_id
+                  ).for_database(database, schema, table, field)
+              )
+            end
+          end
+        end
+        return body
+      end
+
+      def filter_log(log_msg)
+        dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+        if dlp_policy and self.session_id
+          session_id_actions = dlp_policy.get_actions_for_session_id
+          if session_id_actions
+            send_flag = TCellData.filterx(log_msg, session_id_actions.log_event, session_id_actions.log_redact, self.session_id)
+            if send_flag
+              TCellAgent.send_event(
+                TCellAgent::SensorEvents::DlpEvent.new(
+                  self.route_id, 
+                  self.uri, 
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG
+                  ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
+              )
+            end
+          end
+        end
+        self.database_filters.sort_by {|term,properties| -term.length }.each do |term, properties|
+          action_infos = self.database_filters[term]
+          replace_actions = (action_infos.select {|action_info| action_info[4].log_redact == true })
+          event_actions = (action_infos.select {|action_info| (action_info[4].log_redact != true && action_info[4].log_event == true) })
+          send_flag = TCellData.filterx(log_msg, event_actions.length > 0, replace_actions.length > 0, term)
+          if send_flag
+            replace_actions.each do |action_info|
+              database, schema, table, field, action_obj = action_info
+              TCellAgent.send_event(
+                TCellAgent::SensorEvents::DlpEvent.new(
+                  self.route_id, 
+                  self.uri, 
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG,
+                  action_obj.action_id
+                  ).for_database(database, schema, table, field)
+              )
+            end
+            event_actions.each do |action_info|
+              database, schema, table, field, action_obj = action_info
+              TCellAgent.send_event(
+                TCellAgent::SensorEvents::DlpEvent.new(
+                  self.route_id, 
+                  self.uri, 
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG,
+                  action_obj.action_id
+                  ).for_database(database, schema, table, field)
+              )
+            end
+          end        
+        end
+        return log_msg
+      end
+
     end
 
     def self.instrument_frameworks