tcell_agent 0.2.2

data/config/initializers/init.rb

@@ -0,0 +1,8 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/logger'
+require 'tcell_agent/configuration'
+
+if (TCellAgent.configuration.enabled)
+    require 'tcell_agent/start_background_thread'
+end

data/lib/tcell_agent.rb

@@ -0,0 +1,33 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/logger'
+require 'tcell_agent/configuration'
+
+module TCellAgent
+  TCellAgent.logger.debug("Instrumenting")
+  #if (TCellAgent.configuration.enabled && TCellAgent.configuration.instrument_for_events)
+      require 'tcell_agent/rails' if defined?(Rails)
+      require 'tcell_agent/sinatra' if defined?(Sinatra)
+  #end
+end
+
+require 'tcell_agent/agent'
+
+require 'tcell_agent/policies/content_security_policy'
+require 'tcell_agent/policies/http_tx_policy'
+require 'tcell_agent/policies/http_redirect_policy'
+require 'tcell_agent/policies/secure_headers_policy'
+require 'tcell_agent/policies/honeytokens_policy'
+require 'tcell_agent/policies/clickjacking_policy'
+require 'tcell_agent/policies/appsensor_policy'
+require 'tcell_agent/policies/add_script_tag_policy'
+require 'tcell_agent/policies/login_fraud_policy'
+require 'tcell_agent/policies/dataloss_policy'
+
+require 'tcell_agent/sensor_events/app_sensor'
+require 'tcell_agent/sensor_events/dlp'
+require 'tcell_agent/appsensor'
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/util/redirect_utils'
+
+require 'tcell_agent/dataloss'

data/lib/tcell_agent/agent.rb

@@ -0,0 +1,79 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require "tcell_agent/logger"
+require "tcell_agent/version"
+require "tcell_agent/api"
+require "tcell_agent/configuration"
+
+require "tcell_agent/sensor_events/server_agent"
+require "tcell_agent/utils/queue_with_timeout"
+
+require "tcell_agent/agent/event_processor"
+require "tcell_agent/agent/policy_manager"
+require "tcell_agent/agent/static_agent"
+require "tcell_agent/agent/policy_types"
+
+require 'net/http'
+require 'thread'
+require 'logger'
+require 'json'
+
+module TCellAgent
+  class Agent
+
+    attr_accessor :start_pid
+    attr_accessor :eventQueue
+    attr_accessor :policies
+    attr_accessor :eventProcessorThread
+    attr_accessor :route_counter_table
+
+    def initialize(start_pid)
+      @start_pid = start_pid
+      @lock = Monitor.new
+      @metricsLock = Monitor.new
+      @policies = {}
+      @route_counter_table = {}
+
+      @dispatchEventsTimeout = TCellAgent.configuration.event_time_limit_seconds || 30
+      @dispatchEventsLimit = TCellAgent.configuration.event_batch_size_limit || 20
+      @dispatchEvents = []
+
+      @eventQueue = QueueWithTimeout.new
+
+      if TCellAgent.configuration.preload_policy_filename != nil
+        TCellAgent.logger.info("Preloading a policy file");
+        begin
+          policy_file = open(TCellAgent.configuration.preload_policy_filename).read
+          policy_jsons = JSON.parse(policy_file)
+          if policy_jsons.key?("result")
+            policy_jsons = policy_jsons["result"]
+          end
+          processPolicyJson(policy_jsons, false)
+        rescue Exception => e
+          TCellAgent.logger.error(e.message)
+        end
+      end
+      cached_policies = policies_from_cachefile
+      if (cached_policies)
+        processPolicyJson(cached_policies, false)
+      end
+    end
+
+    def start
+      if (TCellAgent.configuration.api_key == nil || 
+          TCellAgent.configuration.app_id == nil) 
+        puts " ********* ********* ********* *********"
+        puts "* tCell.io                               *"
+        puts "* Confiuration info is missing, you may  *"
+        puts "* need to download config file and place *"
+        puts "* it in the config/ directory            *"
+        puts " ********* ********* ********* *********"
+        return
+      end
+      self.start_event_processor()
+      self.start_policy_polling()
+
+    end
+
+  end
+end

data/lib/tcell_agent/agent/event_processor.rb

@@ -0,0 +1,133 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require "tcell_agent/logger"
+require "tcell_agent/version"
+require "tcell_agent/api"
+require "tcell_agent/configuration"
+
+require "tcell_agent/policies/content_security_policy"
+require "tcell_agent/policies/clickjacking_policy"
+require "tcell_agent/policies/http_tx_policy"
+require "tcell_agent/policies/http_redirect_policy"
+require "tcell_agent/policies/secure_headers_policy"
+require "tcell_agent/policies/honeytokens_policy"
+require "tcell_agent/policies/appsensor_policy"
+require "tcell_agent/policies/add_script_tag_policy"
+
+require "tcell_agent/sensor_events/server_agent"
+require "tcell_agent/sensor_events/metrics"
+
+require "tcell_agent/utils/queue_with_timeout"
+
+require 'net/http'
+require 'thread'
+require 'logger'
+require 'json'
+
+
+module TCellAgent
+  class Agent
+
+    def reset_dispatch
+      @dispatchEvents = []
+    end
+    def start_event_processor(send_empties=true)
+      @events_send_empties = send_empties
+
+      @eventProcessorThread = Thread.new do
+        TCellAgent.logger.debug("starting background event listener")
+        tapi = TCellApi.new
+
+        def dispatch(tapi, events)
+          if (@events_send_empties || events.length > 0)
+            metrics_event = TCellAgent::SensorEvents::MetricsEvent.new            
+            metrics_event.set_route_count_table(@eventQueue.get_response_time_table)
+            events.push( metrics_event )
+            @eventQueue.reset_response_time_table
+            return tapi.sendEventSet(events)
+          end
+          return true
+        end
+
+        begin
+          event = TCellAgent::SensorEvents::ServerAgentDetailsSensorEvent.new
+          TCellAgent.send_event(event)
+
+          event = TCellAgent::SensorEvents::ServerAgentPackagesSensorEvent.new
+          TCellAgent.send_event(event)
+        rescue Exception => te
+          TCellAgent.logger.error("Exception sending initial events: #{te.message}")
+          TCellAgent.logger.debug(te.backtrace)
+        end
+        timeout_flag = false
+        last_run_time = Time.now
+        #    orig = (Time.now.to_f * 1000).to_i
+        #    response = @app.call(env)
+        #    response_time = (Time.now.to_f * 1000).to_i - orig
+
+        loop do
+          begin
+            begin
+              now = Time.now
+              wait_for = @dispatchEventsTimeout - (now - last_run_time).to_i.abs
+              #puts "---- ---- "
+              #puts wait_for
+
+              event = @eventQueue.pop_with_timeout([wait_for, 0].max)
+              # puts "Hey, found event"
+              # puts event
+              if event["event_type"] == "RequestRouteTimer"
+              #   puts "Request"
+              #   puts event
+                route_id = event.route_id
+              #   puts "Route"
+              #   puts route_id
+                response_time = event.response_time
+              #   puts "ROUTE"
+                #puts response_time
+                @eventQueue.add_response_time(route_id, response_time)
+                #puts @eventQueue.get_response_time_table
+              #   puts "xxxx"
+              end
+              event.post_process
+              if event.send == true
+                @dispatchEvents.push(event)
+              end
+              if (event.flush or @dispatchEvents.length >= @dispatchEventsLimit or wait_for < 0)
+                last_run_time = Time.now
+                if ( dispatch(tapi, @dispatchEvents) )
+                  self.reset_dispatch
+                else
+                  @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
+                end
+              end
+            rescue ThreadError => te
+              last_run_time = Time.now
+              if ( dispatch(tapi, @dispatchEvents) )
+                self.reset_dispatch
+              else
+                @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
+              end
+            end
+            #puts "AFTER "
+            #puts Thread.current
+          rescue Exception => e
+            last_run_time = Time.now
+            TCellAgent.logger.error("Exception while processing events: #{e.message}")
+            TCellAgent.logger.debug(e.backtrace)
+            @dispatchEvents = @dispatchEvents.find_all{|item| item.ensure == true }
+          end
+        end
+      end
+    end
+
+    def queueSensorEvent(event)
+      if @eventProcessorThread != nil
+        if @eventQueue.length < 200
+          @eventQueue << event
+        end
+      end
+    end
+
+  end
+end

data/lib/tcell_agent/agent/policy_manager.rb

@@ -0,0 +1,138 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require "tcell_agent/logger"
+require "tcell_agent/version"
+require "tcell_agent/api"
+require "tcell_agent/configuration"
+
+require "tcell_agent/agent/policy_types"
+
+require "tcell_agent/policies/content_security_policy"
+require "tcell_agent/policies/clickjacking_policy"
+require "tcell_agent/policies/http_tx_policy"
+require "tcell_agent/policies/http_redirect_policy"
+require "tcell_agent/policies/secure_headers_policy"
+require "tcell_agent/policies/honeytokens_policy"
+require "tcell_agent/policies/appsensor_policy"
+require "tcell_agent/policies/add_script_tag_policy"
+
+require "tcell_agent/sensor_events/server_agent"
+
+require "tcell_agent/utils/queue_with_timeout"
+
+require 'net/http'
+require 'thread'
+require 'logger'
+require 'json'
+
+module TCellAgent
+  class Agent
+
+    def start_policy_polling
+      if TCellAgent.configuration.fetch_policies_from_tcell == true
+        TCellAgent.logger.debug("starting background polling thread")
+        @thread = Thread.new do
+          last_poll_time = 0
+          tapi = TCellApi.new
+          last_run = Time.now
+          loop do
+            begin
+              if (TCellAgent.configuration.version == 0)
+                policy_jsons = tapi.pollOldAPI(last_poll_time)
+              else
+                policy_jsons = tapi.pollAPI(last_poll_time)
+              end
+              if policy_jsons.key?("last_timestamp")
+                if policy_jsons["last_timestamp"] != 0
+                  last_poll_time = policy_jsons["last_timestamp"]
+                end
+              elsif policy_jsons.key?("last_id")
+                if policy_jsons["last_id"] != 0
+                  last_poll_time = policy_jsons["last_id"]
+                end
+              end
+              processPolicyJson(policy_jsons)
+            rescue Exception => e 
+              TCellAgent.logger.error("uncaught exception while handling connection: #{e.message}")
+              TCellAgent.logger.debug(e.backtrace)
+              TCellAgent.logger.debug("Sleeping 60 seconds because the tCell.io request failed...")
+              sleep(60) #wait a minute before trying again
+            end
+            # A little rate limiting
+            if (Time.now - last_run) < 1
+              TCellAgent.logger.debug("Rate limiting: sleeping 30 seconds")
+              sleep(30)
+            end
+            last_run = Time.now
+          end
+        end
+      end   
+    end
+
+    def processPolicyJson(policy_jsons, cache_the_policy=true)
+
+      if policy_jsons == nil
+        return
+      end
+
+      if policy_jsons.key?("data")
+        policy_data = policy_jsons["data"]
+      end              
+
+      TCellAgent::PolicyTypes::ClassMap.each do | policy_type, policy_class |
+        if (policy_jsons.key?(policy_type))
+          new_policy = policy_class.fromJson(policy_jsons[policy_type])
+          if new_policy
+            @lock.synchronize do
+              @policies[policy_type] = new_policy
+              if cache_the_policy
+               cache(policy_type, policy_jsons[policy_type])
+              end
+            end
+          end
+        end
+      end
+
+    end # end of processPolicyJson
+
+    def cache(policy_name, policy)
+      cache_filename = TCellAgent.configuration.cache_filename
+      if TCellAgent.configuration.app_id
+        cache_filename = cache_filename + '.' + TCellAgent.configuration.app_id
+      end
+      policy_cache = policies_from_cachefile()
+      if !policy_cache
+        policy_cache = {}
+      end
+      policy_cache[policy_name] = policy
+      begin
+        File.write(cache_filename, JSON.dump(policy_cache))
+      rescue Exception => e
+        TCellAgent.logger.error(e.message)
+      end
+    end
+
+    def policies_from_cachefile
+      cache_filename = TCellAgent.configuration.cache_filename
+      if TCellAgent.configuration.app_id
+        cache_filename = cache_filename + '.' + TCellAgent.configuration.app_id
+      end
+      cache_exists = File.exist?(cache_filename)
+      if !cache_exists
+        return nil
+      end
+      begin
+        policy_filedata = open(cache_filename).read
+        policy_jsons = JSON.parse(policy_filedata)
+        if policy_jsons.key?("result")
+          return policy_jsons["result"]
+        end
+        return policy_jsons
+      rescue Exception => e
+        TCellAgent.logger.error(e.message)
+      end
+      return nil
+    end
+
+  end
+end

data/lib/tcell_agent/agent/policy_types.rb

@@ -0,0 +1,42 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require "tcell_agent/policies/content_security_policy"
+require "tcell_agent/policies/clickjacking_policy"
+
+require "tcell_agent/policies/http_tx_policy"
+require "tcell_agent/policies/http_redirect_policy"
+require "tcell_agent/policies/secure_headers_policy"
+require "tcell_agent/policies/honeytokens_policy"
+require "tcell_agent/policies/appsensor_policy"
+require "tcell_agent/policies/add_script_tag_policy"
+require "tcell_agent/policies/login_fraud_policy"
+require "tcell_agent/policies/dataloss_policy"
+
+module TCellAgent
+  class PolicyTypes
+    CSP = "csp-headers"
+    Clickjacking = "clickjacking"
+    SecureHeaders = "secure-headers"
+    HttpTx = "http-tx"
+    HttpRedirect = "http-redirect"
+    AppSensor = "appsensor"
+    AddScriptTag = "add-jsagent-script-tag"
+    HoneyTokens = "exp-honeytokens"
+    LoginFraud = "exp-login-fraud"
+    DataLoss = "dlp"
+
+    ClassMap = {
+      CSP=>TCellAgent::Policies::ContentSecurityPolicy,
+      Clickjacking=>TCellAgent::Policies::ClickjackingPolicy,
+      SecureHeaders=>TCellAgent::Policies::SecureHeadersPolicy,
+      HttpTx=>TCellAgent::Policies::HttpTxPolicy,
+      HttpRedirect=>TCellAgent::Policies::HttpRedirectPolicy,
+      AppSensor=>TCellAgent::Policies::AppSensorPolicy,
+      AddScriptTag=>TCellAgent::Policies::AddScriptTagPolicy,
+      HoneyTokens=>TCellAgent::Policies::HoneytokensPolicy,
+      LoginFraud=>TCellAgent::Policies::LoginFraudPolicy,
+      DataLoss=>TCellAgent::Policies::DataLossPolicy
+    }
+
+  end
+end

data/lib/tcell_agent/agent/static_agent.rb

@@ -0,0 +1,22 @@
+# See the file "LICENSE" for the full license governing this code.
+require 'tcell_agent/sensor_events/metrics'
+
+module TCellAgent
+  class << self
+    attr_accessor :thread_agent
+  end
+  def self.send_event(event)
+    if (self.thread_agent != nil)
+        self.thread_agent.queueSensorEvent(event)
+    end
+  end
+  def self.policy(policy_type)
+    if (self.thread_agent != nil)
+        self.thread_agent.policies.fetch(policy_type, nil)
+    end
+  end
+  def self.increment_route(route_id, response_time)
+    event = TCellAgent::SensorEvents::RequestRouteTimer.new(route_id, response_time)
+    self.send_event(event)
+  end
+end