tcell_agent 0.4.0 → 1.0.0

data/lib/tcell_agent/agent/policy_manager.rb

@@ -14,9 +14,6 @@ require "tcell_agent/policies/clickjacking_policy"
 require "tcell_agent/policies/http_tx_policy"
 require "tcell_agent/policies/http_redirect_policy"
 require "tcell_agent/policies/secure_headers_policy"
-require "tcell_agent/policies/honeytokens_policy"
-require "tcell_agent/policies/appsensor_policy"
-require "tcell_agent/policies/patches_policy"
 
 require "tcell_agent/sensor_events/server_agent"
 
@@ -101,7 +98,7 @@ module TCellAgent
             failure_sleep_time *= 2
           end
 
-          return
+          return [failure_sleep_time, last_poll_time]
 
         elsif policy_jsons.key?("last_timestamp")
           if policy_jsons["last_timestamp"] != 0
@@ -119,7 +116,7 @@ module TCellAgent
 
       rescue TCellAgent::ConfigurationException
         Thread.exit
-      rescue Exception => e
+      rescue StandardError => e
         TCellAgent.logger.error("exception while handling connection: #{e.message}")
         TCellAgent.logger.debug(e.backtrace)
         TCellAgent.logger.debug("Sleeping 30 seconds because the tCell.io request failed...")
@@ -141,17 +138,25 @@ module TCellAgent
           new_policy = policy_class.from_json(policy_jsons[policy_type])
           if new_policy
             @lock.synchronize do
-              old_policy = @policies[policy_type]
               @policies[policy_type] = new_policy
-              if cache_the_policy
-               cache(policy_type, policy_jsons[policy_type])
-              end
-
-              old_policy.free_native_memory if old_policy
             end
           end
         end
       end
+
+      @policies[TCellAgent::PolicyTypes::Rust].update_policies(policy_jsons)
+
+      if cache_the_policy
+        (TCellAgent::PolicyTypes::ClassMap.keys +
+        [TCellAgent::PolicyTypes::AppSensor,
+         TCellAgent::PolicyTypes::Patches,
+         TCellAgent::PolicyTypes::CommandInjection,
+         TCellAgent::PolicyTypes::Regex]).each do |policy_type|
+           @lock.synchronize do
+             cache(policy_type, policy_jsons[policy_type]) if policy_jsons[policy_type]
+           end
+         end
+      end
     end
 
     def cache(policy_name, policy)
@@ -164,7 +169,7 @@ module TCellAgent
           TCellAgent.configuration.agent_home_owner
         )
 
-        f1 = open(cache_filename, File::RDWR|File::CREAT)
+        f1 = File.open(cache_filename, File::RDWR|File::CREAT)
 
         Timeout::timeout(0.100) { f1.flock(File::LOCK_EX) }
 
@@ -184,7 +189,7 @@ module TCellAgent
           end
           policy_cache[policy_name] = policy
           @complete_policy_cache = policy_cache
-        rescue Exception => e
+        rescue StandardError => e
           TCellAgent.logger.warn(e.message)
           if @complete_policy_cache
             policy_cache = @complete_policy_cache
@@ -200,7 +205,7 @@ module TCellAgent
           cache_filename,
           TCellAgent.configuration.agent_home_owner
         )
-      rescue Exception => e
+      rescue StandardError => e
         TCellAgent.logger.warn(e.message)
 
       ensure
@@ -227,7 +232,7 @@ module TCellAgent
         @complete_policy_cache = policy_jsons
         return policy_jsons
 
-      rescue Exception => e
+      rescue StandardError => e
         TCellAgent.logger.warn(e.message)
       end
 

data/lib/tcell_agent/agent/policy_types.rb

@@ -8,12 +8,9 @@ require "tcell_agent/policies/clickjacking_policy"
 require "tcell_agent/policies/http_tx_policy"
 require "tcell_agent/policies/http_redirect_policy"
 require "tcell_agent/policies/secure_headers_policy"
-require "tcell_agent/policies/honeytokens_policy"
-require "tcell_agent/policies/appsensor_policy"
 require "tcell_agent/policies/login_fraud_policy"
 require "tcell_agent/policies/dataloss_policy"
-require "tcell_agent/policies/patches_policy"
-require "tcell_agent/policies/command_injection_policy"
+require "tcell_agent/policies/rust_policies"
 
 module TCellAgent
   class PolicyTypes
@@ -22,12 +19,13 @@ module TCellAgent
     SecureHeaders = "secure-headers"
     HttpTx = "http-tx"
     HttpRedirect = "http-redirect"
-    AppSensor = "appsensor"
-    HoneyTokens = "exp-honeytokens"
     LoginFraud = "login"
     DataLoss = "dlp"
+    AppSensor = "appsensor"
     Patches = "patches"
     CommandInjection = "cmdi"
+    Regex = "regex"
+    Rust = "rust"
 
     ClassMap = {
       CSP=>TCellAgent::Policies::ContentSecurityPolicy,
@@ -35,12 +33,8 @@ module TCellAgent
       SecureHeaders=>TCellAgent::Policies::SecureHeadersPolicy,
       HttpTx=>TCellAgent::Policies::HttpTxPolicy,
       HttpRedirect=>TCellAgent::Policies::HttpRedirectPolicy,
-      AppSensor=>TCellAgent::Policies::AppSensorPolicy,
-      HoneyTokens=>TCellAgent::Policies::HoneytokensPolicy,
       LoginFraud=>TCellAgent::Policies::LoginFraudPolicy,
-      DataLoss=>TCellAgent::Policies::DataLossPolicy,
-      Patches=>TCellAgent::Policies::PatchesPolicy,
-      CommandInjection=>TCellAgent::Policies::CommandInjectionPolicy
+      DataLoss=>TCellAgent::Policies::DataLossPolicy
     }
 
   end

data/lib/tcell_agent/agent/static_agent.rb

@@ -24,7 +24,7 @@ module TCellAgent
   end
 
   # setter
-  def self.thread_agent=some_agent
+  def self.thread_agent=(some_agent)
     @@instance_lock.synchronize do
       @@my_thread_agent = some_agent
     end
@@ -60,4 +60,8 @@ module TCellAgent
   def self.ensure_event_processor_running
     self.thread_agent.ensure_event_processor_running
   end
+
+  def self.is_it_safe_to_send_cmdi_events?
+    self.thread_agent.is_it_safe_to_send_cmdi_events?
+  end
 end

data/lib/tcell_agent/agent.rb

@@ -66,21 +66,23 @@ module TCellAgent
       @@policy_tapi = TCellApi.new
 
       # Agent request thread
-      @policies = {}
+      @policies = {
+        TCellAgent::PolicyTypes::Rust => TCellAgent::Policies::RustPolicies.new
+      }
       @lock = Monitor.new
 
       self.initialize_processor_variables
 
       if TCellAgent.configuration.preload_policy_filename != nil
-        TCellAgent.logger.info("Preloading a policy file");
+        TCellAgent.logger.info("Preloading a policy file")
         begin
-          policy_file = open(TCellAgent.configuration.preload_policy_filename).read
+          policy_file = File.open(TCellAgent.configuration.preload_policy_filename).read
           policy_jsons = JSON.parse(policy_file)
           if policy_jsons.key?("result")
             policy_jsons = policy_jsons["result"]
           end
           processPolicyJson(policy_jsons, false)
-        rescue Exception => e
+        rescue StandardError => e
           TCellAgent.logger.error(e.message)
         end
       end

data/lib/tcell_agent/api.rb

@@ -31,25 +31,23 @@ module TCellAgent
       req['Authorization'] = 'Bearer ' + TCellAgent.configuration.api_key
       begin
         req['TCellAgent'] = "RubyAgent " + TCellAgent::VERSION
-      rescue Exception => e
+      rescue StandardError => e
         TCellAgent.logger.debug("tCell.io Could not add agent string: " + e.message)
       end
 
-      res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') { |http| http.request(req) }
+      res = Net::HTTP.start(uri.hostname, uri.port, :use_ssl => (uri.scheme == 'https')) { |http| http.request(req) }
 
       if res.is_a?(Net::HTTPSuccess)
-        TCellAgent.logger.debug("tCell.io API Response: #{res.body}")
+        TCellAgent.logger.debug("tCell.io API Response: #{res.body}".force_encoding("UTF-8"))
         response_json = JSON.parse(res.body)
         if (response_json && response_json.has_key?("result"))
           return response_json["result"]
         end
-
-        return nil
-
       else
         TCellAgent.logger.error("Received error response while contacting api: #{res.inspect}")
-        return nil
       end
+
+      return nil
     end
 
     def send_event_set(events)
@@ -81,11 +79,11 @@ module TCellAgent
 
       begin
         req['TCellAgent'] = "RubyAgent " + TCellAgent::VERSION
-      rescue Exception => e
+      rescue StandardError => e
         TCellAgent.logger.debug("tCell.io Could not add agent string: " + e.message)
       end
 
-      res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') { |http| http.request(req) }
+      res = Net::HTTP.start(uri.hostname, uri.port, :use_ssl => (uri.scheme == 'https')) { |http| http.request(req) }
 
       TCellAgent.logger.debug("tCell.io SendEvents API Response: #{res.code}")
 

data/lib/tcell_agent/appsensor/meta_data.rb

@@ -8,11 +8,17 @@ module TCellAgent
     class MetaData < TCellAgent::SensorEvents::TCellSensorEvent
 
       attr_accessor :get_dict, :post_dict, :body_dict, :cookie_dict, :path_parameters,
-        :remote_address, :method, :route_id, :session_id, :user_id, :transaction_id
+        :remote_address, :method, :route_id, :session_id, :user_id, :transaction_id, :location
 
       attr_reader :headers_dict
 
-      def initialize(method, remote_address, route_id, session_id, user_id, transaction_id)
+      def initialize(method,
+                     remote_address,
+                     route_id,
+                     session_id,
+                     user_id,
+                     transaction_id,
+                     location)
         @send = false
 
         @method = method
@@ -21,6 +27,7 @@ module TCellAgent
         @session_id = session_id
         @user_id = user_id
         @transaction_id = transaction_id
+        @location = location
 
         @body_dict = {}
         @get_dict = {}
@@ -96,7 +103,7 @@ module TCellAgent
           (header_downcased != "http_cookie" && header_downcased.start_with?('http_')) ||
             ["content_type", "content_length"].include?(header_downcased)
         }.inject({}) { |memo, (k,v)|
-          memo[k.downcase.sub(/^http_/, '').gsub('_', '-')] = v
+          memo[k.downcase.sub(/^http_/, '').tr('_', '-')] = v
           memo
         }
       end
@@ -110,7 +117,7 @@ module TCellAgent
             begin
               # don't enqueue parameter values of unknown type to avoid any serialization issues
               @body_dict = TCellAgent::Utils::Params.flatten(JSON.parse(request_body))
-            rescue
+            rescue JSON::ParserError
               TCellAgent.logger.debug("JSON body parameter parsing failed")
               @body_dict = {}
             end

data/lib/tcell_agent/authlogic.rb

@@ -12,10 +12,10 @@ module TCellAgent
         def getUserFromRequest(request)
           orig_user_id = original_getUserFromRequest(request)
           begin
-            if request.session and request.session.has_key?("user_credentials_id")
+            if request.session && request.session.has_key?("user_credentials_id")
               return request.session["user_credentials_id"].to_s
             end
-          rescue Exception => e
+          rescue StandardError => e
             return orig_user_id
           end
           return orig_user_id
@@ -23,4 +23,4 @@ module TCellAgent
       end
     end
   end
-end
+end

data/lib/tcell_agent/cmdi.rb

@@ -7,11 +7,13 @@ module TCellAgent
     def self.block_command?(cmd)
       TCellAgent::Instrumentation.safe_block("Checking Command Injection Policy") do
         if TCellAgent::Utils::Strings.present?(cmd)
-          command_injection_policy = TCellAgent.policy(TCellAgent::PolicyTypes::CommandInjection)
-          if command_injection_policy && command_injection_policy.enabled
-            request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+          rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+          if rust_policies && rust_policies.cmdi_enabled
+            request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(
+              Thread.current.object_id, {}
+            )
             tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
-            return command_injection_policy.block?(cmd, tcell_context)
+            return rust_policies.block_command?(cmd, tcell_context)
           end
         end
       end

data/lib/tcell_agent/config/unknown_options.rb

@@ -12,6 +12,7 @@ module TCellAgent
           "TCELL_AGENT_APP_ID",
           "TCELL_AGENT_API_KEY",
           "TCELL_HMAC_KEY",
+          "TCELL_PASSWORD_HMAC_KEY",
           "TCELL_AGENT_HOST_IDENTIFIER",
           "TCELL_API_URL",
           "TCELL_INPUT_URL",
@@ -59,6 +60,7 @@ module TCellAgent
                   "host_identifier",
                   "hipaaSafeMode",
                   "hmac_key",
+                  "password_hmac_key",
                   "js_agent_api_base_url",
                   "js_agent_url",
                   "max_csp_header_bytes",
@@ -106,7 +108,7 @@ module TCellAgent
             end
 
           end
-        rescue Exception => exception
+        rescue StandardError => exception
           messages << "Something went wrong verifying config file: #{exception}"
         end
 

data/lib/tcell_agent/configuration.rb

@@ -35,7 +35,6 @@ module TCellAgent
       :cache_filename,
       :js_agent_api_base_url,
       :js_agent_url,
-      :startup_js_agent_url,
       :config_filename,
       :agent_log_dir,
       :max_data_ex_db_records_per_request,
@@ -47,7 +46,8 @@ module TCellAgent
       :log_tag,
       :max_csp_header_bytes,
       :demomode,
-      :allow_payloads
+      :allow_payloads,
+      :password_hmac_key
 
     attr_accessor :disable_all,
       :enabled,
@@ -131,14 +131,11 @@ module TCellAgent
       @enable_intercept_requests = true
 
       @enabled_instrumentations = {
-        doorkeeper: true,
-        devise: true,
-        authlogic: true
+        :doorkeeper => true,
+        :devise => true,
+        :authlogic => true
       }
 
-      @agent_home_dir = File.join(Dir.getwd, "tcell")
-      @config_filename = File.join(Dir.getwd, filename)
-
       @log_file_name = "tcell_agent.log"
 
       @event_batch_size_limit = 50
@@ -150,9 +147,18 @@ module TCellAgent
       @allow_payloads = true
 
       @max_csp_header_bytes = nil
+      @password_hmac_key = nil
+
+      @agent_home_dir = ENV["TCELL_AGENT_HOME"] || File.join(Dir.getwd, "tcell")
+      @config_filename = ENV["TCELL_AGENT_CONFIG"] || File.join(Dir.getwd, filename)
 
-      read_config_using_env
       read_config_from_file(@config_filename)
+      read_config_using_env
+
+      if @demomode
+        @event_batch_size_limit = 2
+        @event_time_limit_seconds = 5
+      end
 
       if ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"]
         puts "tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS is deprecated and will be removed in a future release. Please switch to TCELL_AGENT_ALLOW_PAYLOADS."
@@ -175,13 +181,12 @@ module TCellAgent
       @tcell_api_url ||= "https://api.tcell.io/api/v1"
       @tcell_input_url ||= "https://input.tcell.io/api/v1"
       @js_agent_api_base_url ||= nil
-      @js_agent_url ||= "https://api.tcell.io/tcellagent.min.js"
-      @startup_js_agent_url = @js_agent_url
+      @js_agent_url ||= "https://jsagent.tcell.io/tcellagent.min.js"
 
       if (@host_identifier == nil)
         begin
           @host_identifier = (Socket.gethostname() || "localhost")
-        rescue Exception
+        rescue StandardError
           @host_identifier = "host_identifier_not_found"
         end
       end
@@ -200,23 +205,17 @@ module TCellAgent
     end
 
     def read_config_using_env
-      @app_id = ENV["TCELL_AGENT_APP_ID"]
-      @api_key = ENV["TCELL_AGENT_API_KEY"]
-      @hmac_key = ENV["TCELL_HMAC_KEY"]
+      @app_id = ENV["TCELL_AGENT_APP_ID"] || @app_id
+      @api_key = ENV["TCELL_AGENT_API_KEY"] || @api_key
+      @hmac_key = ENV["TCELL_HMAC_KEY"] || @hmac_key
+      @password_hmac_key = ENV["TCELL_PASSWORD_HMAC_KEY"] || @password_hmac_key
       @host_identifier = ENV["TCELL_AGENT_HOST_IDENTIFIER"] || @host_identifier
-      @tcell_api_url = ENV["TCELL_API_URL"]
-      @tcell_input_url = ENV["TCELL_INPUT_URL"]
+      @tcell_api_url = ENV["TCELL_API_URL"] || @tcell_api_url
+      @tcell_input_url = ENV["TCELL_INPUT_URL"] || @tcell_input_url
       @demomode = ENV["TCELL_DEMOMODE"] || @demomode
 
-      @agent_home_dir = ENV["TCELL_AGENT_HOME"] || @agent_home_dir
-      @agent_home_owner = ENV["TCELL_AGENT_HOME_OWNER"]
-      @agent_log_dir = ENV["TCELL_AGENT_LOG_DIR"]
-      @config_filename = ENV["TCELL_AGENT_CONFIG"] || @config_filename
-
-      if @demomode
-        @event_batch_size_limit = 2
-        @event_time_limit_seconds = 5
-      end
+      @agent_home_owner = ENV["TCELL_AGENT_HOME_OWNER"] || @agent_home_owner
+      @agent_log_dir = ENV["TCELL_AGENT_LOG_DIR"] || @agent_log_dir
     end
 
     def read_config_from_file(filename)
@@ -235,9 +234,8 @@ module TCellAgent
             # Required
             app_data = config["applications"][0] #Default
             @version = 1
-            @app_id ||= app_data["app_id"]
-            @app_id ||= app_data["name"]
-            @api_key ||= app_data["api_key"]
+            @app_id = app_data["app_id"]
+            @api_key = app_data["api_key"]
 
             # Optional
             @preload_policy_filename = app_data.fetch("preload_policy_filename", nil)
@@ -253,12 +251,9 @@ module TCellAgent
             @fetch_policies_from_tcell = app_data.fetch("fetch_policies_from_tcell", @fetch_policies_from_tcell)
             @instrument_for_events = app_data.fetch("instrument_for_events", @instrument_for_events)
 
-            @agent_home_owner = app_data.fetch("agent_home_owner",@agent_home_owner)
+            @agent_home_owner = app_data.fetch("agent_home_owner", @agent_home_owner)
 
             @logging_options = app_data.fetch("logging_options", {})
-            # DEPRECATED: this was incorrectly placed here. Keep it here until we can
-            # be sure that no customers are relying on this
-            @agent_log_dir = @logging_options.fetch("log_dir", @agent_log_dir)
             @agent_log_dir = app_data.fetch("log_dir", @agent_log_dir)
             @log_file_name = @logging_options.fetch("filename", @log_file_name)
 
@@ -267,12 +262,18 @@ module TCellAgent
 
             @max_csp_header_bytes = app_data.fetch("max_csp_header_bytes", @max_csp_header_bytes)
 
-            @allow_payloads =
-              app_data.fetch('allow_unencrypted_appsensor_payloads', @allow_payloads)
-            @allow_payloads =
-              app_data.fetch('allow_unencrypted_appfirewall_payloads', @allow_payloads)
-            @allow_payloads =
-              app_data.fetch('allow_payloads', @allow_payloads)
+            @allow_payloads = app_data.fetch(
+              'allow_unencrypted_appsensor_payloads',
+              @allow_payloads
+            )
+            @allow_payloads = app_data.fetch(
+              'allow_unencrypted_appfirewall_payloads',
+              @allow_payloads
+            )
+            @allow_payloads = app_data.fetch(
+              'allow_payloads',
+              @allow_payloads
+            )
 
             data_exposure = app_data.fetch('data_exposure', {})
             @max_data_ex_db_records_per_request = data_exposure.fetch('max_data_ex_db_records_per_request', @max_data_ex_db_records_per_request)
@@ -282,8 +283,11 @@ module TCellAgent
             @reverse_proxy = app_data.fetch('reverse_proxy', @reverse_proxy)
             @reverse_proxy_ip_address_header = app_data.fetch('reverse_proxy_ip_address_header', @reverse_proxy_ip_address_header)
 
-            @host_identifier = @host_identifier || app_data.fetch("host_identifier", @host_identifier)
-            @hmac_key ||= app_data["hmac_key"] # if not already set
+            @host_identifier = app_data.fetch("host_identifier", @host_identifier)
+            @hmac_key = app_data.fetch("hmac_key", @hmac_key)
+
+            @password_hmac_key = app_data.fetch("password_hmac_key", @password_hmac_key)
+
             @uuid = SecureRandom.uuid
             if (@uuid == nil)
               @uuid = "secure-random-failed"
@@ -296,20 +300,14 @@ module TCellAgent
               @js_agent_url = app_data["js_agent_url"]
             end
 
-            if @demomode != true
-              @demomode = app_data.fetch('demomode', false)
-            end
-            if @demomode
-              @event_batch_size_limit = 2
-              @event_time_limit_seconds = 5
-            end
+            @demomode = app_data.fetch('demomode', @demomode)
           else
             puts " ********* ********* ********* *********"
             puts "* tCell.io                               *"
             puts "* Unsupported config file version        *"
             puts " ********* ********* ********* *********"
           end
-        rescue Exception => e
+        rescue StandardError => e
           puts " ********* ********* ********* *********"
           puts "* tCell.io                               *"
           puts "* Could not load config file             *"

data/lib/tcell_agent/devise.rb

@@ -15,7 +15,7 @@ module TCellAgent
         def getUserFromRequest(request)
           orig_user_id = original_getUserFromRequest(request)
           begin
-            if request.session and request.session.has_key?("warden.user.user.key")
+            if request.session && request.session.has_key?("warden.user.user.key")
               userkey = request.session["warden.user.user.key"]
               if (userkey.length == 2)
                 user_id = userkey[0][0]
@@ -26,7 +26,7 @@ module TCellAgent
                 return user_id.to_s
               end
             end
-          rescue Exception => e
+          rescue StandardError => e
             return orig_user_id
           end
           return orig_user_id