tcell_agent 0.2.29.rc2 → 0.2.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72bdaa2198617a42be424cb88f1944cfa867ee7b
-  data.tar.gz: 876dccbc46bcfcb9d927369744db83741bb03e3e
+  metadata.gz: ef449abceb02a45432746a68e50989438292c9c0
+  data.tar.gz: 1b8ce68123c46d3a12cbc2262e4125948973ecf5
 SHA512:
-  metadata.gz: 1940cc2ca2ca9d3d2cafec1b3addc5f1d2b04140391d53923ae7f1230594baf5291adf9ccc596306f3a68397a0a783d174a4e125b2a9e185d65a0555499a4a83
-  data.tar.gz: 9163fd10bae3bc0c05de4ff1af96040dea0068851598612caad8ee8d55ca5720d79d7cad9e570dc8329c4a5c844155655a80c5d2402ec6c69aa3b12c691ea3f6
+  metadata.gz: ac50286364897b7743cfd3dcfa9ed539f425142311b2eb675167a3362a84ba23c581ddb72342923723c35eeb3fbef7f4d0bc3b10ef4ff4ad0490b44fcd68a2f9
+  data.tar.gz: a0b0b0e374279de26805a4710f13217c7682f58901621dd8b430f64d6a322aef4b6aa15f4441c2c26043249dca75b1b5cb4f41525faeac84c22264438c719475

data/bin/tcell_agent

@@ -54,7 +54,7 @@ global = OptionParser.new do |opts|
   opts.separator subtext
 end
 
-subcommands = { 
+subcommands = {
   'setup' => OptionParser.new do |opts|
       opts.banner = "Usage: setup"
   end,
@@ -150,7 +150,7 @@ elsif (command == 'loglevel')
     end
     loglevel = loglevel.upcase
     if ["ERROR","WARN","INFO","DEBUG"].include?(loglevel)
-      logging_options["enabled"] = true    
+      logging_options["enabled"] = true
       logging_options["level"] = loglevel
     else
       puts subcommands[command]
@@ -239,6 +239,18 @@ elsif (command == 'test')
   end
   puts "passed"
 
+  printf "%-50s", "Check for unknown settings... "
+  require "tcell_agent/config/unknown_options"
+  messages = TCellAgent::Config::Validate.get_unknown_options(config_hash)
+  if messages.size > 0
+    puts "failed"
+    messages.each do |message|
+        puts message
+    end
+    Kernel.exit(1)
+  end
+  puts "passed"
+
   printf "%-50s", "Requiring configuration library... "
   require 'tcell_agent/configuration'
   require 'tcell_agent/api'
@@ -246,11 +258,11 @@ elsif (command == 'test')
 
   printf "%-50s", "Make test API call for policies... "
   api = TCellAgent::TCellApi.new
-  api.pollAPI
+  api.poll_api()
   puts "passed"
 
   printf "%-50s", "Sending a Test event... "
-  send_succeeded = api.sendEventSet([])
+  send_succeeded = api.send_event_set([])
   if !send_succeeded
     puts "failed"
     Kernel.exit(1)

data/lib/tcell_agent/agent/event_processor.rb

@@ -80,7 +80,7 @@ module TCellAgent
           end
           events_to_send.push( sessions_to_send )
         end
-        success = tapi.sendEventSet(events_to_send)
+        success = tapi.send_event_set(events_to_send)
         if ( success == false )
           ensured_events = events_to_send.find_all{|item| item.ensure == true }
           @event_dispatch_monitor.synchronize {
@@ -129,16 +129,13 @@ module TCellAgent
                     self.send_dispatch_events(tapi)
                   end
                 end
-              rescue ThreadError => thread_error
-                TCellAgent.logger.error("Thread error while processing events: #{thread_error.message}")
-                TCellAgent.logger.debug(thread_error.backtrace)
+              rescue ThreadError
                 last_run_time = Time.now
                 @event_dispatch_monitor.synchronize {
                   @dispatchEvents = []
                 }
               end
             rescue TCellAgent::ConfigurationException
-              TCellAgent.logger.error("Exiting event processor. App is not configured properly.")
               Thread.exit
             rescue Exception => e
               last_run_time = Time.now
@@ -199,7 +196,6 @@ module TCellAgent
     def queue_forked_event(event)
       begin
         self.ensure_fork_event_thread_running
-        TCellAgent.logger.debug("fork_event_queue length: #{@fork_event_queue.length}")
         if (@fork_event_queue.length() > 100)
           TCellAgent.logger.debug("Dropping (forked) event because queue full")
           return
@@ -256,7 +252,6 @@ module TCellAgent
       return if TCellAgent.configuration.should_consume_event? == false
       begin
         self.ensure_metrics_event_thread_running
-        TCellAgent.logger.debug("metrics_event_queue length: #{@metrics_event_queue.length}")
         if (@metrics_event_queue.length() > 100)
           TCellAgent.logger.debug("Dropping (forked) metric because queue full")
           return
@@ -277,7 +272,6 @@ module TCellAgent
 
         begin
           self.ensure_event_processor_running
-          TCellAgent.logger.debug("eventQueue length: #{@eventQueue.size}")
           @event_queue_monitor.synchronize {
             @eventQueue.push(event, 10)
           }

data/lib/tcell_agent/agent/fork_pipe_manager.rb

@@ -108,11 +108,9 @@ module TCellAgent
       @@event_pipe_manager.is_parent?
     end
     def self.send_to_metrics_pipe(hash_value)
-      TCellAgent.logger.debug("Send metric to parent")
       @@metrics_pipe_manager.send_to_parent(hash_value)
     end
     def self.send_to_event_pipe(event)
-      TCellAgent.logger.debug("Send event to parent")
       @@event_pipe_manager.send_to_parent(event)
     end
 

data/lib/tcell_agent/agent/policy_manager.rb

@@ -73,14 +73,16 @@ module TCellAgent
           loop do
             failure_sleep_time, last_poll_time = policy_polling_iteration(failure_sleep_time, last_poll_time)
 
-            if (Time.now - last_run) < 5
-              random = Random.new
-              sleeptime = sleep(random.rand(5..20))
-              TCellAgent.logger.debug("Rate limiting: sleeping #{sleeptime} seconds")
-              sleep(sleeptime)
-            end
+            unless TCellAgent.configuration.demomode
+              if (Time.now - last_run) < 2
+                random = Random.new
+                sleeptime = sleep(random.rand(5..20))
+                TCellAgent.logger.debug("Rate limiting: sleeping #{sleeptime} seconds")
+                sleep(sleeptime)
+              end
 
-            last_run = Time.now
+              last_run = Time.now
+            end
           end
         end
       end
@@ -88,7 +90,7 @@ module TCellAgent
 
     def policy_polling_iteration(failure_sleep_time, last_poll_time)
       begin
-        policy_jsons = @@policy_tapi.pollAPI(last_poll_time)
+        policy_jsons = @@policy_tapi.poll_api(last_poll_time)
 
         if policy_jsons == nil
           TCellAgent.logger.error("Policy was nil. Sleeping for #{failure_sleep_time}")
@@ -115,16 +117,8 @@ module TCellAgent
 
         processPolicyJson(policy_jsons)
 
-      rescue RestClient::Exception => rce
-        TCellAgent.logger.error("Received error response while contacting api [#{rce.http_code}]: #{rce.message}")
-        TCellAgent.logger.debug(rce.backtrace)
-        TCellAgent.logger.debug("Sleeping #{failure_sleep_time} seconds because the request failed...")
-        sleep(failure_sleep_time)
-
-        if failure_sleep_time < 480
-          failure_sleep_time *= 2
-        end
-
+      rescue TCellAgent::ConfigurationException
+        Thread.exit
       rescue Exception => e
         TCellAgent.logger.error("exception while handling connection: #{e.message}")
         TCellAgent.logger.debug(e.backtrace)

data/lib/tcell_agent/api.rb

@@ -1,72 +1,95 @@
 # encoding: utf-8
 # See the file "LICENSE" for the full license governing this code.
 require 'json'
-require 'rest-client'
 require 'tcell_agent/logger'
 require 'tcell_agent/configuration'
 require 'tcell_agent/version'
 require 'date'
 
+require 'net/http'
+
 module TCellAgent
   class TCellApi
 
     def initialize
     end
 
-    def pollAPI(last_timestamp=nil)
+    def poll_api(last_timestamp=nil)
+      if !TCellAgent.configuration || !TCellAgent.configuration.tcell_api_url || !TCellAgent.configuration.app_id
+        raise TCellAgent::ConfigurationException.new("Config Information Not Found, can't poll for policy updates")
+      end
+
       full_url = TCellAgent.configuration.tcell_api_url + "/app/" + TCellAgent.configuration.app_id + "/update"
       if (last_timestamp && last_timestamp != "")
         full_url = full_url + "?last_timestamp=" + last_timestamp.to_s
       end
 
       TCellAgent.logger.debug "tCell.io API Request: " + full_url
-      request_headers = {
-        :Authorization => 'Bearer ' + TCellAgent.configuration.api_key
-      }
 
+      uri = URI(full_url)
+      req = Net::HTTP::Get.new(uri.request_uri)
+      req['Authorization'] = 'Bearer ' + TCellAgent.configuration.api_key
       begin
-        request_headers[:TCellAgent] = "RubyAgent " + TCellAgent::VERSION
+        req['TCellAgent'] = "RubyAgent " + TCellAgent::VERSION
       rescue Exception => e
         TCellAgent.logger.debug("tCell.io Could not add agent string: " + e.message)
       end
 
-      response = RestClient.get full_url,request_headers
-      TCellAgent.logger.debug "tCell.io API Response: " + response
-      response_json = JSON.parse(response)
-      if (response_json && response_json.has_key?("result"))
-        return response_json["result"]
+      res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') { |http| http.request(req) }
+
+      if res.is_a?(Net::HTTPSuccess)
+        TCellAgent.logger.debug("tCell.io API Response: #{res.body}")
+        response_json = JSON.parse(res.body)
+        if (response_json && response_json.has_key?("result"))
+          return response_json["result"]
+        end
+
+        return nil
+
+      else
+        TCellAgent.logger.error("Received error response while contacting api: #{res.inspect}")
+        return nil
       end
-      # else result was null and no new information exists...
-      return nil
     end
 
-    def sendEventSet(events)
+    def send_event_set(events)
       if !TCellAgent.configuration || !TCellAgent.configuration.tcell_input_url || !TCellAgent.configuration.app_id
         raise TCellAgent::ConfigurationException.new("Config Information Not Found, can't send events")
       end
+
       if (events == nil)
         return false
       end
-      eventset = { "uuid"=>TCellAgent.configuration.uuid,
-                   "hostname"=>TCellAgent.configuration.host_identifier,
-                   "events"=>events }
+
+      eventset = {"uuid" => TCellAgent.configuration.uuid,
+                  "hostname" => TCellAgent.configuration.host_identifier,
+                  "events" => events }
       TCellAgent.logger.debug("Sending #{JSON.dump(eventset)}")
-      full_url = TCellAgent.configuration.tcell_input_url + "/app/" + TCellAgent.configuration.app_id + "/server_agent"
+      full_url = TCellAgent.configuration.tcell_input_url +
+        "/app/" +
+        TCellAgent.configuration.app_id +
+        "/server_agent"
 
       TCellAgent.logger.debug("tCell.io SendEvents API Request: " + full_url)
-      request_headers = {
-        :Authorization => 'Bearer ' + TCellAgent.configuration.api_key, 
-        :content_type => "application/json", 
-        :accept => "application/json",
-      }
+
+      uri = URI(full_url)
+      req = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
+      req.body = JSON.dump(eventset)
+      req['Authorization'] = 'Bearer ' + TCellAgent.configuration.api_key
+      req['Content-Type'] = 'application/json'
+      req['Accept'] = 'application/json'
+
       begin
-        request_headers[:TCellAgent] = "RubyAgent " + TCellAgent::VERSION
+        req['TCellAgent'] = "RubyAgent " + TCellAgent::VERSION
       rescue Exception => e
         TCellAgent.logger.debug("tCell.io Could not add agent string: " + e.message)
       end
-      response = RestClient.post full_url, JSON.dump(eventset), request_headers
-      TCellAgent.logger.debug("tCell.io SendEvents API Response: " + response.code.to_s)
-      return response.code == 200
+
+      res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') { |http| http.request(req) }
+
+      TCellAgent.logger.debug("tCell.io SendEvents API Response: #{res.code}")
+
+      return res.is_a?(Net::HTTPSuccess)
     end
 
     def valid_header?(str)

data/lib/tcell_agent/appsensor/injections_reporter.rb

@@ -20,11 +20,12 @@ module TCellAgent
         COOKIE_PARAM => 'cookie'
       }
 
-      attr_accessor :injections_matcher, :payloads_policy
+      attr_accessor :injections_matcher, :payloads_policy, :collect_full_uri
 
-      def initialize(injections_matcher, payloads_policy)
+      def initialize(injections_matcher, payloads_policy, collect_full_uri)
         @injections_matcher = injections_matcher
         @payloads_policy = payloads_policy
+        @collect_full_uri = collect_full_uri
       end
 
       def check(appsensor_meta)
@@ -51,14 +52,15 @@ module TCellAgent
             vuln_param,
             meta,
             payload,
-            pattern)
+            pattern,
+            @collect_full_uri)
         end
       end
 
-      def self.from_json(version, data_json, payloads_policy)
+      def self.from_json(version, data_json, payloads_policy, collect_full_uri=false)
         injections_matcher = InjectionsMatcher.from_json(version, data_json)
 
-        InjectionsReporter.new(injections_matcher, payloads_policy)
+        InjectionsReporter.new(injections_matcher, payloads_policy, collect_full_uri)
       end
 
     end

data/lib/tcell_agent/appsensor/sensor.rb

@@ -5,7 +5,8 @@ module TCellAgent
 
     class Sensor
       class << self
-        def send_event(appsensor_meta, detection_point, parameter, meta, payload, pattern)
+        def send_event(appsensor_meta, detection_point, parameter, meta,
+                       payload, pattern, collect_full_uri)
           event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
             appsensor_meta.location,
             detection_point,
@@ -17,13 +18,15 @@ module TCellAgent
             appsensor_meta.session_id,
             appsensor_meta.user_id,
             payload,
-            pattern
+            pattern,
+            collect_full_uri
           )
 
           TCellAgent.send_event(event)
         end
 
-        def send_event_from_tcell_data(tcell_data, detection_point, parameter, meta)
+        def send_event_from_tcell_data(tcell_data, detection_point, parameter,
+                                       meta, collect_full_uri)
           payload = pattern = nil
           event = TCellAgent::SensorEvents::TCellAppSensorEvent.new(
             tcell_data.uri,
@@ -36,7 +39,8 @@ module TCellAgent
             tcell_data.session_id,
             tcell_data.user_id,
             payload,
-            pattern
+            pattern,
+            collect_full_uri
           )
 
           TCellAgent.send_event(event)

data/lib/tcell_agent/config/unknown_options.rb

@@ -0,0 +1,116 @@
+require 'set'
+
+module TCellAgent
+  module Config
+    module Validate
+
+      def self.get_unknown_options(config_json)
+        messages = []
+
+        known_tcell_env_vars = Set.new([
+          "TCELL_AGENT_SERVER", # this is only meant for specs
+          "TCELL_AGENT_APP_ID",
+          "TCELL_AGENT_API_KEY",
+          "TCELL_HMAC_KEY",
+          "TCELL_AGENT_HOST_IDENTIFIER",
+          "TCELL_API_URL",
+          "TCELL_INPUT_URL",
+          "TCELL_DEMOMODE",
+          "TCELL_AGENT_HOME",
+          "TCELL_AGENT_LOG_DIR",
+          "TCELL_AGENT_CONFIG",
+          "TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS",
+          "TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS",
+          "TCELL_AGENT_HOME_OWNER"])
+
+        ENV.keys.each do |environment_key|
+          if environment_key =~ /^TCELL_/ && !known_tcell_env_vars.include?(environment_key)
+            messages << "Unrecognized environment parameter (TCELL_*) found: #{environment_key}"
+          end
+        end
+
+        begin
+          key_differences = []
+
+          if config_json
+            first_level_keys = ["version", "applications"]
+
+            key_differences = config_json.keys - first_level_keys
+
+            applications = config_json.fetch("applications", nil)
+            if applications
+
+              if applications.size > 1
+                messages << "Multiple applications detected in config file"
+
+              elsif applications.size == 1
+                application = applications[0]
+
+                second_level_keys = [
+                  "name",
+                  "app_id",
+                  "api_key",
+                  "fetch_policies_from_tcell",
+                  "preload_policy_filename",
+                  "log_dir",
+                  "tcell_api_url",
+                  "tcell_input_url",
+                  "host_identifier",
+                  "hipaaSafeMode",
+                  "hmac_key",
+                  "js_agent_api_base_url",
+                  "js_agent_url",
+                  "max_csp_header_bytes",
+                  "event_batch_size_limit",
+                  "allow_unencrypted_appsensor_payloads",
+                  "allow_unencrypted_appfirewall_payloads",
+                  "reverse_proxy",
+                  "reverse_proxy_ip_address_header",
+                  "demomode",
+                  "logging_options",
+                  "data_exposure",
+                  "disable_all",
+                  "enabled",
+                  "enable_event_manager",
+                  "enable_event_consumer",
+                  "enable_policy_polling",
+                  "enable_instrumentation",
+                  "enable_intercept_requests",
+                  "instrument_for_events",
+                  "agent_home_owner",
+                  "enabled_instrumentations"]
+
+                key_differences = key_differences + (application.keys - second_level_keys)
+
+                if application.fetch("logging_options", nil)
+                  logging_options = application["logging_options"]
+                  key_differences = key_differences + (logging_options.keys - ["enabled", "level", "filename"])
+                end
+
+                if application.fetch("data_exposure", nil)
+                  data_exposure = application["data_exposure"]
+                  key_differences = key_differences + (data_exposure.keys - ["max_data_ex_db_records_per_request"])
+                end
+
+                if application.fetch("enabled_instrumentations", nil)
+                  enabled_instrumentations = application["enabled_instrumentations"]
+                  key_differences = key_differences + (enabled_instrumentations.keys - ["doorkeeper", "devise", "authlogic"])
+                end
+              end
+            end
+
+            key_differences.each do |key|
+              messages << "Unrecognized config setting key: #{key}"
+            end
+
+          end
+        rescue Exception => exception
+          messages << "Something went wrong verifying config file: #{exception}"
+        end
+
+        messages
+      end
+
+    end
+  end
+end