tcell_agent 0.2.29.rc2 → 0.2.29

data/lib/tcell_agent/policies/appsensor/sqli_sensor.rb

@@ -1,3 +1,4 @@
+require 'libinjection/libinjection'
 require 'tcell_agent/policies/appsensor/injection_sensor'
 
 module TCellAgent
@@ -21,11 +22,8 @@ module TCellAgent
       end
 
       def find_vulnerability(param_name, param_value)
-        if @libinjection
-          require 'libinjection/libinjection'
-          if Libinjection.is_sqli(param_value) == 1
-            return {"param" => param_name, "value" => param_value, "pattern" => "li"}
-          end
+        if @libinjection && Libinjection.is_sqli(param_value) == 1
+          return {"param" => param_name, "value" => param_value, "pattern" => "li"}
         end
 
         super(param_name, param_value)

data/lib/tcell_agent/policies/appsensor/user_agent_sensor.rb

@@ -6,16 +6,18 @@ module TCellAgent
     class UserAgentSensor
       DP_CODE = "uaempty"
 
-      attr_accessor :enabled, :empty_enabled, :excluded_route_ids
+      attr_accessor :enabled, :empty_enabled, :excluded_route_ids, :collect_full_uri
 
       def initialize(policy_json=nil)
         @enabled = false
         @empty_enabled = false
         @excluded_route_ids = {}
+        @collect_full_uri = false
 
         if policy_json
           @enabled = policy_json.fetch("enabled", false)
           @empty_enabled = policy_json.fetch("empty_enabled", false)
+          @collect_full_uri = policy_json.fetch("collect_full_uri", @collect_full_uri)
 
           policy_json.fetch("exclude_routes", []).each do |excluded_route|
             @excluded_route_ids[excluded_route] = true
@@ -30,7 +32,9 @@ module TCellAgent
 
         user_agent = appsensor_meta.user_agent
         if !user_agent || user_agent.strip == ""
-          TCellAgent::AppSensor::Sensor.send_event(appsensor_meta, DP_CODE, nil, nil, nil, nil)
+          TCellAgent::AppSensor::Sensor.send_event(
+            appsensor_meta, DP_CODE, nil, nil, nil, nil, @collect_full_uri
+          )
         end
       end
 

data/lib/tcell_agent/policies/appsensor/xss_sensor.rb

@@ -1,3 +1,4 @@
+require 'libinjection/libinjection'
 require 'tcell_agent/policies/appsensor/injection_sensor'
 
 module TCellAgent
@@ -21,11 +22,8 @@ module TCellAgent
       end
 
       def find_vulnerability(param_name, param_value)
-        if @libinjection
-          require 'libinjection/libinjection'
-          if Libinjection.is_xss(param_value) == 1
-            return {"param" => param_name, "value" => param_value, "pattern" => "li"}
-          end
+        if @libinjection && Libinjection.is_xss(param_value) == 1
+          return {"param" => param_name, "value" => param_value, "pattern" => "li"}
         end
 
         super(param_name, param_value)

data/lib/tcell_agent/policies/appsensor_policy.rb

@@ -149,26 +149,31 @@ module TCellAgent
           if policy_json["version"] && policy_json["version"] == 2
             if data_json
               sensors_json = data_json.fetch("sensors", {})
+
               if sensors_json.empty?
                 sensor_policy.enabled = false
 
               else
                 sensor_policy.enabled = true
 
+                options_hash = data_json.fetch("options", {})
+                collect_full_uri = options_hash.fetch("uri_options", {}).fetch("collect_full_uri", false)
+
                 DETECTION_POINTS_V2_NON_INJECTION.each do |sensor_name, sensor_class|
                   settings = sensors_json.fetch(sensor_name, {})
                   updated_settings = {
-                    "enabled" => sensors_json.has_key?(sensor_name)
+                    "enabled" => sensors_json.has_key?(sensor_name),
+                    "collect_full_uri" => collect_full_uri
                   }.merge(settings)
 
-                  sensor_policy.options[sensor_name] = sensor_class.new(updated_settings)
+                  sensor_policy.options[sensor_name] =
+                    sensor_class.new(updated_settings)
                 end
 
-                payloads_policy = PayloadsPolicy.from_json(
-                  data_json.fetch("options", {})
-                )
+                payloads_policy = PayloadsPolicy.from_json(options_hash)
                 sensor_policy.injections_reporter =
-                  TCellAgent::AppSensor::InjectionsReporter.from_json(2, sensors_json, payloads_policy)
+                  TCellAgent::AppSensor::InjectionsReporter.from_json(
+                    2, sensors_json, payloads_policy, collect_full_uri)
               end
             end
 

data/lib/tcell_agent/policies/content_security_policy.rb

@@ -40,11 +40,6 @@ module TCellAgent
           self.raw_value = value
           self.report_uri = report_uri
         end
-        def self.jhash(str)
-          str.each_char.reduce(0) do |result, char|
-            [((result << 5) - result) + char.ord].pack('L').unpack('l').first
-          end
-        end
         def value(transaction_id=nil, route_id=nil, session_id=nil, user_id=nil)
           if !self.report_uri
             return self.raw_value
@@ -66,7 +61,7 @@ module TCellAgent
             end
             report_uri = uri.to_s
             if self.policy_id
-              checksum = ContentSecurityPolicyHeader.jhash(self.policy_id + report_uri)
+              checksum = TCellAgent::Utils::Strings.java_hashcode(self.policy_id + report_uri)
               if new_query_ar != []
                 report_uri = report_uri + "&"
               else
@@ -85,17 +80,27 @@ module TCellAgent
       attr_accessor :policy_id
       attr_accessor :js_agent_api_key
 
-      def each(transaction_id=nil, route_id=nil, hmac_session_id=nil, user_id=nil, &block)
-        result = []
-        headers.each do | header |
+      def each_header_pair(transaction_id=nil, route_id=nil, hmac_session_id=nil, user_id=nil, path=nil)
+        max_csp_header_bytes = TCellAgent.configuration.max_csp_header_bytes
+
+        headers.each do |header|
           header_value = header.value(transaction_id, route_id, hmac_session_id)
-          header_names = ContentSecurityPolicy.cspHeadersForType(header.type)
-          header_names.each do | header_name |
-            result.push( {"name"=>header_name, "value"=>header_value} )
-          end #doloop
+
+          if !max_csp_header_bytes || header_value.bytesize <= max_csp_header_bytes
+            header_names = ContentSecurityPolicy.cspHeadersForType(header.type)
+            header_names.each do | header_name |
+              yield(header_name, header_value)
+            end
+
+          else
+            TCellAgent.logger.warn(
+              "[RouteID=#{route_id},Path=#{path}] CSP header(#{header_value.bytesize}) " +
+              "is bigger than configured max_csp_header_bytes(#{max_csp_header_bytes})"
+            )
+          end
         end
-        result.each(&block)
       end
+
       def self.from_json(policy_json)
         if (!policy_json)
           return nil

data/lib/tcell_agent/rails/dlp.rb

@@ -127,7 +127,7 @@ module TCellAgent
               if appsensor_policy
                 request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
                 tcell_data = request_env[TCellAgent::Instrumentation::TCELL_ID]
-                if tcell_data && e.is_a?(ActiveRecord::StatementInvalid)
+                if tcell_data && result.is_a?(ActiveRecord::StatementInvalid)
                   appsensor_policy.sql_exception_detected(tcell_data, result)
                 end
               end

data/lib/tcell_agent/rails/middleware/headers_middleware.rb

@@ -57,12 +57,15 @@ module TCellAgent
               content_security_policy = TCellAgent.policy(TCellAgent::PolicyTypes::CSP)
 
               if content_security_policy
-                content_security_policy.each(
-                  request.env[TCellAgent::Instrumentation::TCELL_ID].transaction_id,
-                  request.env[TCellAgent::Instrumentation::TCELL_ID].route_id,
-                  request.env[TCellAgent::Instrumentation::TCELL_ID].hmac_session_id,
-                  request.env[TCellAgent::Instrumentation::TCELL_ID].user_id) do | header_pair |
-                  headers[header_pair["name"]] = header_pair["value"]
+                tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
+                content_security_policy.each_header_pair(
+                  tcell_context.transaction_id,
+                  tcell_context.route_id,
+                  tcell_context.hmac_session_id,
+                  tcell_context.user_id,
+                  tcell_context.path
+                ) do |header_name, header_value|
+                  headers[header_name] = header_value
                 end
               end
               response = [status, headers, active_response]
@@ -187,7 +190,7 @@ module TCellAgent
               appsensor_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AppSensor)
               if appsensor_policy
                 event = TCellAgent::SensorEvents::AppSensorMetaEvent.build(
-                  request, content_length, status_code.to_i, response_headers
+                  request, content_length, status_code, response_headers
                 )
                 TCellAgent.send_event(event)
 

data/lib/tcell_agent/rails/on_start.rb

@@ -8,7 +8,6 @@ require 'tcell_agent/configuration'
 
 require 'tcell_agent/rails/routes'
 require 'tcell_agent/rails/dlp/process_request'
-require 'tcell_agent/rails/tracing'
 
 TCellAgent::Instrumentation::Rails.send_language_info
 TCellAgent::Instrumentation::Rails.send_framework_info

data/lib/tcell_agent/sensor_events/appsensor_event.rb

@@ -12,11 +12,12 @@ module TCellAgent
                      remote_addr,
                      param,
                      route_id,
-                     meta=nil,
-                     hmac_session_id=nil,
-                     user_id=nil,
-                     payload=nil,
-                     pattern=nil)
+                     meta,
+                     hmac_session_id,
+                     user_id,
+                     payload,
+                     pattern,
+                     collect_full_uri)
         super("as")
         self["dp"] = detection_point
         self["param"] = param.to_s if param
@@ -30,6 +31,7 @@ module TCellAgent
         self["pattern"] = pattern if pattern
         self["meta"] = meta if meta
         self["rid"] = route_id if route_id
+        self["full_uri"] = location if collect_full_uri && location
       end
 
       def post_process

data/lib/tcell_agent/sinatra.rb

@@ -8,18 +8,15 @@ require 'tcell_agent/instrumentation'
 module TCellAgent
     class Sinatra::Response
       include Sinatra
+
       alias_method :original_finish, :finish
       def finish
         status, headers, response = original_finish
-        
         TCellAgent::Instrumentation.safe_block("Setting CSP Headers") {
           content_security_policy = TCellAgent.policy(TCellAgent::PolicyTypes::CSP)
           if content_security_policy
-            content_security_policy.each(
-              nil, 
-              nil, 
-              nil) do | header_pair | 
-              headers[header_pair["name"]] = header_pair["value"]
+            content_security_policy.each_header_pair do |header_name, header_value|
+              headers[header_name] = header_value
             end
           end
         }

data/lib/tcell_agent/start_background_thread.rb

@@ -2,12 +2,6 @@
 require 'tcell_agent/configuration'
 
 if (TCellAgent.configuration.disable_all == false)
-  require 'objspace'
-
-  if ObjectSpace.respond_to?(:trace_object_allocations_start)
-    ObjectSpace.trace_object_allocations_start
-  end
-
   require 'tcell_agent/logger'
   require 'tcell_agent/agent'
   require 'thread'
@@ -18,7 +12,6 @@ if (TCellAgent.configuration.disable_all == false)
     def self.run_instrumentation(server_name, send_startup_events=true)
 
       require 'tcell_agent/rails/on_start' if defined?(Rails)
-      require 'rbtrace'
 
       TCellAgent::Instrumentation.safe_block("Starting thread agent") do
         TCellAgent.logger.debug("Instrumenting: #{server_name}")

data/lib/tcell_agent/utils/strings.rb

@@ -10,6 +10,24 @@ module TCellAgent
       def self.present?(str)
         !self.blank?(str)
       end
+
+      def self.remove_trailing_slash(path)
+        if path && path != "/"
+          return path.chomp("/")
+        end
+
+        return path
+      end
+
+      # emulate the java String.hashcode() without upcasting to BigInt
+      def self.java_hashcode(str)
+        result = 0
+        str.each_codepoint do |cp|
+          # prevent overflow into BigInt which would cause heap allocs + emulate c-style int32 signed add overflow
+          result = ((((((result & 0x07FFFFFF) << 5) - result) + cp) + 0x80000000) & 0xFFFFFFFF) - 0x80000000
+        end
+        result
+      end
     end
   end
 end

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "0.2.29.rc2"
+  VERSION = "0.2.29"
 end

data/spec/lib/tcell_agent/api/api_spec.rb

@@ -26,7 +26,7 @@ module TCellAgent
           } })
 
         #  to_return(:body => resbody,
-        result = tapi.pollAPI
+        result = tapi.poll_api()
         TCellAgent.configuration.app_id = nil
         TCellAgent.configuration.api_key = nil
         expect(result["csp-headers"]["app_id"]).to eq("testapp-Becwu")

data/spec/lib/tcell_agent/appsensor/injections_reporter_spec.rb

@@ -9,7 +9,7 @@ module TCellAgent
         before(:each) do
           @payloads_policy = double("payloads_policy")
           @injections_matcher = double("injections_matcher")
-          @injections_reporter = InjectionsReporter.new(@injections_matcher, @payloads_policy)
+          @injections_reporter = InjectionsReporter.new(@injections_matcher, @payloads_policy, false)
 
           @appsensor_meta = TCellAgent::SensorEvents::AppSensorMetaEvent.new
           @appsensor_meta.remote_address = "remote_address"

data/spec/lib/tcell_agent/config/unknown_options_spec.rb

@@ -0,0 +1,188 @@
+require 'spec_helper'
+
+module TCellAgent
+  module Config
+
+    describe Validate do
+      describe ".get_unknown_options" do
+        context "with an unknown tcell environment variable set" do
+          it "should return a message about the unknown variable" do
+
+            orig_allow_uap = ENV.fetch("TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS", nil)
+            orig_allow_uafp = ENV.fetch("TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS", nil)
+            orig_demomode = ENV.fetch("TCELL_DEMOMODE", nil)
+            orig_agent_home = ENV.fetch("TCELL_AGENT_HOME", nil)
+            orig_agent_log_dir = ENV.fetch("TCELL_AGENT_LOG_DIR", nil)
+            orig_agent_config = ENV.fetch("TCELL_AGENT_CONFIG", nil)
+            orig_agent_app_id = ENV.fetch("TCELL_AGENT_APP_ID", nil)
+            orig_agent_api_key = ENV.fetch("TCELL_AGENT_API_KEY", nil)
+            orig_agent_host_identifier = ENV.fetch("TCELL_AGENT_HOST_IDENTIFIER", nil)
+            orig_input_url = ENV.fetch("TCELL_INPUT_URL", nil)
+            orig_hmac_key = ENV.fetch("TCELL_HMAC_KEY", nil)
+            orig_api_url = ENV.fetch("TCELL_API_URL", nil)
+
+            ENV["TCELL_HACK"] = "hack the system"
+            ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"] = "valid"
+            ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"] = "valid"
+            ENV["TCELL_DEMOMODE"] = "valid"
+            ENV["TCELL_AGENT_HOME"] = "valid"
+            ENV["TCELL_AGENT_LOG_DIR"] = "valid"
+            ENV["TCELL_AGENT_CONFIG"] = "valid"
+            ENV["TCELL_AGENT_APP_ID"] = "valid"
+            ENV["TCELL_AGENT_API_KEY"] = "valid"
+            ENV["TCELL_AGENT_HOST_IDENTIFIER"] = "valid"
+            ENV["TCELL_INPUT_URL"] = "valid"
+            ENV["TCELL_HMAC_KEY"] = "valid"
+            ENV["TCELL_API_URL"] = "valid"
+
+            messages = Validate.get_unknown_options(nil)
+
+            ENV.delete "TCELL_HACK"
+            if orig_allow_uap
+              ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"] = orig_allow_uap
+            else
+              ENV.delete "TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS"
+            end
+
+            if orig_allow_uafp
+                ENV["TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"] = orig_allow_uafp
+            else
+              ENV.delete "TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS"
+            end
+            if orig_demomode
+              ENV["TCELL_DEMOMODE"] = orig_demomode
+            else
+              ENV.delete "TCELL_DEMOMODE"
+            end
+            if orig_agent_home
+              ENV["TCELL_AGENT_HOME"] = orig_agent_home
+            else
+              ENV.delete "TCELL_AGENT_HOME"
+            end
+            if orig_agent_log_dir
+              ENV["TCELL_AGENT_LOG_DIR"] = orig_agent_log_dir
+            else
+              ENV.delete "TCELL_AGENT_LOG_DIR"
+            end
+            if orig_agent_config
+              ENV["TCELL_AGENT_CONFIG"] = orig_agent_config
+            else
+              ENV.delete "TCELL_AGENT_CONFIG"
+            end
+            if orig_agent_app_id
+              ENV["TCELL_AGENT_APP_ID"] = orig_agent_app_id
+            else
+              ENV.delete "TCELL_AGENT_APP_ID"
+            end
+            if orig_agent_api_key
+              ENV["TCELL_AGENT_API_KEY"] = orig_agent_api_key
+            else
+              ENV.delete "TCELL_AGENT_API_KEY"
+            end
+            if orig_agent_host_identifier
+              ENV["TCELL_AGENT_HOST_IDENTIFIER"] = orig_agent_host_identifier
+            else
+              ENV.delete "TCELL_AGENT_HOST_IDENTIFIER"
+            end
+            if orig_input_url
+              ENV["TCELL_INPUT_URL"] = orig_input_url
+            else
+              ENV.delete "TCELL_INPUT_URL"
+            end
+            if orig_hmac_key
+              ENV["TCELL_HMAC_KEY"] = orig_hmac_key
+            else
+              ENV.delete "TCELL_HMAC_KEY"
+            end
+            if orig_api_url
+              ENV["TCELL_API_URL"] = orig_api_url
+            else
+              ENV.delete "TCELL_API_URL"
+            end
+
+            expect(messages.sort).to eq([
+              "Unrecognized environment parameter (TCELL_*) found: TCELL_HACK"
+            ])
+          end
+        end
+
+        context "with a config json with all options including some extra ones" do
+          it "should report the extra options in messages" do
+            config_json = {
+                "first_level" => "boo",
+                "version" => 1,
+                "applications" => [{
+                    "second_level" => "boo",
+                    "name" => "name",
+                    "app_id" => "app id",
+                    "api_key" => "api key",
+                    "fetch_policies_from_tcell" => true,
+                    "preload_policy_filename" => "preload policy filename",
+                    "log_dir" => "custom log dir",
+                    "logging_options" => {
+                        "logging_level" => "boo",
+                        "enabled" => true,
+                        "level" => "DEBUG",
+                        "filename" => "filename"},
+                    "tcell_api_url" => "tcell api url",
+                    "tcell_input_url" => "tcell input url",
+                    "host_identifier" => "host identifier",
+                    "hipaaSafeMode" => "hipaa safe mode",
+                    "hmac_key" => "hmac key",
+                    "js_agent_api_base_url" => "js agent api base url",
+                    "js_agent_url" => "js agent url",
+                    "max_csp_header_bytes" => 512,
+                    "event_batch_size_limit" => 50,
+                    "allow_unencrypted_appsensor_payloads" => true,
+                    "allow_unencrypted_appfirewall_payloads" => true,
+                    "data_exposure" => {
+                        "data_ex_level" => "boo",
+                        "max_data_ex_db_records_per_request" => 10000},
+                    "reverse_proxy" => true,
+                    "reverse_proxy_ip_address_header" => "reverse proxy ip address header",
+                    "demomode" => true,
+                    # Ruby only
+                    "disable_all" => false,
+                    "enabled" => true,
+                    "enable_event_manager" => true,
+                    "enable_event_consumer" => true,
+                    "enable_policy_polling" => true,
+                    "enable_instrumentation" => true,
+                    "enable_intercept_requests" => true,
+                    "instrument_for_events" => true,
+                    "agent_home_owner" => true,
+                    "enabled_instrumentations" => {
+                      "enabled_instrumentations_level" => "blah",
+                      "doorkeeper" => true,
+                      "devise" => true,
+                      "authlogic" => true}}]}
+
+            messages = Validate.get_unknown_options(config_json)
+
+            expect(messages.sort).to eq([
+              "Unrecognized config setting key: data_ex_level",
+              "Unrecognized config setting key: enabled_instrumentations_level",
+              "Unrecognized config setting key: first_level",
+              "Unrecognized config setting key: logging_level",
+              "Unrecognized config setting key: second_level"
+            ])
+          end
+        end
+
+        context "with a config json that has more than one application" do
+          it "should report the misconfiguration" do
+            config_json = {"version" => 1, "applications" => [{}, {}]}
+
+            messages = Validate.get_unknown_options(config_json)
+
+            expect(messages.sort).to eq([
+              "Multiple applications detected in config file"
+            ])
+          end
+        end
+      end
+    end
+
+  end
+end
+