tcell_agent 0.2.19 → 0.2.21

data/lib/tcell_agent/sensor_events/util/sanitizer_utilities.rb

@@ -8,146 +8,155 @@ require 'uri'
 require 'openssl'
 
 module TCellAgent
-    module SensorEvents
-        module Util
-            def self.getHmacKey
-                if (TCellAgent.configuration.hmac_key)
-                    return TCellAgent.configuration.hmac_key
-                elsif (TCellAgent.configuration.app_id)
-                    return TCellAgent.configuration.app_id
-                end
-                return "tcell_hmac_key"
-            end
-            def self.hmac(data, hmacKey)
-                hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'), hmacKey.to_s, data)
-                return hmac
-            end
-            def self.request_sanitized_json(request)
-                sanitized_headers = Hash.new
-                headers = request.headers.select {|k,v| k.start_with? 'HTTP_'}
-                    .collect {|pair| [pair[0].sub(/^HTTP_/, ''), pair[1]]}
-                    .sort
-                headers.each do |header_name, header_value|
-                    lower_header_name = header_name.downcase
-                    if lower_header_name == "cookie"
-                        sanitized_headers[header_name] = [self.santize_request_cookie_string(header_value)]
-                    elsif ["content_type", "content_length","user_agent","csp"].include?(lower_header_name)
-                        sanitized_headers[header_name] = [header_value]
-                    else
-                        sanitized_headers[header_name] = []
-                    end
-                end
-                new_request = {"method"=>request.request_method,
-                               "uri"=>self.sanitize_uri(request.fullpath),
-                               "headers"=>sanitized_headers}
-                request_body = request.body.read    
-                if request_body
-                    new_request["post_data"] = sanitize_query_string(request_body)
-                end
-                new_request
-            end
-            def self.response_sanitized_json(response)
-                status, headers, body = *response
-                sanitized_headers = Hash.new
-                content_type = "unknown"
-                headers.each do |header_name, header_value|
-                    lower_header_name = header_name.downcase
-                    if lower_header_name == "set-cookie"
-                        sanitized_headers[header_name] = [self.santize_response_cookie_string(header_value)]
-                    else
-                        if lower_header_name == "content-type"
-                            content_type = header_value
-                        end
-                        if ["content-type", "content-length"].include?(lower_header_name)
-                            sanitized_headers[header_name] = [header_value]
-                        else
-                            sanitized_headers[header_name] = []
-                        end
-                    end
-                end
-                new_response = {"status"=> status, 
-                                "headers"=>sanitized_headers}
-                new_response
-            end
-            def self.santize_request_cookie_string(request_cookie_string)
-                hmacKey = Util.getHmacKey()
-                sanitized_cookies = Hash.new
-                cookies = CGI::Cookie::parse(request_cookie_string)
-                cookies.each do |cookie_name, cookie_value|
-                    if cookie_value.length != 1
-                        next
-                    end
-                    sanitized_cookies[cookie_name] = Util.hmac(cookie_value[0], hmacKey)
-                end
-                sanitized_cookies.map{|k,v| "#{k}=#{v}"}.join(';')
-            end
-            def self.santize_response_cookie_string(response_cookie_string_value)
-                hmacKey = Util.getHmacKey()
-                cookie_parts = response_cookie_string_value.split('; ')
-                cookie_string = cookie_parts[0]
-                cookies = CGI::Cookie::parse(cookie_string)
-                if cookies.length != 1
-                    return "[COOKIEMALFORMED]"
-                end
-                cookie_name = cookies.keys.first
-                cookie_values = cookies.values.first
-                if (cookie_values.length != 1)
-                    return "[COOKIEHADTOOMANYVALUES]"
-                end
-                h = Util.hmac(cookie_values[0], hmacKey)
-                new_cookie_string = "#{cookie_name}=#{h}"
-                cookie_parts[0] = new_cookie_string  
-                cookie_parts.map{|k,v| "#{k}=#{v}"}.join('; ')
-            end
-            def self.sanitize_query_string(query)
-                hmacKey = Util.getHmacKey()
-                params = CGI::parse(query)
-                params.each do |param_name, param_values|
-                    if param_values == nil || param_values.length == 0
-                        next
-                    end
-                    if (param_name.match(/password/i) || 
-                        param_name.match(/passwd/i) || 
-                        param_name.match(/token/i) || 
-                        param_name.match(/sessionid/i))
-                        params[param_name] = ["?"]
-                        next
-                    end
-                    new_param_values = []
-                    param_values.each do |param_value|
-                        h = Util.hmac(param_value, hmacKey)
-                        new_param_values.push << h
-                    end
-                    params[param_name] = new_param_values
-                end
-                params.map{|k,v| "#{k}=#{v.join(',')}"}.join('&')
-            end
-            def self.strip_values_query_string(query)
-                params = CGI::parse(query)
-                params.each do |param_name, param_values|
-                    if param_values == nil || param_values.length == 0
-                        next
-                    end
-                    params[param_name] = [""]
-                end
-                params.map{|k,v| "#{k}=#{v.join(',')}"}.join('&')
-            end 
-            def self.sanitize_uri(uri_string)
-                uri = URI(uri_string)
-                query = uri.query
-                if (query)
-                    uri.query = sanitize_query_string(query)
-                end
-                return uri.to_s
+  module SensorEvents
+    module Util
+      def self.hmac(data)
+        hmac_key = Util.get_hmac_key()
+
+        h = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'), hmac_key.to_s, data)
+
+        return h[0...h.length/2]
+      end
+
+      def self.request_sanitized_json(request)
+        sanitized_headers = Hash.new
+        headers = request.headers.select {|k,v| k.start_with? 'HTTP_'}
+          .collect {|pair| [pair[0].sub(/^HTTP_/, ''), pair[1]]}
+          .sort
+        headers.each do |header_name, header_value|
+          lower_header_name = header_name.downcase
+          if lower_header_name == "cookie"
+            sanitized_headers[header_name] = [self.santize_request_cookie_string(header_value)]
+          elsif ["content_type", "content_length","user_agent","csp"].include?(lower_header_name)
+            sanitized_headers[header_name] = [header_value]
+          else
+            sanitized_headers[header_name] = []
+          end
+        end
+        new_request = {"method"=>request.request_method,
+                       "uri"=>self.sanitize_uri(request.fullpath),
+                       "headers"=>sanitized_headers}
+        request_body = request.body.read
+        if request_body
+          new_request["post_data"] = sanitize_query_string(request_body)
+        end
+        new_request
+      end
+
+      def self.response_sanitized_json(response)
+        status, headers, body = *response
+        sanitized_headers = Hash.new
+        content_type = "unknown"
+        headers.each do |header_name, header_value|
+          lower_header_name = header_name.downcase
+          if lower_header_name == "set-cookie"
+            sanitized_headers[header_name] = [self.santize_response_cookie_string(header_value)]
+          else
+            if lower_header_name == "content-type"
+              content_type = header_value
             end
-            def self.strip_uri_values(uri_string)
-                uri = URI(uri_string)
-                query = uri.query
-                if (query)
-                    uri.query = strip_values_query_string(query)
-                end
-                return uri.to_s
+            if ["content-type", "content-length"].include?(lower_header_name)
+              sanitized_headers[header_name] = [header_value]
+            else
+              sanitized_headers[header_name] = []
             end
+          end
+        end
+        new_response = {"status"=> status,
+                        "headers"=>sanitized_headers}
+        new_response
+      end
+
+      def self.santize_request_cookie_string(request_cookie_string)
+        sanitized_cookies = Hash.new
+        cookies = CGI::Cookie::parse(request_cookie_string)
+        cookies.each do |cookie_name, cookie_value|
+          if cookie_value.length != 1
+            next
+          end
+          sanitized_cookies[cookie_name] = Util.hmac(cookie_value[0])
+        end
+        sanitized_cookies.map{|k,v| "#{k}=#{v}"}.join(';')
+      end
+
+      def self.santize_response_cookie_string(response_cookie_string_value)
+        cookie_parts = response_cookie_string_value.split('; ')
+        cookie_string = cookie_parts[0]
+        cookies = CGI::Cookie::parse(cookie_string)
+        if cookies.length != 1
+          return "[COOKIEMALFORMED]"
+        end
+        cookie_name = cookies.keys.first
+        cookie_values = cookies.values.first
+        if (cookie_values.length != 1)
+          return "[COOKIEHADTOOMANYVALUES]"
+        end
+        h = Util.hmac(cookie_values[0])
+        new_cookie_string = "#{cookie_name}=#{h}"
+        cookie_parts[0] = new_cookie_string
+        cookie_parts.map{|k,v| "#{k}=#{v}"}.join('; ')
+      end
+
+      def self.sanitize_query_string(query)
+        params = CGI::parse(query)
+        params.each do |param_name, param_values|
+          if param_values == nil || param_values.length == 0
+            next
+          end
+          if (param_name.match(/password/i) ||
+              param_name.match(/passwd/i) ||
+              param_name.match(/token/i) ||
+              param_name.match(/sessionid/i))
+            params[param_name] = ["?"]
+            next
+          end
+          new_param_values = []
+          param_values.each do |param_value|
+            h = Util.hmac(param_value)
+            new_param_values.push << h
+          end
+          params[param_name] = new_param_values
+        end
+        params.map{|k,v| "#{k}=#{v.join(',')}"}.join('&')
+      end
+
+      def self.strip_values_query_string(query)
+        params = CGI::parse(query)
+        params.each do |param_name, param_values|
+          if param_values == nil || param_values.length == 0
+            next
+          end
+          params[param_name] = [""]
+        end
+        params.map{|k,v| "#{k}=#{v.join(',')}"}.join('&')
+      end
+
+      def self.sanitize_uri(uri_string)
+        uri = URI(uri_string)
+        query = uri.query
+        if (query)
+          uri.query = sanitize_query_string(query)
+        end
+        return uri.to_s
+      end
+
+      def self.strip_uri_values(uri_string)
+        uri = URI(uri_string)
+        query = uri.query
+        if (query)
+          uri.query = strip_values_query_string(query)
+        end
+        return uri.to_s
+      end
+
+      def self.get_hmac_key
+        if (TCellAgent.configuration.hmac_key)
+          return TCellAgent.configuration.hmac_key
+        elsif (TCellAgent.configuration.app_id)
+          return TCellAgent.configuration.app_id
         end
+        return "tcell_hmac_key"
+      end
     end
-end
+  end
+end

data/lib/tcell_agent/utils/params.rb

@@ -4,35 +4,38 @@ module TCellAgent
       GET_PARAM = "get"
       POST_PARAM = "post"
       JSON_PARAM = "json"
+      URI_PARAM = "uri"
       COOKIE_PARAM = "cookies"
 
-      def param_deep_loop(param_name, param_value, &block)
-        return nil unless param_name and param_value
+      def self.flatten(param_dict, namespace=nil)
+        flattened = {}
+        namespace = [] unless namespace
+        param_dict.each do |param_name, param_value|
+          if param_value.is_a?(Hash)
+            flattened = flattened.merge(flatten(param_value, namespace.dup << param_name.to_s))
 
-        if param_value.is_a?(Hash)
-          param_value.each do |k, v|
-            result = param_deep_loop(k, v, &block)
-            if result
-              return result
+          elsif param_value.is_a?(Array)
+            param_value.each_with_index do |val, index|
+              new_namespace = namespace.dup + [index, param_name.to_s]
+              if val.is_a?(Hash)
+                flattened = flattened.merge(flatten(val, new_namespace))
+              elsif val.is_a?(String) || val.is_a?(Symbol)
+                flattened[new_namespace.freeze] = val.to_s
+              else
+                # DROP any unrecognized types (like Tempfiles and other such possible things)
+              end
             end
-          end
-
-        elsif param_value.is_a?(Array)
-          param_value.each do |v|
-            result = param_deep_loop(param_name, v, &block)
-            return result if result
-          end
 
-        elsif param_value.is_a?(String)
-          #if isinstance(param_value, bytes):
-            #param_value = param_value.decode('utf-8')
-          #param_type = str(type(param_value))
+          elsif param_value.is_a?(String) || param_value.is_a?(Symbol)
+            new_key = namespace.dup << param_name.to_s
+            flattened[new_key.freeze] = param_value.to_s
 
-          match = block.call(param_name, param_value)
-          return match if match
+          else
+            # DROP any unrecognized types (like Tempfiles and other such possible things)
+          end
         end
 
-        return nil
+        return flattened
       end
 
     end

data/lib/tcell_agent/version.rb

@@ -1,5 +1,5 @@
 # See the file "LICENSE" for the full license governing this code.
 
 module TCellAgent
-  VERSION = "0.2.19"
+  VERSION = "0.2.21"
 end

data/spec/lib/tcell_agent/configuration_spec.rb

@@ -3,176 +3,6 @@ require 'spec_helper'
 module TCellAgent
 
   describe Configuration do
-    describe "#load_app_sensor_restrictions" do
-      before(:each) do
-        @config_file = double("config", read: {
-          version: 1,
-          applications: [
-            {
-              allow_unencrypted_appsensor_payloads: true
-            }
-          ]
-        }.to_json)
-      end
-
-      context "with no payloads file present" do
-        it "should set blacklist to default params and whitelist to empty" do
-          expect(File).to receive(:file?).with(
-            File.join(Dir.getwd, "config/tcell_agent.config")
-          ).and_return(true)
-          expect(File).to receive(:open).with(
-            File.join(Dir.getwd, "config/tcell_agent.config")
-          ).and_return(@config_file)
-          expect(File).to receive(:file?).with(
-            "config/tcell_agent_payloads.config"
-          ).and_return(false)
-          expect(File).to_not receive(:open)
-          configuration = TCellAgent::Configuration.new
-
-          expect(configuration.allow_unencrypted_appfirewall_payloads).to eq(true)
-          expect(configuration.blacklisted_params).to eq({
-            "token" => true,
-            "client_secret" => true,
-            "password" => true,
-            "passwd" => true,
-            "refresh_token" => true,
-            "pf.pass" => true,
-            "user.password" => true
-          })
-          expect(configuration.whitelisted_params).to eq({})
-          expect(configuration.whitelist_present).to eq(false)
-        end
-      end
-
-      context "with a payloads file present" do
-        context "with a malformed payloads file" do
-          it "should set blacklist to default params and whitelist to empty" do
-            payloads_file = double("payloads", read: "{ whitelist: { test } ")
-            expect(File).to receive(:file?).with(
-              File.join(Dir.getwd, "config/tcell_agent.config")
-            ).and_return(true)
-            expect(File).to receive(:open).with(
-              File.join(Dir.getwd, "config/tcell_agent.config")
-            ).and_return(@config_file)
-            expect(File).to receive(:file?).with("config/tcell_agent_payloads.config").and_return(true)
-            expect(File).to receive(:open).with("config/tcell_agent_payloads.config").and_return(payloads_file)
-            configuration = TCellAgent::Configuration.new
-
-            expect(configuration.allow_unencrypted_appfirewall_payloads).to eq(false)
-            expect(configuration.blacklisted_params).to eq({
-              "token" => true,
-              "client_secret" => true,
-              "password" => true,
-              "passwd" => true,
-              "refresh_token" => true,
-              "pf.pass" => true,
-              "user.password" => true
-            })
-            expect(configuration.whitelisted_params).to eq({})
-            expect(configuration.whitelist_present).to eq(false)
-          end
-        end
-
-        context "with a payloads file" do
-          context "with empty json" do
-            it "should set blacklist to default params and whitelist to empty" do
-              payloads_file = double("payloads", read: "{}")
-              expect(File).to receive(:file?).with(
-                File.join(Dir.getwd, "config/tcell_agent.config")
-              ).and_return(true)
-              expect(File).to receive(:open).with(
-                File.join(Dir.getwd, "config/tcell_agent.config")
-              ).and_return(@config_file)
-              expect(File).to receive(:file?).with("config/tcell_agent_payloads.config").and_return(true)
-              expect(File).to receive(:open).with("config/tcell_agent_payloads.config").and_return(payloads_file)
-              configuration = TCellAgent::Configuration.new
-
-              expect(configuration.allow_unencrypted_appfirewall_payloads).to eq(true)
-              expect(configuration.blacklisted_params).to eq({
-                "token" => true,
-                "client_secret" => true,
-                "password" => true,
-                "passwd" => true,
-                "refresh_token" => true,
-                "pf.pass" => true,
-                "user.password" => true
-              })
-              expect(configuration.whitelisted_params).to eq({})
-              expect(configuration.whitelist_present).to eq(false)
-            end
-          end
-
-          context "with blacklisted present" do
-            it "should set blacklist but whitelist is empty" do
-              payloads_file = double("payloads", read:{blacklisted:["passwd"]}.to_json)
-              expect(File).to receive(:file?).with(
-                File.join(Dir.getwd, "config/tcell_agent.config")
-              ).and_return(true)
-              expect(File).to receive(:open).with(
-                File.join(Dir.getwd, "config/tcell_agent.config")
-              ).and_return(@config_file)
-              expect(File).to receive(:file?).with("config/tcell_agent_payloads.config").and_return(true)
-              expect(File).to receive(:open).with("config/tcell_agent_payloads.config").and_return(payloads_file)
-              configuration = TCellAgent::Configuration.new
-
-              expect(configuration.allow_unencrypted_appfirewall_payloads).to eq(true)
-              expect(configuration.blacklisted_params).to eq({"passwd" => true})
-              expect(configuration.whitelisted_params).to eq({})
-              expect(configuration.whitelist_present).to eq(false)
-            end
-          end
-
-          context "with whitelist present" do
-            it "should set whitelist and blacklist to default params" do
-              payloads_file = double("payloads", read: {whitelisted: ["passwd"]}.to_json)
-              expect(File).to receive(:file?).with(
-                File.join(Dir.getwd, "config/tcell_agent.config")
-              ).and_return(true)
-              expect(File).to receive(:open).with(
-                File.join(Dir.getwd, "config/tcell_agent.config")
-              ).and_return(@config_file)
-              expect(File).to receive(:file?).with("config/tcell_agent_payloads.config").and_return(true)
-              expect(File).to receive(:open).with("config/tcell_agent_payloads.config").and_return(payloads_file)
-              configuration = TCellAgent::Configuration.new
-
-              expect(configuration.allow_unencrypted_appfirewall_payloads).to eq(true)
-              expect(configuration.blacklisted_params).to eq({
-                "token" => true,
-                "client_secret" => true,
-                "password" => true,
-                "passwd" => true,
-                "refresh_token" => true,
-                "pf.pass" => true,
-                "user.password" => true
-              })
-              expect(configuration.whitelisted_params).to eq({"passwd" => true})
-              expect(configuration.whitelist_present).to eq(true)
-            end
-          end
-
-          context "with blacklist and whitelist present" do
-            it "should set whitelist and blacklist" do
-              payloads_file = double("payloads", read: {blacklisted: ["ssn"], whitelisted: ["passwd"]}.to_json)
-              expect(File).to receive(:file?).with(
-                File.join(Dir.getwd, "config/tcell_agent.config")
-              ).and_return(true)
-              expect(File).to receive(:open).with(
-                File.join(Dir.getwd, "config/tcell_agent.config")
-              ).and_return(@config_file)
-              expect(File).to receive(:file?).with("config/tcell_agent_payloads.config").and_return(true)
-              expect(File).to receive(:open).with("config/tcell_agent_payloads.config").and_return(payloads_file)
-              configuration = TCellAgent::Configuration.new
-
-              expect(configuration.allow_unencrypted_appfirewall_payloads).to eq(true)
-              expect(configuration.blacklisted_params).to eq({"ssn" => true})
-              expect(configuration.whitelisted_params).to eq({"passwd" => true})
-              expect(configuration.whitelist_present).to eq(true)
-            end
-          end
-        end
-      end
-
-    end
 
     describe "#agent_home_dir" do
       context "no TCELL_AGENT_HOME defined" do
@@ -287,9 +117,6 @@ module TCellAgent
           expect(File).to receive(:open).with(
             File.join(Dir.getwd, "no_data_ex.config")
           ).and_return(no_data_ex)
-          expect(File).to receive(:file?).with(
-            "config/tcell_agent_payloads.config"
-          ).and_return(false)
           configuration = Configuration.new("no_data_ex.config")
 
           expect(configuration.max_data_ex_db_records_per_request).to eq(1000)
@@ -316,9 +143,6 @@ module TCellAgent
           expect(File).to receive(:open).with(
             File.join(Dir.getwd, "no_data_ex.config")
           ).and_return(no_data_ex)
-          expect(File).to receive(:file?).with(
-            "config/tcell_agent_payloads.config"
-          ).and_return(false)
           configuration = Configuration.new("no_data_ex.config")
 
           expect(configuration.max_data_ex_db_records_per_request).to eq(1000)
@@ -347,9 +171,6 @@ module TCellAgent
           expect(File).to receive(:open).with(
             File.join(Dir.getwd, "no_data_ex.config")
           ).and_return(no_data_ex)
-          expect(File).to receive(:file?).with(
-            "config/tcell_agent_payloads.config"
-          ).and_return(false)
           configuration = Configuration.new("no_data_ex.config")
 
           expect(configuration.max_data_ex_db_records_per_request).to eq(5000)

data/spec/lib/tcell_agent/policies/appsensor/database_sensor_spec.rb

@@ -92,8 +92,9 @@ module TCellAgent
                   "dp" => DatabaseSensor::DP_CODE,
                   "param" => nil,
                   "remote_addr" => "ip_address",
-                  "rou" => "route_id",
-                  "m" => "get"
+                  "rid" => "route_id",
+                  "m" => "get",
+                  "meta" => {"rows" => 11}
                 }
               )
               sensor.check(tcell_data, 11)
@@ -147,8 +148,9 @@ module TCellAgent
                     "dp" => DatabaseSensor::DP_CODE,
                     "param" => nil,
                     "remote_addr" => "ip_address",
-                    "rou" => "route_id",
-                    "m" => "get"
+                    "rid" => "route_id",
+                    "m" => "get",
+                    "meta" => {"rows" => 11}
                   }
                 )
                 sensor.check(tcell_data, 11)