tcell_agent 0.2.2

data/lib/tcell_agent/sensor_events/dlp.rb

@@ -0,0 +1,58 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/sensor'
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+module TCellAgent
+    module SensorEvents
+        class DlpEvent < TCellSensorEvent
+            FOUND_IN_BODY = "body"
+            FOUND_IN_LOG = "log"
+            FOUND_IN_CONSOLE = "console"
+
+            FRAMEWORK_VARIABLE_SESSION_ID="session_id"
+
+            REQUEST_CONTEXT_FORM = "form"
+            REQUEST_CONTEXT_COOKIE = "cookie"
+            REQUEST_CONTEXT_HEADER = "header"
+
+            def initialize(route_id, raw_uri, found_in, hmac_session_id=nil, user_id=nil)
+                super("dlp")
+                self["rid"] = route_id
+                self["found_in"] = found_in
+                @raw_uri = raw_uri
+                if hmac_session_id
+                    self["sid"] = hmac_session_id
+                end
+                if user_id
+                    self["uid"] = user_id
+                end
+            end
+            def for_database(database, schema, table, field)
+                self["type"] = "db"
+                self["db"] = database
+                self["schema"] = schema
+                self["table"] = table
+                self["field"] = field
+                return self
+            end
+            def for_framework(variable)
+                self["type"] = "framework"
+                self["context"] = "framework"
+                self["variable"] = variable
+                return self
+            end
+            def for_request(variable_context, variable)
+                self["type"] = "request"
+                self["context"] = variable_context
+                self["variable"] = variable
+                return self
+            end
+            def post_process
+                if @raw_uri
+                    self["uri"] = Util.strip_uri_values(@raw_uri)
+                end
+            end
+        end
+    end
+end

data/lib/tcell_agent/sensor_events/honeytokens.rb

@@ -0,0 +1,16 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/sensor'
+
+module TCellAgent
+    module SensorEvents
+        class HoneytokensSensorEvent < TCellSensorEvent
+            def initialize(request, token_id)
+                super("honeytoken")
+                self["id"] = token_id
+                self["ip"] = request.remote_ip
+            end
+        end
+    end
+end

data/lib/tcell_agent/sensor_events/login_fraud.rb

@@ -0,0 +1,43 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/sensor'
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+module TCellAgent
+    module SensorEvents
+
+        class LoginFailure < TCellSensorEvent
+            def initialize(request, response, user_id, hmac_session_id)
+                super("login")
+                headers = request.env.select {|k,v| k.start_with? 'HTTP_'}
+                  .collect {|k,v| k.sub(/^HTTP_/, '') }
+                self["event_name"] = "login-failure"
+                self["user_agent"] = request.env['HTTP_USER_AGENT']
+                self["referrer"] = request.referer
+                self["remote_addr"] = request.remote_ip
+                self["header_keys"] = headers
+                self["user_id"] = user_id
+                self["document_uri"] = TCellAgent::SensorEvents::Util.strip_uri_values(request.path)
+                self["session"] = hmac_session_id
+            end
+        end
+
+        class LoginSuccess < TCellSensorEvent
+            attr_accessor :event_name
+            def initialize(request, response, user_id, hmac_session_id)
+                super("login")
+                headers = request.env.select {|k,v| k.start_with? 'HTTP_'}
+                  .collect {|k,v| k.sub(/^HTTP_/, '') }
+                self["event_name"] = "login-success"
+                self["user_agent"] = request.env['HTTP_USER_AGENT']
+                self["referrer"] = request.referer
+                self["remote_addr"] = request.remote_ip
+                self["header_keys"] = headers
+                self["user_id"] = user_id
+                self["document_uri"] = TCellAgent::SensorEvents::Util.strip_uri_values(request.path)
+                self["session"] = hmac_session_id
+            end
+        end
+
+    end
+end

data/lib/tcell_agent/sensor_events/metrics.rb

@@ -0,0 +1,24 @@
+# See the file "LICENSE" for the full license governing this code.
+
+module TCellAgent
+    module SensorEvents
+        class RequestRouteTimer < TCellSensorEvent
+            attr_accessor :route_id
+            attr_accessor :response_time
+            def initialize(route_id, response_time)
+                super("RequestRouteTimer")
+                self.route_id = route_id
+                self.response_time = response_time
+                @send = false
+            end
+        end
+        class MetricsEvent < TCellSensorEvent
+            def initialize()
+                super("metrics")
+            end
+            def set_route_count_table(route_count_table)
+                self["rct"] = route_count_table
+            end
+        end
+    end
+end

data/lib/tcell_agent/sensor_events/sensor.rb

@@ -0,0 +1,85 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/logger'
+require 'uri'
+
+module TCellAgent
+    module SensorEvents
+        class TCellSensorEvent < Hash
+            attr_accessor :send
+            attr_accessor :flush
+            attr_accessor :ensure
+            def initialize(event_type)
+                @send = true
+                @flush = false
+                @ensure = false
+                @timestamp = DateTime.now.to_time.to_i
+                self["event_type"] = event_type
+            end
+            def calculateOffset(from_timestamp)
+                self["offset"] = from_timestamp - @timestamp
+            end
+            def post_process
+                # This is called in the background thread, so any
+                #   santization, analysis, etc doesn't get in the way
+            end
+        end
+        class TCellHttpTxSensorEvent < TCellSensorEvent
+            def initialize(request, response)
+                super("http_tx")
+                @raw_request = request
+                @raw_response = response
+            end
+            def post_process
+                if defined?@raw_request
+                    self["request"] = Util.request_sanitized_json(@raw_request)
+                end
+                if defined?@raw_response
+                    self["response"] = Util.response_sanitized_json(@raw_response)
+                end
+            end
+        end
+        class TCellRedirectSensorEvent < TCellSensorEvent
+            def initialize(redirect_domain, original_domain, original_url, method, status_code, remote_addr, user_id=nil, session_id=nil)
+                super("redirect")
+                @raw_original_url = original_url
+                self["method"] = method
+                self["from_domain"] = original_domain
+                self["status_code"] = status_code
+                self["remote_addr"] = remote_addr
+                @raw_redirect_domain = redirect_domain
+                @user_id = user_id
+                @raw_session_id = session_id
+            end
+            def post_process
+                self["from"] = Util.strip_uri_values(@raw_original_url)
+                self["to"] = @raw_redirect_domain
+                if @raw_session_id
+                    hmac_key = Util.getHmacKey()
+                    self["sid"] = Util.hmac(@raw_session_id, hmac_key)
+                end
+            end
+        end
+        class TCellFingerprintSensorEvent < TCellSensorEvent
+            def initialize(request, session_id, user_id=nil)
+                super("fingerprint")
+                @raw_request = request
+                @raw_session_id = session_id
+                @user_id = user_id
+            end
+            def post_process
+                if !(@raw_request.headers.key?("HTTP_USER_AGENT"))
+                    raise "User Agent not Found!"
+                end 
+                self["ua"] = @raw_request.headers["HTTP_USER_AGENT"]
+                self["ip"] = @raw_request.remote_ip
+                hmac_key = Util.getHmacKey()
+                self["sid"] = Util.hmac(@raw_session_id, hmac_key)
+                if @user_id
+                    self["uid"] = @user_id
+                end
+            end
+        end
+    end
+end

data/lib/tcell_agent/sensor_events/server_agent.rb

@@ -0,0 +1,101 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/sensor'
+require 'tcell_agent/sensor_events/util/utils'
+require 'etc'
+
+module TCellAgent
+    module SensorEvents
+        class ServerAgentDetailsSensorEvent < TCellSensorEvent
+            def initialize
+                super("server_agent_details")
+                @flush = true
+                @ensure = true
+                login = Etc.getlogin
+                self["user"] = Etc.getlogin
+                info = Etc.getpwnam(login)
+                self["group"] = info.gid.to_s
+            end
+        end
+        class ServerAgentAppFrameworkEvent < TCellSensorEvent
+            def initialize(framework_name, framework_version)
+                super("server_agent_details")
+                @flush = true
+                @ensure = true
+                self["app_framework"] = framework_name
+                self["app_framework_version"] = framework_version
+            end
+        end
+        class ServerAgentPackagesSensorEvent < TCellSensorEvent
+            def initialize
+                super("server_agent_packages")
+                @flush = true
+                @ensure = true
+                packages = []
+                Gem.loaded_specs.values.map { |x| 
+                    package = {"n"=>x.name, "v"=>x.version.version}
+                    packages.push(package)
+                }
+                self["packages"] = packages
+            end
+        end
+        class AppFramework < TCellSensorEvent
+            def initialize(name, version)
+                super("appserver_framework")
+                @flush = false
+                @ensure = true
+                self["n"] = name
+                self["v"] = version
+            end
+        end
+        class AppAuthFramework < TCellSensorEvent
+            def initialize(name, version)
+                super("appserver_auth_framework")
+                @flush = false
+                @ensure = true
+                self["n"] = name
+                self["v"] = version
+            end
+        end
+        class AppFrameworkSetting < TCellSensorEvent
+            def initialize(framework_name, setting, value)
+                super("appserver_framework_setting")
+                @flush = false
+                @ensure = true
+                self["framework"] = framework_name
+                self["s"] = setting
+                self["v"] = value
+            end
+        end
+        class AppCookie < TCellSensorEvent
+            def initialize(name, value, secure, http_only, session)
+                super("appserver_framework_setting")
+                @flush = false
+                @ensure = true
+                self["n"] = name
+                self["v"] = value
+                self["http_only"] = http_only
+                self["secure"] = secure
+                self["session"] = session
+            end
+        end
+        class AppRoutesSensorEvent < TCellSensorEvent
+            def initialize(uri, method, route_id, params=nil, destination=nil)
+                super("appserver_routes")
+                @flush = false
+                @ensure = true
+                method = method.downcase
+                self["uri"] = uri
+                self["method"] = method
+                self["rid"] = route_id
+                if (params)
+                    self["params"] = params
+                end
+                if (destination) 
+                    self["destination"] = destination
+                end
+            end
+        end
+    end
+end

data/lib/tcell_agent/sensor_events/util/redirect_utils.rb

@@ -0,0 +1,22 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'logger'
+require 'cgi'
+require 'uri'
+require 'openssl'
+
+module TCellAgent
+  module SensorEvents
+    module Util
+      def self.wildcardMatch(target, wildcardPattern)
+        escaped = Regexp.escape(wildcardPattern).gsub('\*','.*?')
+        regex = Regexp.new "^#{escaped}$", Regexp::IGNORECASE
+        !!(target =~ regex)
+      end
+      def self.domainFromUrl(url)
+        uri = URI.parse(url)
+        uri.host
+      end
+    end
+  end
+end

data/lib/tcell_agent/sensor_events/util/sanitizer_utilities.rb

@@ -0,0 +1,153 @@
+# encoding: utf-8
+
+# See the file "LICENSE" for the full license governing this code.
+
+require 'logger'
+require 'cgi'
+require 'uri'
+require 'openssl'
+
+module TCellAgent
+    module SensorEvents
+        module Util
+            def self.getHmacKey
+                if (TCellAgent.configuration.hmac_key)
+                    return TCellAgent.configuration.hmac_key
+                elsif (TCellAgent.configuration.app_id)
+                    return TCellAgent.configuration.app_id
+                end
+                return "tcell_hmac_key"
+            end
+            def self.hmac(data, hmacKey)
+                hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'), hmacKey.to_s, data)
+                return hmac
+            end
+            def self.request_sanitized_json(request)
+                sanitized_headers = Hash.new
+                headers = request.headers.select {|k,v| k.start_with? 'HTTP_'}
+                    .collect {|pair| [pair[0].sub(/^HTTP_/, ''), pair[1]]}
+                    .sort
+                headers.each do |header_name, header_value|
+                    lower_header_name = header_name.downcase
+                    if lower_header_name == "cookie"
+                        sanitized_headers[header_name] = [self.santize_request_cookie_string(header_value)]
+                    elsif ["content_type", "content_length","user_agent","csp"].include?(lower_header_name)
+                        sanitized_headers[header_name] = [header_value]
+                    else
+                        sanitized_headers[header_name] = []
+                    end
+                end
+                new_request = {"method"=>request.request_method,
+                               "uri"=>self.sanitize_uri(request.fullpath),
+                               "headers"=>sanitized_headers}
+                request_body = request.body.read    
+                if request_body
+                    new_request["post_data"] = sanitize_query_string(request_body)
+                end
+                new_request
+            end
+            def self.response_sanitized_json(response)
+                status, headers, body = *response
+                sanitized_headers = Hash.new
+                content_type = "unknown"
+                headers.each do |header_name, header_value|
+                    lower_header_name = header_name.downcase
+                    if lower_header_name == "set-cookie"
+                        sanitized_headers[header_name] = [self.santize_response_cookie_string(header_value)]
+                    else
+                        if lower_header_name == "content-type"
+                            content_type = header_value
+                        end
+                        if ["content-type", "content-length"].include?(lower_header_name)
+                            sanitized_headers[header_name] = [header_value]
+                        else
+                            sanitized_headers[header_name] = []
+                        end
+                    end
+                end
+                new_response = {"status"=> status, 
+                                "headers"=>sanitized_headers}
+                new_response
+            end
+            def self.santize_request_cookie_string(request_cookie_string)
+                hmacKey = Util.getHmacKey()
+                sanitized_cookies = Hash.new
+                cookies = CGI::Cookie::parse(request_cookie_string)
+                cookies.each do |cookie_name, cookie_value|
+                    if cookie_value.length != 1
+                        next
+                    end
+                    sanitized_cookies[cookie_name] = Util.hmac(cookie_value[0], hmacKey)
+                end
+                sanitized_cookies.map{|k,v| "#{k}=#{v}"}.join(';')
+            end
+            def self.santize_response_cookie_string(response_cookie_string_value)
+                hmacKey = Util.getHmacKey()
+                cookie_parts = response_cookie_string_value.split('; ')
+                cookie_string = cookie_parts[0]
+                cookies = CGI::Cookie::parse(cookie_string)
+                if cookies.length != 1
+                    return "[COOKIEMALFORMED]"
+                end
+                cookie_name = cookies.keys.first
+                cookie_values = cookies.values.first
+                if (cookie_values.length != 1)
+                    return "[COOKIEHADTOOMANYVALUES]"
+                end
+                h = Util.hmac(cookie_values[0], hmacKey)
+                new_cookie_string = "#{cookie_name}=#{h}"
+                cookie_parts[0] = new_cookie_string  
+                cookie_parts.map{|k,v| "#{k}=#{v}"}.join('; ')
+            end
+            def self.sanitize_query_string(query)
+                hmacKey = Util.getHmacKey()
+                params = CGI::parse(query)
+                params.each do |param_name, param_values|
+                    if param_values == nil || param_values.length == 0
+                        next
+                    end
+                    if (param_name.match(/password/i) || 
+                        param_name.match(/passwd/i) || 
+                        param_name.match(/token/i) || 
+                        param_name.match(/sessionid/i))
+                        params[param_name] = ["?"]
+                        next
+                    end
+                    new_param_values = []
+                    param_values.each do |param_value|
+                        h = Util.hmac(param_value, hmacKey)
+                        new_param_values.push << h
+                    end
+                    params[param_name] = new_param_values
+                end
+                params.map{|k,v| "#{k}=#{v.join(',')}"}.join('&')
+            end
+            def self.strip_values_query_string(query)
+                params = CGI::parse(query)
+                params.each do |param_name, param_values|
+                    if param_values == nil || param_values.length == 0
+                        next
+                    end
+                    params[param_name] = [""]
+                end
+                params.map{|k,v| "#{k}=#{v.join(',')}"}.join('&')
+            end 
+            def self.sanitize_uri(uri_string)
+                uri = URI(uri_string)
+                query = uri.query
+                if (query)
+                    uri.query = sanitize_query_string(query)
+                end
+                return uri.to_s
+            end
+            def self.strip_uri_values(uri_string)
+                uri = URI(uri_string)
+                query = uri.query
+                if (query)
+                    uri.query = strip_values_query_string(query)
+                end
+                return uri.to_s
+            end
+        end
+    end
+end