tcell_agent 0.2.2

data/lib/tcell_agent/rails.rb

@@ -0,0 +1,146 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/authlogic' if defined?(Authlogic)
+require 'tcell_agent/devise' if defined?(Devise)
+      
+require 'rails'
+require 'uri'
+require 'tcell_agent/logger'
+require 'tcell_agent/agent'
+require 'tcell_agent/sensor_events/sensor'
+require 'tcell_agent/sensor_events/app_sensor'
+require 'tcell_agent/sensor_events/server_agent'
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/util/redirect_utils'
+
+require 'tcell_agent/rails/middleware/global_middleware'
+require 'tcell_agent/rails/middleware/body_filter_middleware'
+require 'tcell_agent/rails/middleware/headers_middleware'
+require 'tcell_agent/rails/middleware/context_middleware'
+
+require 'tcell_agent/rails/routes'
+require 'tcell_agent/rails/settings_reporter'
+require 'tcell_agent/rails/dlp'
+
+require 'rails/all'
+
+require 'tcell_agent/userinfo'
+require 'cgi'
+require 'thread'
+
+# ensure ActiveRecord's version has been required already
+
+module TCellAgent
+  class Railtie < Rails::Railtie
+    initializer "tcell_agent.insert_middleware" do |app|
+      app.config.middleware.insert_before(0, "TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware")
+      app.config.middleware.insert_after(0, "TCellAgent::Instrumentation::Rails::Middleware::HeadersMiddleware")
+      app.config.middleware.use "TCellAgent::Instrumentation::Rails::Middleware::BodyFilterMiddleware"
+      app.config.middleware.use "TCellAgent::Instrumentation::Rails::Middleware::GlobalMiddleware"
+    end
+    config.after_initialize do
+      TCellAgent::Instrumentation::Rails.send_framework_info    
+      TCellAgent::Instrumentation::Rails.send_settings(Rails.application)
+    end
+  end
+  class Engine < Rails::Engine
+    config.after_initialize do
+      TCellAgent::Instrumentation::Rails.get_routes()
+    end
+  end
+  #   # if (Rails::VERSION::MAJOR == 3)
+  #   #   config.after_initialize do
+  #   #     Rails.application.reload_routes!
+  #   #     Rails.application.routes.routes.each do |route|
+  #   #        methods = ['GET','POST','PUT','DELETE','HEAD',
+  #   #         'PATCH','TRACE','CONNECT','OPTIONS']
+  #   #         if (route.constraints.has_key? :request_method)
+  #   #          route_path = "#{route.path.spec}"
+  #   #          if (route_path.end_with?("(.:format)"))
+  #   #            route_path = route_path.chomp("(.:format)")
+  #   #          end
+  #   #          route_destination = route.defaults.to_s
+  #   #          route_params =  route.path.required_names
+  #   #          route_methods = methods.select {|x| route.verb.match(x) }
+  #   #          route_methods.each { |route_method|
+  #   #            #puts "#{route_path}, #{route_method.downcase}"
+  #   #            TCellAgent::AgentThread.sendEvent(
+  #   #              TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
+  #   #                route_path, route_method, nil, "#{route_destination}"
+  #   #                )
+  #   #              )
+  #   #          }
+  #   #         end
+  #   #     end
+  #   #   end
+  #   # end
+  #   ActionDispatch::Request.class_eval do
+  #     attr_accessor :_tcell_transaction_id
+  #   end
+  #   config.after_initialize do
+  #       puts "Framework"
+  #       puts "Rails"
+  #       puts Rails.version
+  #       puts Rails.application.config.session_options
+  #       if defined?(Devise)
+  #         puts "Devise"
+  #         puts "ominauth"
+  #         puts Devise.password_length
+  #         puts Devise.remember_for
+  #         puts Devise.expire_all_remember_me_on_sign_out
+  #         puts Devise.maximum_attempts
+  #         puts Devise.unlock_in
+  #         puts Devise.paranoid
+  #         puts Devise.token_generator
+  #         puts "warden"
+  #         puts Devise.warden_config
+  #       end
+  #   end
+  #   if (Rails::VERSION::MAJOR == 4)
+  #     ActionDispatch::Journey::Routes.class_eval do
+  #       alias_method :original_add_route, :add_route
+  #       def add_route(app, path, conditions, defaults, name = nil)
+  #         route = original_add_route(app, path, conditions, defaults, name)
+  #         methods = ['GET','POST','PUT','DELETE','HEAD',
+  #          'PATCH','TRACE','CONNECT','OPTIONS']
+  #          if (route.constraints.has_key? :request_method)
+  #           route_path = "#{route.path.spec}"
+  #           if (route_path.end_with?("(.:format)"))
+  #             route_path = route_path.chomp("(.:format)")
+  #           end
+  #           route_destination = route.defaults.to_s
+  #           route_params =  route.path.required_names
+  #           route_methods = methods.select {|x| route.verb.match(x) }
+  #           route_methods.each { |route_method|
+  #             #puts "#{route_path}, #{route_method.downcase}"
+  #             TCellAgent.send_event(
+  #               TCellAgent::SensorEvents::AppRoutesSensorEvent.new(
+  #                 route_path, route_method, nil, "#{route_destination}"
+  #               )
+  #             )
+  #           }
+  #         end
+  #         route
+  #       end
+  #     end
+  #   end
+  #   ActiveSupport.on_load(:action_controller) do
+  #     ActionController::Base.class_eval do
+  #       #around_filter :global_request_logging
+  #       # def _tcell_route_name
+  #       #   begin
+  #       #     route = Rails.application.routes.router.recognize(request) { |route, _| route }.first
+  #       #     route_path = "#{route[2].path.spec}"
+  #       #     if (route_path.end_with?("(.:format)"))
+  #       #       route_path = route_path.chomp("(.:format)")
+  #       #     end
+  #       #     #puts "#{route_path}, #{request.method.downcase}"
+  #       #     TCellAgent::SensorEvents::Util.calculateRouteId(request.method.downcase, route_path)
+  #       #   rescue Exception => inner_excetion
+  #       #     TCellAgent.logger.debug("Could not figure out path #{inner_excetion.message}")
+  #       #   end
+  #       # end #def global
+  #     end #ac classeval
+  #   end #as onload
+  # end #class
+end #module

data/lib/tcell_agent/rails/dlp.rb

@@ -0,0 +1,204 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'tcell_agent/authlogic' if defined?(Authlogic)
+require 'tcell_agent/devise' if defined?(Devise)
+      
+require 'rails'
+require 'uri'
+require 'tcell_agent/logger'
+require 'tcell_agent/agent'
+require 'tcell_agent/sensor_events/sensor'
+require 'tcell_agent/sensor_events/app_sensor'
+require 'tcell_agent/sensor_events/server_agent'
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/util/redirect_utils'
+
+require 'tcell_agent/sensor_events/dlp'
+
+require 'tcell_agent/rails/middleware/global_middleware'
+require 'tcell_agent/rails/middleware/body_filter_middleware'
+require 'tcell_agent/rails/middleware/headers_middleware'
+require 'tcell_agent/rails/middleware/context_middleware'
+
+require 'tcell_agent/rails/routes'
+require 'tcell_agent/rails/settings_reporter'
+require 'tcell_agent/rails/dlp'
+
+require 'rails/all'
+
+require 'tcell_agent/userinfo'
+require 'cgi'
+require 'thread'
+
+# if defined?(SQLite3)
+#   require 'active_record/connection_adapters/sqlite3_adapter'
+#   ActiveRecord::ConnectionAdapters::SQLite3Adapter.class_eval do
+#         alias_method :original_exec, :exec_query
+#         def exec_query(sql, name = nil, binds = [])
+#           puts "----v----"
+#           puts sql
+#           puts name
+#           puts binds
+#           puts "----^----"
+#           result = original_exec(sql, name, binds)
+#           puts result.inspect
+#           puts ";-----------------------;"
+#           result
+#         end
+#         def postgresql_version
+#           80200
+#         end
+#   end
+# end
+
+# class ActiveRecord::Base
+#   after_initialize do |user|
+#     puts "You have initialized an object!"
+#     puts user
+#   end
+
+#   after_find do |record|
+#     dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+#     if dlp_policy
+#       request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
+#       if request_env
+#         model = record.class
+#         model.columns.each do |col|
+#           #puts "#{model.table_name} .. #{col.name}"
+#           actions = dlp_policy.get_actions_for(model.table_name, col.name)
+#           if (actions.include?("body_redact"))
+#             (request_env["filter_body_set"] ||= Set.new).add(record[col.name.to_sym])
+#             #request_env["filter_body_set"].add()
+#           end
+#           if (actions.include?("log_redact"))
+#             (request_env["filter_log_set"] ||= Set.new).add(record[col.name.to_sym])
+#             #request_env["filter_log_set"].add(record[col.name.to_sym])
+#           end
+#         end
+#       end
+#     end
+#   end
+# end
+
+# - Request
+# -   Session Id event
+# -   Session Id redact
+# -   Session Id hash
+# -   Session Id mask
+# -   Database-Stuff - [event, redact]
+# 
+# - Log
+#
+
+module TCellAgent
+  module Policies
+    class DataLossPolicy
+      def log_enforce(tcell_context, sanitize_string)
+        if (tcell_context && tcell_context.session_id)
+          session_id_actions = self.get_actions_for_session_id
+          if session_id_actions
+            send_event = false
+            sanitize_string.gsub!(tcell_context.session_id) {|m|
+              if session_id_actions.log_redact
+                send_event = true
+                m = "[session_id]"
+              elsif session_id_actions.log_hash
+                send_event = true
+                m = "[hash]"
+              elsif session_id_actions.log_event
+                send_event = true
+              end
+              m
+            }
+            if send_event
+              TCellAgent.send_event(
+                TCellAgent::SensorEvents::DlpEvent.new(
+                  tcell_context.route_id, 
+                  tcell_context.uri, 
+                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG
+                  ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
+              )
+            end
+          end
+        end
+        sanitize_string
+      end
+      def response_body_enforce(tcell_context, sanitize_string)
+        if (tcell_context && tcell_context.session_id)
+          session_id_actions = self.get_actions_for_session_id
+          if session_id_actions
+            send_event = false
+            sanitize_string.gsub!(tcell_context.session_id) {|m|
+              if session_id_actions.body_redact
+                # m = "[session_id]"
+                send_event = true
+              elsif session_id_actions.body_hash
+                # m = "[hash]"
+                send_event = true
+              elsif session_id_actions.body_event
+                send_event = true
+              end
+              m
+            }
+          end
+          if send_event
+            TCellAgent.send_event(
+              TCellAgent::SensorEvents::DlpEvent.new(
+                tcell_context.route_id, 
+                tcell_context.uri, 
+                TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY
+                ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
+            )
+          end
+        end
+        sanitize_string
+      end
+    end
+  end
+end
+
+module TCellAgent
+  class Engine < Rails::Engine
+    ActiveSupport.on_load(:action_controller) do
+      ActionController::Base.class_eval do
+        around_filter :global_request_logging
+        def global_request_logging 
+          begin 
+            yield
+            TCellAgent::Instrumentation.safe_block("Handling JSAgent add") {
+              dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+              if dlp_policy
+                tcell_context = request.env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+                response.body = dlp_policy.response_body_enforce(tcell_context, response.body)
+              end
+            }
+          end
+        end
+      end
+    end
+  end
+end
+
+class Logger
+  alias_method :tcell_old_add, :add
+  def add(severity, message = nil, progname = nil, &block)
+    progname ||= @progname
+    if message.nil?
+      if block_given?
+        message = yield
+      else
+        message = progname
+        progname = @progname
+      end
+    end
+    TCellAgent::Instrumentation.safe_block_no_log("Handling JSAgent add") {
+      dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+      request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
+      if message && dlp_policy && request_env
+        tcell_context = request_env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID]
+        dlp_policy.log_enforce(tcell_context, message)
+      end
+    }
+    tcell_old_add(severity, message, progname)
+  end
+end

data/lib/tcell_agent/rails/middleware/body_filter_middleware.rb

@@ -0,0 +1,69 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'rails'
+require 'uri'
+require 'tcell_agent/logger'
+require 'tcell_agent/agent'
+require 'tcell_agent/sensor_events/sensor'
+require 'tcell_agent/sensor_events/app_sensor'
+require 'tcell_agent/sensor_events/server_agent'
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/util/redirect_utils'
+
+require 'tcell_agent/instrumentation'
+
+module TCellAgent
+  module Instrumentation
+    module Rails
+      module Middleware
+        class BodyFilterMiddleware
+          HEAD_SEARCH_REGEX=/<head>/
+          def initialize(app)
+            @app = app
+          end
+          def call(env)
+            request = Rack::Request.new(env)
+            orig = (Time.now.to_f * 1000).to_i
+            response = @app.call(env)
+            response_time = (Time.now.to_f * 1000).to_i - orig
+            TCellAgent::Instrumentation.safe_block("Handling Route Time") {
+              route_id = env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].route_id
+              if route_id
+                TCellAgent.increment_route(route_id, response_time)
+              else
+                TCellAgent.increment_route("", response_time)
+              end
+            }
+            response = self._handle_js_agent_add(request, response)
+            response
+          end
+          def replace_in_body(script_tag_policy, body)
+              base_url_vars = ""
+              if (script_tag_policy.js_agent_api_base_url)
+                base_url_vars = " tcellbaseurl=\"#{script_tag_policy.js_agent_api_base_url}\""
+              end
+              script_insert = "\n<script src=\"#{script_tag_policy.js_agent_url}\" tcellapikey=\"#{script_tag_policy.js_agent_api_key}\" tcellappid=\"#{script_tag_policy.js_agent_app_id}\"#{base_url_vars}></script>\n"
+              body.sub!(BodyFilterMiddleware::HEAD_SEARCH_REGEX,"<head>#{script_insert}")
+          end
+          def _handle_js_agent_add(request, response)
+            TCellAgent::Instrumentation.safe_block("Handling JSAgent add") {
+              status, headers, rack_body = response
+              if (headers.fetch("Content-Type","").start_with?'text/html')
+                  script_tag_policy = TCellAgent.policy(TCellAgent::PolicyTypes::AddScriptTag)
+                  if (script_tag_policy &&
+                      script_tag_policy.enabled)
+                    newbody = []
+                    rack_body.each { |str| 
+                      newbody << self.replace_in_body(script_tag_policy, str) 
+                    }
+                    response = [status, headers, newbody]
+                  end
+              end
+            }
+            response
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tcell_agent/rails/middleware/context_middleware.rb

@@ -0,0 +1,50 @@
+# See the file "LICENSE" for the full license governing this code.
+
+require 'rails'
+require 'uri'
+require 'tcell_agent/logger'
+require 'tcell_agent/agent'
+require 'tcell_agent/sensor_events/sensor'
+require 'tcell_agent/sensor_events/app_sensor'
+require 'tcell_agent/sensor_events/server_agent'
+require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+require 'tcell_agent/sensor_events/util/redirect_utils'
+
+require 'tcell_agent/rails/routes'
+
+require 'tcell_agent/userinfo'
+require 'cgi'
+
+require 'tcell_agent/instrumentation'
+require 'thread'
+
+module TCellAgent
+  module Instrumentation
+    module Rails
+      module Middleware
+        TCELL_ID = "tcell.request_data"
+
+        class ContextMiddleware
+          THREADS = {}
+          def initialize(app)
+            @app = app
+          end
+
+          def call(env)
+            env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID] = TCellAgent::Instrumentation::TCellData.new
+            TCellAgent::Instrumentation.safe_block("Setting transaction_id") {
+              env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].transaction_id = SecureRandom.uuid
+              request = Rack::Request.new(env)
+              env[TCellAgent::Instrumentation::Rails::Middleware::TCELL_ID].uri = request.fullpath
+            }
+            env["filter_body_set"] = Set.new
+            ContextMiddleware::THREADS[Thread.current.object_id] = env
+            response = @app.call(env)
+            ContextMiddleware::THREADS.delete(Thread.current.object_id)
+            response
+          end
+        end
+      end
+    end
+  end
+end