tcell_agent 1.1.3 → 1.1.4

data/lib/tcell_agent/rails/auth/doorkeeper.rb

@@ -14,7 +14,7 @@ if TCellAgent.configuration.should_instrument_doorkeeper?
             TCellAgent::Instrumentation.safe_block('Doorkeeper Token Authorize') do
               if TCellAgent.configuration.enabled &&
                  TCellAgent.configuration.should_intercept_requests?
-                login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
+                login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LOGINFRAUD)
                 if login_fraud_policy &&
                    login_fraud_policy.enabled &&
                    login_fraud_policy.login_failed_enabled
@@ -59,7 +59,7 @@ if TCellAgent.configuration.should_instrument_doorkeeper?
               if TCellAgent.configuration.enabled &&
                  TCellAgent.configuration.should_intercept_requests?
                 if pre_auth.error
-                  login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LoginFraud)
+                  login_fraud_policy = TCellAgent.policy(TCellAgent::PolicyTypes::LOGINFRAUD)
                   if login_fraud_policy &&
                      login_fraud_policy.enabled &&
                      login_fraud_policy.login_failed_enabled

data/lib/tcell_agent/rails/better_ip.rb

@@ -1,28 +1,25 @@
-require "tcell_agent/utils/strings"
+require 'tcell_agent/utils/strings'
 require 'tcell_agent/instrumentation'
 
-
 module TCellAgent
   module Utils
     module Rails
-
       def self.better_ip(request)
         if TCellAgent.configuration.reverse_proxy
-          TCellAgent::Instrumentation.safe_block("Extracting reverse proxy IP") do
+          TCellAgent::Instrumentation.safe_block('Extracting reverse proxy IP') do
             reverse_proxy_header = TCellAgent.configuration.reverse_proxy_ip_address_header
-            if TCellAgent::Utils::Strings.present?(reverse_proxy_header)
-              reverse_proxy_header = "HTTP_" + reverse_proxy_header.upcase().tr('-','_')
-            else
-              reverse_proxy_header = "HTTP_X_FORWARDED_FOR"
-            end
+            reverse_proxy_header = if TCellAgent::Utils::Strings.present?(reverse_proxy_header)
+                                     'HTTP_' + reverse_proxy_header.upcase.tr('-', '_')
+                                   else
+                                     'HTTP_X_FORWARDED_FOR'
+                                   end
 
             x_forwarded_for = request.env[reverse_proxy_header]
-
-            if TCellAgent::Utils::Strings.present?(x_forwarded_for)
-              ip = x_forwarded_for.split(',')[0].strip()
-            else
-              ip = request.ip
-            end
+            ip = if TCellAgent::Utils::Strings.present?(x_forwarded_for)
+                   x_forwarded_for.split(',')[0].strip
+                 else
+                   request.ip
+                 end
 
             return ip
           end
@@ -30,7 +27,6 @@ module TCellAgent
 
         request.ip
       end
-
     end
   end
 end

data/lib/tcell_agent/rails/csrf_exception.rb

@@ -1,15 +1,14 @@
 require 'tcell_agent/instrumentation'
 
 module TCellAgent
-
   module CsrfExceptionReporter
     def handle_unverified_request
-      TCellAgent::Instrumentation.safe_block("AppSensor CSRF Exception processing") do
-        rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+      TCellAgent::Instrumentation.safe_block('AppSensor CSRF Exception processing') do
+        rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
         if rust_policies && rust_policies.appfirewall_enabled
           tcell_data = request.env[TCellAgent::Instrumentation::TCELL_ID]
           if tcell_data
-            tcell_data.csrf_exception_name = ActionController::InvalidAuthenticityToken.name
+            tcell_data.csrf_exception_name = 'ActionController::InvalidAuthenticityToken'
           end
         end
       end
@@ -19,12 +18,10 @@ module TCellAgent
   end
 
   class MyRailtie < Rails::Railtie
-
-    initializer "tcell.sensors" do |app|
+    initializer 'tcell.sensors' do |_app|
       ActiveSupport.on_load :action_controller do
         ActionController::Base.send(:include, TCellAgent::CsrfExceptionReporter)
       end
     end
-
   end
 end

data/lib/tcell_agent/rails/dlp.rb

@@ -29,75 +29,173 @@ require 'thread'
 require 'tcell_agent/configuration'
 require 'tcell_agent/rails/responses'
 
-
 module TCellAgent
   module DLP
-    def self.instrument_find_by_sql(results)
-      if results.size > 0
-
-        if TCellAgent.configuration.enabled &&
-            TCellAgent.configuration.should_instrument? &&
-            TCellAgent.configuration.should_intercept_requests?
-
-          dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+    def self.instrument_pluck(results, column_names, model)
+      return if results.empty?
+
+      if TCellAgent.configuration.enabled &&
+         TCellAgent.configuration.should_instrument? &&
+         TCellAgent.configuration.should_intercept_requests?
+
+        dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
+        request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+        tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
+
+        if tcell_context
+          tcell_context.database_result_sizes.push(results.size)
+
+          if dlp_policy && dlp_policy.enabled
+            database_name = model.connection_config.fetch(
+              :database, '*'
+            ).split('/').last
+            table_name = model.table_name
+            column_names = if column_names.size.zero?
+                             model.columns.map(&:name)
+                           else
+                             column_names.map(&:to_s)
+                           end
+
+            if dlp_policy.database_discovery_enabled
+              TCellAgent.discover_database_fields(
+                tcell_context.route_id,
+                database_name,
+                '*',
+                table_name,
+                column_names
+              )
+            end
 
-          request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
-          tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
+            normalized_column_names = {}
+            column_name_to_rules = column_names.each_with_object({}) do |namespaced_column_name, memo|
+              namespace = nil
+              column_name = namespaced_column_name
+              if column_name =~ /\./
+                namespace, column_name = column_name.split(/\./)
+              end
+              normalized_column_names[namespaced_column_name] = column_name
+
+              next unless column_name && (!namespace || namespace == table_name)
+              rules = dlp_policy.get_actions_for_table(
+                database_name,
+                '*',
+                table_name,
+                column_name,
+                tcell_context.route_id
+              )
 
-          if tcell_context
-            tcell_context.database_result_sizes.push(results.size)
+              memo[namespaced_column_name] = rules if rules
+            end
 
-            if dlp_policy && dlp_policy.enabled
-              first_record = results.first
-              database_name = first_record.class.connection_config().fetch(:database,"*").split('/').last
-              model = first_record.class
-              column_names = model.columns.map { |col| col.name }
-              table_name = model.table_name
+            if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
+              TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
+            end
 
-              if dlp_policy.database_discovery_enabled
-                TCellAgent.discover_database_fields(
-                  tcell_context.route_id,
-                  database_name,
-                  "*",
-                  table_name,
-                  column_names
-                )
+            return if column_name_to_rules.empty?
+
+            # column_names.size == 1
+            # results => [1, 2, 3, 4]
+            # column_names.size > 1
+            # results => [[1, 'email'], [2, 'email']]
+            if column_names.size == 1
+              results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |result|
+                namespaced_column_name = column_names[0]
+                rules = column_name_to_rules.fetch(namespaced_column_name, [])
+                rules.each do |rule|
+                  tcell_context.add_response_db_filter(
+                    result,
+                    rule,
+                    database_name,
+                    '*',
+                    table_name,
+                    normalized_column_names[namespaced_column_name]
+                  )
+                end
               end
-
-              if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
-                TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
+            else
+              results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |result|
+                result.each_with_index do |val, index|
+                  namespaced_column_name = column_names[index]
+                  rules = column_name_to_rules.fetch(namespaced_column_name, [])
+                  rules.each do |rule|
+                    tcell_context.add_response_db_filter(
+                      val,
+                      rule,
+                      database_name,
+                      '*',
+                      table_name,
+                      normalized_column_names[namespaced_column_name]
+                    )
+                  end
+                end
               end
+            end
+          end
+        end
+      end
+    end
 
-              column_name_to_rules = column_names.inject({}) do |memo, column_name|
-                rules = dlp_policy.get_actions_for_table(
-                  database_name,
-                  "*",
-                  table_name,
-                  column_name,
-                  tcell_context.route_id
-                )
+    def self.instrument_find_by_sql(results)
+      return if results.empty?
+
+      if TCellAgent.configuration.enabled &&
+         TCellAgent.configuration.should_instrument? &&
+         TCellAgent.configuration.should_intercept_requests?
+
+        dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
+        request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+        tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
+
+        if tcell_context
+          tcell_context.database_result_sizes.push(results.size)
+
+          if dlp_policy && dlp_policy.enabled
+            first_record = results.first
+            database_name = first_record.class.connection_config.fetch(:database, '*').split('/').last
+            model = first_record.class
+            column_names = model.columns.map(&:name)
+            table_name = model.table_name
+
+            if dlp_policy.database_discovery_enabled
+              TCellAgent.discover_database_fields(
+                tcell_context.route_id,
+                database_name,
+                '*',
+                table_name,
+                column_names
+              )
+            end
 
-                memo[column_name] = rules if rules
+            if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
+              TCellAgent.logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
+            end
 
-                memo
-              end
+            column_name_to_rules = column_names.each_with_object({}) do |column_name, memo|
+              rules = dlp_policy.get_actions_for_table(
+                database_name,
+                '*',
+                table_name,
+                column_name,
+                tcell_context.route_id
+              )
 
-              return if column_name_to_rules.empty?
-
-              results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |record|
-                column_name_to_rules.each do |column_name, rules|
-                  if rules
-                    rules.each do |rule|
-                      tcell_context.add_response_db_filter(
-                        record[column_name.to_sym],
-                        rule,
-                        database_name,
-                        "*",
-                        table_name,
-                        column_name
-                      )
-                    end
-                  end
+              memo[column_name] = rules if rules
+            end
+
+            return if column_name_to_rules.empty?
+
+            results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |record|
+              column_name_to_rules.each do |column_name, rules|
+                next unless rules
+                rules.each do |rule|
+                  tcell_context.add_response_db_filter(
+                    record[column_name.to_sym],
+                    rule,
+                    database_name,
+                    '*',
+                    table_name,
+                    column_name
+                  )
                 end
               end
             end
@@ -108,23 +206,24 @@ module TCellAgent
   end
 
   class MyRailtie < Rails::Railtie
-    initializer 'activeservice.autoload', :after => :set_autoload_paths do |app|
-
+    initializer 'activeservice.autoload', :after => :set_autoload_paths do |_app|
       if defined?(ActiveRecord)
         ActiveRecord::ConnectionAdapters::AbstractAdapter.class_eval do
           alias_method :tcell_translate_exception, :translate_exception
-          def translate_exception(e, message)
-            result = tcell_translate_exception(e, message)
+          def translate_exception(exception, message)
+            result = tcell_translate_exception(exception, message)
 
-            TCellAgent::Instrumentation.safe_block("Set sql_exception_detected in meta") do
-              rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::Rust)
+            TCellAgent::Instrumentation.safe_block('Set sql_exception_detected in meta') do
+              rust_policies = TCellAgent.policy(TCellAgent::PolicyTypes::RUST)
               if rust_policies && rust_policies.appfirewall_enabled
-                request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
+                request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(
+                  Thread.current.object_id, {}
+                )
                 tcell_data = request_env[TCellAgent::Instrumentation::TCELL_ID]
                 if tcell_data && result.is_a?(ActiveRecord::StatementInvalid)
-                  tcell_data.sql_exceptions.push({
-                    "exception_name" => result.class.name, "exception_payload" => message
-                  })
+                  tcell_data.sql_exceptions.push(
+                    { 'exception_name' => result.class.name, 'exception_payload' => message }
+                  )
                 end
               end
             end
@@ -133,86 +232,90 @@ module TCellAgent
           end
         end
 
-        ActiveRecord::Querying.module_eval do
+        ActiveRecord::Calculations.module_eval do
+          alias_method :tcell_pluck, :pluck
+          def pluck(*column_names)
+            results = tcell_pluck(*column_names)
 
-          if (::Rails::VERSION::MAJOR == 5)
+            TCellAgent::Instrumentation.safe_block('Running DLP on pluck') do
+              TCellAgent::DLP.instrument_pluck(results, column_names, model)
+            end
+
+            results
+          end
+        end
+
+        ActiveRecord::Querying.module_eval do
+          if ::Rails::VERSION::MAJOR == 5
             alias_method :tcell_find_by_sql, :find_by_sql
             def find_by_sql(*args)
               results = tcell_find_by_sql(*args)
 
-              TCellAgent::Instrumentation.safe_block("Running DLP on find_by_sql") do
+              TCellAgent::Instrumentation.safe_block('Running DLP on find_by_sql') do
                 TCellAgent::DLP.instrument_find_by_sql(results)
               end
 
               results
             end
 
-          elsif (::Rails::VERSION::MAJOR < 5)
+          elsif ::Rails::VERSION::MAJOR < 5
             alias_method :tcell_find_by_sql, :find_by_sql
             def find_by_sql(sql, binds = [])
               results = tcell_find_by_sql(sql, binds)
 
-              TCellAgent::Instrumentation.safe_block("Running DLP on find_by_sql") do
+              TCellAgent::Instrumentation.safe_block('Running DLP on find_by_sql') do
                 TCellAgent::DLP.instrument_find_by_sql(results)
               end
 
               results
             end
           end
-
         end
-
       end
-
     end
   end
 end
 
-
-
-
-
 # - Request
 # -   Session Id event
 # -   Session Id redact
 # -   Session Id hash
 # -   Session Id mask
 # -   Database-Stuff - [event, redact]
-# 
+#
 # - Log
 #
 
 module TCellAgent
   module Policies
     class DataLossPolicy
-
       def log_enforce(tcell_context, sanitize_string)
         if TCellAgent.configuration.enabled &&
-            TCellAgent.configuration.should_instrument? &&
-            TCellAgent.configuration.should_intercept_requests?
-          if (tcell_context && tcell_context.session_id)
-            session_id_actions = self.get_actions_for_session_id
+           TCellAgent.configuration.should_instrument? &&
+           TCellAgent.configuration.should_intercept_requests?
+          if tcell_context && tcell_context.session_id
+            session_id_actions = get_actions_for_session_id
             if session_id_actions
               send_event = false
-              sanitize_string.gsub!(tcell_context.session_id) {|m|
+              sanitize_string.gsub!(tcell_context.session_id) do |m|
                 if session_id_actions.log_redact
                   send_event = true
-                  m = "[session_id]"
+                  m = '[session_id]'
                 elsif session_id_actions.log_hash
                   send_event = true
-                  m = "[hash]"
+                  m = '[hash]'
                 elsif session_id_actions.log_event
                   send_event = true
                 end
                 m
-              }
+              end
               if send_event
                 TCellAgent.send_event(
                   TCellAgent::SensorEvents::DlpEvent.new(
                     tcell_context.route_id,
                     tcell_context.uri,
                     TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG
-                    ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
+                  ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
                 )
               end
             end
@@ -224,13 +327,13 @@ module TCellAgent
 
       def response_body_enforce(tcell_context, sanitize_string)
         if TCellAgent.configuration.enabled &&
-            TCellAgent.configuration.should_instrument? &&
-            TCellAgent.configuration.should_intercept_requests?
-          if (tcell_context && tcell_context.session_id)
-            session_id_actions = self.get_actions_for_session_id
+           TCellAgent.configuration.should_instrument? &&
+           TCellAgent.configuration.should_intercept_requests?
+          if tcell_context && tcell_context.session_id
+            session_id_actions = get_actions_for_session_id
             if session_id_actions
               send_event = false
-              sanitize_string.gsub!(tcell_context.session_id) {|m|
+              sanitize_string.gsub!(tcell_context.session_id) do |m|
                 if session_id_actions.body_redact
                   # m = "[session_id]"
                   send_event = true
@@ -241,7 +344,7 @@ module TCellAgent
                   send_event = true
                 end
                 m
-              }
+              end
             end
             if send_event
               TCellAgent.send_event(
@@ -249,7 +352,7 @@ module TCellAgent
                   tcell_context.route_id,
                   tcell_context.uri,
                   TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY
-                  ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
+                ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
               )
             end
           end
@@ -263,11 +366,11 @@ end
 
 class Logger
   alias_method :tcell_old_add, :add
-  def add(severity, message = nil, progname = nil, &block)
+  def add(severity, message = nil, progname = nil)
     if TCellAgent.configuration.enabled &&
-        TCellAgent.configuration.should_instrument? &&
-        TCellAgent.configuration.should_intercept_requests? &&
-        severity >= self.level
+       TCellAgent.configuration.should_instrument? &&
+       TCellAgent.configuration.should_intercept_requests? &&
+       severity >= level
 
       progname ||= @progname
       if message.nil?
@@ -279,16 +382,14 @@ class Logger
         end
       end
 
-      TCellAgent::Instrumentation.safe_block_no_log("Handling DLP log message filtering") {
-        dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DataLoss)
+      TCellAgent::Instrumentation.safe_block_no_log('Handling DLP log message filtering') do
+        dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
         request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
         if message && dlp_policy && request_env
           tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
-          if tcell_context
-            tcell_context.filter_log(message)
-          end
+          tcell_context.filter_log(message) if tcell_context
         end
-      }
+      end
     end
 
     tcell_old_add(severity, message, progname)