tcell_agent 2.7.0 → 2.7.1

data/lib/tcell_agent/rails/dlp.rb

@@ -1,410 +0,0 @@
-# See the file "LICENSE" for the full license governing this code.
-
-require 'rails'
-require 'uri'
-require 'tcell_agent/agent'
-require 'tcell_agent/sensor_events/sensor'
-require 'tcell_agent/sensor_events/server_agent'
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
-
-require 'tcell_agent/sensor_events/dlp'
-
-require 'tcell_agent/rails/middleware/global_middleware'
-require 'tcell_agent/rails/middleware/body_filter_middleware'
-require 'tcell_agent/rails/middleware/headers_middleware'
-require 'tcell_agent/rails/middleware/context_middleware'
-
-require 'tcell_agent/rails/settings_reporter'
-
-require 'tcell_agent/instrumentation'
-
-require 'cgi'
-require 'thread'
-
-require 'tcell_agent/configuration'
-require 'tcell_agent/rails/responses'
-
-module TCellAgent
-  module DLP
-    def self.get_dlp_logger
-      unless defined?(@rails_dlp_logger)
-        @rails_dlp_logger = TCellAgent::ModuleLogger.new(
-          TCellAgent.logger, name
-        )
-      end
-
-      @rails_dlp_logger
-    end
-
-    def self.instrument_pluck(results, column_names, model)
-      return if results.empty?
-
-      if TCellAgent.configuration.should_instrument? &&
-         TCellAgent.configuration.should_intercept_requests?
-
-        dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
-        request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
-        tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
-
-        if tcell_context
-          tcell_context.database_result_sizes.push(results.size)
-
-          if dlp_policy && dlp_policy.enabled
-            database_name = model.connection_config.fetch(
-              :database, '*'
-            ).split('/').last
-            table_name = model.table_name
-            column_names = if column_names.size.zero?
-                             model.columns.map(&:name)
-                           else
-                             column_names.map(&:to_s)
-                           end
-
-            if dlp_policy.database_discovery_enabled
-              TCellAgent.discover_database_fields(
-                tcell_context.route_id,
-                database_name,
-                '*',
-                table_name,
-                column_names
-              )
-            end
-
-            normalized_column_names = {}
-            column_name_to_rules = column_names.each_with_object({}) do |namespaced_column_name, memo|
-              namespace = nil
-              column_name = namespaced_column_name
-              if column_name =~ /\./
-                namespace, column_name = column_name.split(/\./)
-              end
-              normalized_column_names[namespaced_column_name] = column_name
-
-              next unless column_name && (!namespace || namespace == table_name)
-
-              rules = dlp_policy.get_actions_for_table(
-                database_name,
-                '*',
-                table_name,
-                column_name,
-                tcell_context.route_id
-              )
-
-              memo[namespaced_column_name] = rules if rules
-            end
-
-            if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
-              get_dlp_logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
-            end
-
-            return if column_name_to_rules.empty?
-
-            # column_names.size == 1
-            # results => [1, 2, 3, 4]
-            # column_names.size > 1
-            # results => [[1, 'email'], [2, 'email']]
-            if column_names.size == 1
-              results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |result|
-                namespaced_column_name = column_names[0]
-                rules = column_name_to_rules.fetch(namespaced_column_name, [])
-                rules.each do |rule|
-                  tcell_context.add_response_db_filter(
-                    result,
-                    rule,
-                    database_name,
-                    '*',
-                    table_name,
-                    normalized_column_names[namespaced_column_name]
-                  )
-                end
-              end
-            else
-              results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |result|
-                result.each_with_index do |val, index|
-                  namespaced_column_name = column_names[index]
-                  rules = column_name_to_rules.fetch(namespaced_column_name, [])
-                  rules.each do |rule|
-                    tcell_context.add_response_db_filter(
-                      val,
-                      rule,
-                      database_name,
-                      '*',
-                      table_name,
-                      normalized_column_names[namespaced_column_name]
-                    )
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-
-    def self.instrument_find_by_sql(results)
-      return if results.empty?
-
-      if TCellAgent.configuration.should_instrument? &&
-         TCellAgent.configuration.should_intercept_requests?
-
-        dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
-        request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, {})
-        tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
-
-        if tcell_context
-          tcell_context.database_result_sizes.push(results.size)
-
-          if dlp_policy && dlp_policy.enabled
-            first_record = results.first
-            database_name = first_record.class.connection_config.fetch(:database, '*').split('/').last
-            model = first_record.class
-            column_names = model.columns.map(&:name)
-            table_name = model.table_name
-
-            if dlp_policy.database_discovery_enabled
-              TCellAgent.discover_database_fields(
-                tcell_context.route_id,
-                database_name,
-                '*',
-                table_name,
-                column_names
-              )
-            end
-
-            if results.size > TCellAgent.configuration.max_data_ex_db_records_per_request
-              get_dlp_logger.warn("Route (#{tcell_context.route_id}) retrieved too many records")
-            end
-
-            column_name_to_rules = column_names.each_with_object({}) do |column_name, memo|
-              rules = dlp_policy.get_actions_for_table(
-                database_name,
-                '*',
-                table_name,
-                column_name,
-                tcell_context.route_id
-              )
-
-              memo[column_name] = rules if rules
-            end
-
-            return if column_name_to_rules.empty?
-
-            results[0...TCellAgent.configuration.max_data_ex_db_records_per_request].each do |record|
-              column_name_to_rules.each do |column_name, rules|
-                next unless rules
-
-                rules.each do |rule|
-                  tcell_context.add_response_db_filter(
-                    record[column_name.to_sym],
-                    rule,
-                    database_name,
-                    '*',
-                    table_name,
-                    column_name
-                  )
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-
-  class MyRailtie < Rails::Railtie
-    initializer 'activeservice.autoload', :after => :set_autoload_paths do |_app|
-      if defined?(ActiveRecord)
-        ActiveRecord::ConnectionAdapters::AbstractAdapter.class_eval do
-          alias_method :tcell_translate_exception, :translate_exception
-          def translate_exception(exception, message)
-            result = tcell_translate_exception(exception, message)
-
-            TCellAgent::Instrumentation.safe_block('Set sql_exception_detected in meta') do
-              appfirewall_policy = TCellAgent.policy(TCellAgent::PolicyTypes::APPSENSOR)
-              if appfirewall_policy.enabled
-                request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(
-                  Thread.current.object_id, {}
-                )
-                tcell_data = request_env[TCellAgent::Instrumentation::TCELL_ID]
-                if tcell_data && result.is_a?(ActiveRecord::StatementInvalid)
-                  if message.is_a? Hash
-                    tcell_data.sql_exceptions.push(
-                      { 'exception_name' => result.class.name, 'exception_payload' => message[:message] }
-                    )
-                  else
-                    tcell_data.sql_exceptions.push(
-                      { 'exception_name' => result.class.name, 'exception_payload' => message }
-                    )
-                  end
-                end
-              end
-            end
-
-            result
-          end
-        end
-
-        ActiveRecord::Calculations.module_eval do
-          alias_method :tcell_pluck, :pluck
-          def pluck(*column_names)
-            results = tcell_pluck(*column_names)
-
-            TCellAgent::Instrumentation.safe_block('Running DLP on pluck') do
-              TCellAgent::DLP.instrument_pluck(results, column_names, model)
-            end
-
-            results
-          end
-        end
-
-        ActiveRecord::Querying.module_eval do
-          if ::Rails::VERSION::MAJOR >= 5
-            alias_method :tcell_find_by_sql, :find_by_sql
-            def find_by_sql(*args)
-              results = tcell_find_by_sql(*args)
-
-              TCellAgent::Instrumentation.safe_block('Running DLP on find_by_sql') do
-                TCellAgent::DLP.instrument_find_by_sql(results)
-              end
-
-              results
-            end
-
-          elsif ::Rails::VERSION::MAJOR < 5
-            alias_method :tcell_find_by_sql, :find_by_sql
-            def find_by_sql(sql, binds = [])
-              results = tcell_find_by_sql(sql, binds)
-
-              TCellAgent::Instrumentation.safe_block('Running DLP on find_by_sql') do
-                TCellAgent::DLP.instrument_find_by_sql(results)
-              end
-
-              results
-            end
-          end
-        end
-      end
-    end
-  end
-end
-
-# - Request
-# -   Session Id event
-# -   Session Id redact
-# -   Session Id hash
-# -   Session Id mask
-# -   Database-Stuff - [event, redact]
-#
-# - Log
-#
-
-module TCellAgent
-  module Policies
-    class DataLossPolicy
-      def log_enforce(tcell_context, sanitize_string)
-        if TCellAgent.configuration.should_instrument? &&
-           TCellAgent.configuration.should_intercept_requests?
-          session_id_actions = get_actions_for_session_id
-          if tcell_context && tcell_context.session_id && session_id_actions
-            send_event = false
-            sanitize_string.gsub!(tcell_context.session_id) do |m|
-              if session_id_actions.log_redact
-                send_event = true
-                m = '[session_id]'
-              elsif session_id_actions.log_hash
-                send_event = true
-                m = '[hash]'
-              elsif session_id_actions.log_event
-                send_event = true
-              end
-              m
-            end
-            if send_event
-              TCellAgent.send_event(
-                TCellAgent::SensorEvents::DlpEvent.new(
-                  tcell_context.route_id,
-                  tcell_context.uri,
-                  TCellAgent::SensorEvents::DlpEvent::FOUND_IN_LOG
-                ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
-              )
-            end
-          end
-        end
-
-        sanitize_string
-      end
-
-      def response_body_enforce(tcell_context, sanitize_string)
-        if TCellAgent.configuration.should_instrument? &&
-           TCellAgent.configuration.should_intercept_requests?
-          session_id_actions = get_actions_for_session_id
-          if tcell_context && tcell_context.session_id && session_id_actions
-            send_event = false
-            sanitize_string.gsub!(tcell_context.session_id) do |m|
-              # rubocop:disable Lint/DuplicateBranch
-              if session_id_actions.body_redact
-                # m = "[session_id]"
-                send_event = true
-              elsif session_id_actions.body_hash
-                # m = "[hash]"
-                send_event = true
-              elsif session_id_actions.body_event
-                send_event = true
-              end
-              # rubocop:enable Lint/DuplicateBranch
-              m
-            end
-          end
-          if send_event
-            TCellAgent.send_event(
-              TCellAgent::SensorEvents::DlpEvent.new(
-                tcell_context.route_id,
-                tcell_context.uri,
-                TCellAgent::SensorEvents::DlpEvent::FOUND_IN_BODY
-              ).for_framework(TCellAgent::SensorEvents::DlpEvent::FRAMEWORK_VARIABLE_SESSION_ID)
-            )
-          end
-        end
-
-        sanitize_string
-      end
-    end
-  end
-end
-
-class Logger
-  alias_method :tcell_old_add, :add
-  def add(severity, message = nil, progname = nil)
-    return tcell_old_add(severity, message, progname) unless severity >= level
-
-    if severity >= level
-      progname ||= @progname
-      if message.nil?
-        if block_given?
-          message = yield
-        else
-          message = progname
-          progname = @progname
-        end
-      end
-
-      if TCellAgent.configuration.enabled &&
-         TCellAgent.configuration.should_instrument? &&
-         TCellAgent.configuration.should_intercept_requests?
-
-        TCellAgent::Instrumentation.safe_block_no_log('Handling DLP log message filtering') do
-          dataloss_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
-          return tcell_old_add(severity, message, progname) unless dataloss_policy && dataloss_policy.enabled
-
-          request_env = TCellAgent::Instrumentation::Rails::Middleware::ContextMiddleware::THREADS.fetch(Thread.current.object_id, nil)
-
-          if message && request_env
-            tcell_context = request_env[TCellAgent::Instrumentation::TCELL_ID]
-            tcell_context.filter_log(message) if tcell_context
-          end
-        end
-      end
-    end
-
-    tcell_old_add(severity, message, progname)
-  end
-end

data/lib/tcell_agent/rails/dlp_handler.rb

@@ -1,63 +0,0 @@
-require 'tcell_agent/instrumentation'
-require 'tcell_agent/rails/responses'
-
-module TCellAgent
-  module Instrumentation
-    module Rails
-      module DLPHandler
-        def self.report_and_redact_now(dlp_handler, tcell_context, rack_body, content_length)
-          TCellAgent::Instrumentation.safe_block('Handling DLP Report and Redact Now') do
-            if dlp_handler
-              new_content_length = 0
-              new_body = []
-              rack_body.each do |str|
-                dlp_handler.call(tcell_context, str)
-                new_body << str
-                new_content_length += str.bytesize
-              end
-              rack_body.close if rack_body.respond_to?(:close)
-
-              rack_body = new_body
-              content_length = new_content_length
-            end
-          end
-
-          [rack_body, content_length]
-        end
-
-        def self.handle_dlp!(tcell_context, response)
-          TCellAgent::Instrumentation.safe_block('Running DLP Logging Filters') do
-            tcell_context.filter_body!(response)
-          end
-
-          response
-        end
-
-        def self.get_handler_and_context(request)
-          dlp_handler = nil
-          tcell_context = nil
-
-          TCellAgent::Instrumentation.safe_block('DLP Handler get handler and context') do
-            if TCellAgent.configuration.should_instrument? &&
-               TCellAgent.configuration.should_intercept_requests?
-
-              # do all this work so that dlp doesn't run at all unless it's on and there
-              # are rules to run
-              dlp_policy = TCellAgent.policy(TCellAgent::PolicyTypes::DATALOSS)
-              if dlp_policy && dlp_policy.get_actions_for_session_id
-                tcell_context = request.env[TCellAgent::Instrumentation::TCELL_ID]
-                if tcell_context && tcell_context.session_id
-                  dlp_handler = proc { |tc, resp|
-                    handle_dlp!(tc, resp)
-                  }
-                end
-              end
-            end
-          end
-
-          [dlp_handler, tcell_context]
-        end
-      end
-    end
-  end
-end

data/lib/tcell_agent/sensor_events/dlp.rb

@@ -1,53 +0,0 @@
-# See the file "LICENSE" for the full license governing this code.
-
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
-require 'tcell_agent/sensor_events/sensor'
-
-module TCellAgent
-  module SensorEvents
-    class DlpEvent < TCellSensorEvent
-      FOUND_IN_BODY = 'body'.freeze
-      FOUND_IN_LOG = 'log'.freeze
-      FOUND_IN_CONSOLE = 'console'.freeze
-
-      FRAMEWORK_VARIABLE_SESSION_ID = 'session_id'.freeze
-
-      REQUEST_CONTEXT_FORM = 'form'.freeze
-      REQUEST_CONTEXT_COOKIE = 'cookie'.freeze
-      REQUEST_CONTEXT_HEADER = 'header'.freeze
-
-      def initialize(route_id, raw_uri, found_in, id = nil, hmac_session_id = nil, user_id = nil)
-        super('dlp')
-        self['rid'] = route_id if route_id
-        self['found_in'] = found_in
-        self['uri'] = Util.strip_uri_values(raw_uri) if raw_uri
-        self['sid'] = hmac_session_id if hmac_session_id
-        self['uid'] = user_id if user_id
-        self['rule'] = id if id
-      end
-
-      def for_database(database, schema, table, field)
-        self['type'] = 'db'
-        self['db'] = database
-        self['schema'] = schema
-        self['table'] = table
-        self['field'] = field
-        self
-      end
-
-      def for_framework(variable)
-        self['type'] = 'fw'
-        self['context'] = 'framework'
-        self['variable'] = variable
-        self
-      end
-
-      def for_request(variable_context, variable)
-        self['type'] = 'req'
-        self['context'] = variable_context
-        self['variable'] = variable
-        self
-      end
-    end
-  end
-end

data/lib/tcell_agent/sinatra.rb

@@ -1,38 +0,0 @@
-# See the file "LICENSE" for the full license governing this code.
-
-require 'sinatra/base'
-require 'sinatra'
-require 'tcell_agent/agent'
-require 'tcell_agent/instrumentation'
-
-module TCellAgent
-  class Sinatra::Response # rubocop:disable Style/ClassAndModuleChildren
-    include Sinatra
-
-    alias_method :original_finish, :finish
-    def finish
-      status, headers, response = original_finish
-
-      TCellAgent::Instrumentation.safe_block('Setting Headers') do
-        headers_policy = TCellAgent.policy(TCellAgent::PolicyTypes::HEADERS)
-        policy_headers = headers_policy.get_headers(
-          headers['Content-Type'],
-          request.env[TCellAgent::Instrumentation::TCELL_ID]
-        )
-        policy_headers.each do |header_info|
-          header_name = header_info['name']
-          header_value = header_info['value']
-          existing_header_value = headers[header_name]
-          headers[header_name] = if existing_header_value
-                                   "#{existing_header_value}, #{header_value}"
-                                 else
-                                   header_value
-                                 end
-        end
-        response = [status, headers, active_response]
-      end
-
-      [status, headers, response]
-    end
-  end
-end