tcell_agent 1.1.12 → 2.2.0

data/lib/tcell_agent/agent/route_manager.rb

@@ -1,13 +1,7 @@
 # See the file "LICENSE" for the full license governing this code.
 
-require 'tcell_agent/logger'
-require 'tcell_agent/version'
-require 'tcell_agent/api'
-require 'tcell_agent/configuration'
-
 require 'tcell_agent/routes/table'
 require 'tcell_agent/sensor_events/discovery'
-require 'tcell_agent'
 
 module TCellAgent
   class Agent
@@ -23,16 +17,6 @@ module TCellAgent
     def discover_database_fields(route_id, database, schema, table, fields)
       return if route_id.nil? || database.nil? || schema.nil? || table.nil? || fields.nil?
 
-      if TCellAgent::Agent.parent_process? == false
-        TCellAgent.queue_metric('_type' => 'discover_database_fields',
-                                'route_id' => route_id,
-                                'database' => database,
-                                'schema' => schema,
-                                'table' => table,
-                                'fields' => fields)
-        return
-      end
-
       query_hash = TCellAgent::Agent.get_database_discovery_identifier(database, schema, table, fields)
 
       return if @route_table.routes[route_id].database_queries_discovered.fetch(query_hash, false)

data/lib/tcell_agent/agent/static_agent.rb

@@ -1,24 +1,15 @@
-# See the file "LICENSE" for the full license governing this code.
-require 'tcell_agent/sensor_events/metrics'
-require 'monitor'
-
 module TCellAgent
   @@instance_lock = Mutex.new
   @@my_thread_agent = nil
 
   def self.thread_agent
-    if thread_agent_defined? == false
+    unless @@my_thread_agent
       @@instance_lock.synchronize do
-        if thread_agent_defined? == false
-          @@my_thread_agent = TCellAgent::Agent.new(Process.pid)
-        end
+        @@my_thread_agent ||= TCellAgent::Agent.new
       end
     end
-    @@my_thread_agent
-  end
 
-  def self.thread_agent_defined?
-    @@my_thread_agent != nil
+    @@my_thread_agent
   end
 
   def self.thread_agent=(some_agent)
@@ -31,35 +22,23 @@ module TCellAgent
     thread_agent.queue_sensor_event(event)
   end
 
-  def self.queue_metric(event)
-    thread_agent._queue_metric(event)
+  def self.report_metrics(response_time, tcell_context)
+    thread_agent.report_metrics(response_time, tcell_context)
   end
 
   def self.policy(policy_type)
     thread_agent.policies.fetch(policy_type, nil)
   end
 
-  def self.increment_session_info(hmac_session_id, user_id, ip_address, user_agent)
-    thread_agent.increment_session_info(hmac_session_id, user_id, ip_address, user_agent)
-  end
-
-  def self.increment_route(route_id, response_time)
-    thread_agent.increment_route(route_id, response_time)
-  end
-
   def self.discover_database_fields(route_id, database, schema, table, fields)
     thread_agent.discover_database_fields(route_id, database, schema, table, fields)
   end
 
-  def self.stop_agent
-    thread_agent.stop_agent = true
+  def self.safe_to_check_cmdi?
+    thread_agent && thread_agent.safe_to_check_cmdi
   end
 
-  def self.ensure_event_processor_running
-    thread_agent.ensure_event_processor_running
-  end
-
-  def self.safe_to_send_cmdi_events?
-    thread_agent.safe_to_send_cmdi_events?
+  def self.stop_agent
+    thread_agent.stop_agent = true
   end
 end

data/lib/tcell_agent/config_initializer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module TCellAgent
+  class ConfigInitializer < Configuration
+    # Common config shared across agents
+    attr_accessor :app_id, :api_key, :tcell_api_url,
+                  :tcell_input_url, :log_dir, :fetch_policies_from_tcell,
+                  :preload_policy_filename, :reverse_proxy,
+                  :reverse_proxy_ip_address_header, :host_identifier,
+                  :hmac_key, :password_hmac_key,
+                  :js_agent_url, :js_agent_api_base_url,
+                  :max_csp_header_bytes, :allow_payloads
+
+    attr_reader :logging_options
+
+    # Ruby only config
+    attr_accessor :disable_all, :enabled, :enable_event_manager,
+                  :enable_policy_polling,
+                  :data_exposure
+
+    attr_reader :instrument_for_events,
+                :enable_instrumentation
+
+    # New config
+    attr_accessor :disabled_instrumentation, :instrument
+
+    def initialize
+      # ruby agent defaults
+      @logging_options = {}
+    end
+
+    def logging_options=(hash)
+      @logging_options = hash.each_with_object({}) do |(key, val), memo|
+        memo[key.to_sym] = val
+      end
+    end
+
+    # legacy config mapping
+    def enabled_instrumentations=(hash)
+      @disabled_instrumentation ||= []
+      hash.each do |key, val|
+        @disabled_instrumentation << key.to_s.strip.downcase unless TCellAgent.configuration.to_bool(val)
+      end
+    end
+
+    def agent_log_dir=(path)
+      @log_dir = path
+    end
+
+    def instrument_for_events=(bool)
+      @instrument = bool
+    end
+
+    def enable_instrumentation=(bool)
+      @instrument = bool
+    end
+
+    def enable_intercept_requests=(bool)
+      @disabled_instrumentation << 'intercept_requests' unless TCellAgent.configuration.to_bool(bool)
+    end
+
+    def get_config_file_dir
+      super
+    end
+  end
+end

data/lib/tcell_agent/configuration.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # See the file "LICENSE" for the full license governing this code.
 require 'fileutils'
 require 'json'
@@ -6,396 +8,118 @@ require 'socket'
 require 'securerandom'
 require 'uri'
 
-require 'tcell_agent/config/unknown_options'
-
 module TCellAgent
   class ConfigurationException < StandardError
   end
 
   class << self
     attr_accessor :configuration
+    attr_accessor :initializer_configuration
   end
 
   def self.configure
-    self.configuration ||= Configuration.new
-    yield(configuration)
-  end
-
-  class Configuration # rubocop:disable Metrics/ClassLength
-    attr_accessor :version,
-                  :app_id,
-                  :api_key,
-                  :hmac_key,
-                  :tcell_input_url,
-                  :logging_options,
-                  :logger,
-                  :appfirewall_payloads_logger, # appfirewall_payloads_logger can be specified from initializers
-                  :fetch_policies_from_tcell, :instrument_for_events,
-                  :preload_policy_filename,
-                  :host_identifier,
-                  :uuid,
-                  :event_batch_size_limit, :event_time_limit_seconds,
-                  :base_dir,
-                  :cache_filename,
-                  :js_agent_api_base_url,
-                  :js_agent_url,
-                  :config_filename,
-                  :agent_log_dir,
-                  :max_data_ex_db_records_per_request,
-                  :agent_home_dir,
-                  :agent_home_owner,
-                  :reverse_proxy,
-                  :reverse_proxy_ip_address_header,
-                  :log_file_name,
-                  :log_tag,
-                  :max_csp_header_bytes,
-                  :demomode,
-                  :allow_payloads,
-                  :password_hmac_key
-
-    attr_reader   :tcell_api_url
-
-    attr_accessor :disable_all,
-                  :enabled,
-                  :enable_event_manager,       # false = Do not start the even manager
-                  :enable_event_consumer,      # false = Do not consume events, drop them
-                  :enable_policy_polling,      # false = Do not poll for policies
-                  :enable_instrumentation,     # false = Do not add instrumentation
-                  :enable_intercept_requests,  # false = Do not insert middleware
-                  :enable_child_process_events # true = Start an event processor on all processes, even children
+    require 'tcell_agent/config_initializer'
+    self.initializer_configuration ||= ConfigInitializer.new
 
-    attr_accessor :enabled_instrumentations
+    yield(initializer_configuration)
+  rescue NoMethodError => e
+    logger = TCellAgent::ModuleLogger.new(TCellAgent::RubyLogger.new, name)
+    logger.error("Error configuring tcell_agent with initializers: #{e}")
+  end
 
-    attr_accessor :exp_config_settings
+  class Configuration
+    # internal Ruby configurations
+    attr_accessor :disable_all
 
-    attr_accessor :disable_cmdi_exec_instrumentation # true = disable cmdi Kernel::exec instrumentation
+    # Common config returned by libtcellagent
+    attr_accessor :api_key, :app_id, :disabled_instrumentation, :enabled,
+                  :enable_intercept_requests, :fetch_policies_from_tcell, :hmac_key,
+                  :host_identifier, :instrument, :log_dir, :logging_options,
+                  :password_hmac_key, :reverse_proxy, :reverse_proxy_ip_address_header,
+                  :tcell_api_url
 
-    def tcell_api_url=(value)
-      @tcell_api_url = value
-      @tcell_api_url = compose_api_url!
-    end
-
-    def should_start_event_manager?
-      @enabled && @enable_event_manager
-    end
-
-    def should_start_event_manager_in_child_processes?
-      @enabled && @enable_event_manager && @enable_child_process_events
-    end
-
-    def should_consume_event?
-      @enabled && @enable_event_manager && @enable_event_consumer
-    end
+    # Ruby config returned by libtcellagent
+    attr_accessor :enable_policy_polling
 
     def should_start_policy_poll?
       @enabled && @enable_policy_polling && @fetch_policies_from_tcell # fetch_policies_from_tcel = legacy
     end
 
-    def should_instrument?
-      @enabled && @enable_instrumentation && @instrument_for_events # instrument_for_events = legacy
-    end
-
-    def should_intercept_requests?
-      @enabled && @enable_instrumentation && @enable_intercept_requests
-    end
-
-    def should_instrument_doorkeeper?
-      if @enabled_instrumentations.key?('doorkeeper') || @enabled_instrumentations.key?(:doorkeeper)
-        !!(@enabled_instrumentations['doorkeeper'] || @enabled_instrumentations[:doorkeeper]) # rubocop:disable Style/DoubleNegation
-      else
-        true
-      end
-    end
-
-    def should_instrument_devise?
-      if @enabled_instrumentations.key?('devise') || @enabled_instrumentations.key?(:devise)
-        !!(@enabled_instrumentations['devise'] || @enabled_instrumentations[:devise]) # rubocop:disable Style/DoubleNegation
-      else
-        true
-      end
-    end
+    def should_instrument?(func = nil)
+      return false unless @enabled && @instrument # never instrument if disabled
+      return true if func.nil?                    # always instrument if enabled and nothing given
 
-    def should_instrument_authlogic?
-      if @enabled_instrumentations.key?('authlogic') || @enabled_instrumentations.key?(:authlogic)
-        !!(@enabled_instrumentations['authlogic'] || @enabled_instrumentations[:authlogic]) # rubocop:disable Style/DoubleNegation
-      else
-        true
-      end
+      !@disabled_instrumentation.include?(func)
     end
 
-    def should_instrument_cmdi_exec?
-      !@disable_cmdi_exec_instrumentation
+    def should_intercept_requests?
+      @enabled && @enable_intercept_requests
     end
 
-    def initialize(filename = 'config/tcell_agent.config', _useapp = nil)
-      # These will be set when the agent starts up, to give rails initializers
-      # a chance to run
-      @cache_filename = nil
-      @agent_log_dir = nil
-      @log_tag = nil
-
-      @logger = nil
-      @appfirewall_payloads_logger = nil
-
-      @version = 0
-      @exp_config_settings = true
-      @demomode = false
-
-      @fetch_policies_from_tcell = true
-      @instrument_for_events = true
-
+    def initialize
+      # ruby agent defaults
       @disable_all = false
-      @enabled = true
-      @enable_event_manager = true
-      @enable_event_consumer = true
-      @enable_policy_polling = true
-      @enable_instrumentation = true
-      @enable_intercept_requests = true
-      @enable_child_process_events = false
-
-      @enabled_instrumentations = {
-        :doorkeeper => true,
-        :devise => true,
-        :authlogic => true
-      }
-
-      @disable_cmdi_exec_instrumentation = false
-
-      @log_file_name = 'tcell_agent.log'
-
-      @event_batch_size_limit = 50
-      @event_time_limit_seconds = 15
-
-      @max_data_ex_db_records_per_request = 1000
-      @reverse_proxy = true
-      @reverse_proxy_ip_address_header = nil
-      @allow_payloads = true
-
-      @max_csp_header_bytes = nil
-      @password_hmac_key = nil
-
-      @agent_home_dir = ENV['TCELL_AGENT_HOME'] || File.join(Dir.getwd, 'tcell')
-      @config_filename = ENV['TCELL_AGENT_CONFIG'] || File.join(Dir.getwd, filename)
-
-      read_config_from_file(@config_filename)
-      read_config_using_env
-
-      if @demomode
-        @event_batch_size_limit = 1
-        @event_time_limit_seconds = 2
-      end
-
-      if ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS']
-        puts 'tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS is deprecated and will be removed in a future release. Please switch to TCELL_AGENT_ALLOW_PAYLOADS.'
-      end
+      @logging_options = {}
 
-      if ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS']
-        puts 'tCell.io Agent: [DEPRECATED] TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS is deprecated and will be removed in a future release. Please switch to TCELL_AGENT_ALLOW_PAYLOADS.'
-      end
-
-      unless ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS'].nil?
-        @allow_payloads = [true, 'true', 'yes', '1'].include?(ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS'])
-      end
-      unless ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS'].nil?
-        @allow_payloads = [true, 'true', 'yes', '1'].include?(ENV['TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS'])
-      end
-      unless ENV['TCELL_AGENT_ALLOW_PAYLOADS'].nil?
-        @allow_payloads = [true, 'true', 'yes', '1'].include?(ENV['TCELL_AGENT_ALLOW_PAYLOADS'])
-      end
-
-      @tcell_api_url = compose_api_url!
-      @tcell_input_url ||= 'https://input.tcell.io/api/v1'
-      @js_agent_url ||= 'https://jsagent.tcell.io/tcellagent.min.js'
-
-      if @host_identifier.nil?
-        begin
-          @host_identifier = (Socket.gethostname || 'localhost')
-        rescue StandardError
-          @host_identifier = 'host_identifier_not_found'
-        end
-      end
-
-      @uuid = SecureRandom.uuid
+      check_for_disabled_instrumentation(get_config_file_name)
     end
 
-    def compose_api_url!
-      @tcell_api_url ||= 'https://api.tcell.io'
-      parsed_uri = URI.parse(@tcell_api_url)
-
-      api_url = [
-        parsed_uri.scheme,
-        '://',
-        parsed_uri.host
-      ]
+    def get_config_file_dir
+      return nil unless ENV['TCELL_AGENT_HOME']
+      return nil if ENV['TCELL_AGENT_CONFIG']
 
-      api_url.push(":#{parsed_uri.port}") unless [80, 443].include?(parsed_uri.port)
-
-      @js_agent_api_base_url ||= "#{api_url.join('')}/api/v1"
-
-      [
-        api_url.join(''),
-        '/agents/api/v1/apps/',
-        '{app_id}',
-        '/policies/latest',
-        '?',
-        'type=jsagentinjection:v1',
-        '&type=http-redirect:v1',
-        '&type=clickjacking:v1',
-        '&type=secure-headers:v1',
-        '&type=cmdi:v1',
-        '&type=csp-headers:v1',
-        '&type=dlp:v1',
-        '&type=login:v1',
-        '&type=regex:v1',
-        '&type=appsensor:v2',
-        '&type=patches:v1'
-      ].join('')
-    end
-
-    def cache_filename_with_app_id
-      @cache_filename ||= File.join(@agent_home_dir, 'cache', 'tcell_agent.cache')
-
-      if @app_id
-        "#{@cache_filename}.#{@app_id}"
-      else
-        @cache_filename
-      end
+      File.join(Dir.getwd, '/config')
     end
 
-    def read_config_using_env
-      @app_id = ENV['TCELL_AGENT_APP_ID'] || @app_id
-      @api_key = ENV['TCELL_AGENT_API_KEY'] || @api_key
-      @hmac_key = ENV['TCELL_HMAC_KEY'] || @hmac_key
-      @password_hmac_key = ENV['TCELL_PASSWORD_HMAC_KEY'] || @password_hmac_key
-      @host_identifier = ENV['TCELL_AGENT_HOST_IDENTIFIER'] || @host_identifier
-      @tcell_api_url = ENV['TCELL_API_URL'] || @tcell_api_url
-      @tcell_input_url = ENV['TCELL_INPUT_URL'] || @tcell_input_url
-      @demomode = ENV['TCELL_DEMOMODE'] || @demomode
-
-      @agent_home_owner = ENV['TCELL_AGENT_HOME_OWNER'] || @agent_home_owner
-      @agent_log_dir = ENV['TCELL_AGENT_LOG_DIR'] || @agent_log_dir
-
-      @disable_cmdi_exec_instrumentation = ENV['TCELL_CMDI_EXEC_DISABLED'] || @disable_cmdi_exec_instrumentation
-
-      @enabled = ENV['TCELL_AGENT_ENABLED'].to_s.casecmp('true').zero? if %w[true false].include? ENV['TCELL_AGENT_ENABLED'].to_s.downcase
+    def get_config_file_name
+      ENV['TCELL_AGENT_CONFIG'] || File.join(Dir.getwd, '/config/tcell_agent.config')
     end
 
-    def read_config_from_file(filename)
+    def check_for_disabled_instrumentation(filename)
       return unless File.file?(filename)
 
       begin
-        config_text = File.open(filename).read
-        config = JSON.parse(config_text)
-
-        messages = TCellAgent::Config::Validate.get_unknown_options(config)
-        messages.each do |message|
-          puts message
-        end
+        config = JSON.parse(File.open(filename).read)
 
         if config['version'] == 1
-          # Required
-          app_data = config['applications'][0] # Default
-          @version = 1
-          @app_id = app_data['app_id']
-          @api_key = app_data['api_key']
-
-          # Optional
-          @preload_policy_filename = app_data.fetch('preload_policy_filename', nil)
-
-          @disable_all = app_data.fetch('disable_all', @disable_all)
-          @enabled = app_data.fetch('enabled', @enabled)
-
-          @enable_event_manager = app_data.fetch('enable_event_manager', @enable_event_manager)
-          @enable_event_consumer = app_data.fetch('enable_event_consumer', @enable_event_consumer)
-          @enable_policy_polling = app_data.fetch('enable_policy_polling', @enable_policy_polling)
-          @enable_instrumentation = app_data.fetch('enable_instrumentation', @enable_instrumentation)
-          @enable_intercept_requests = app_data.fetch('enable_intercept_requests', @enable_intercept_requests)
-          @fetch_policies_from_tcell = app_data.fetch('fetch_policies_from_tcell', @fetch_policies_from_tcell)
-          @instrument_for_events = app_data.fetch('instrument_for_events', @instrument_for_events)
-          @enable_child_process_events = app_data.fetch('enable_child_process_events', @enable_child_process_events)
-
-          @agent_home_owner = app_data.fetch('agent_home_owner', @agent_home_owner)
-
-          @logging_options = app_data.fetch('logging_options', {})
-          @agent_log_dir = app_data.fetch('log_dir', @agent_log_dir)
-          @log_file_name = @logging_options.fetch('filename', @log_file_name)
-
-          @tcell_api_url = app_data.fetch('tcell_api_url', @tcell_api_url)
-          @tcell_input_url = app_data.fetch('tcell_input_url', @tcell_input_url)
-
-          @max_csp_header_bytes = app_data.fetch('max_csp_header_bytes', @max_csp_header_bytes)
-
-          @allow_payloads = app_data.fetch(
-            'allow_unencrypted_appsensor_payloads',
-            @allow_payloads
-          )
-          @allow_payloads = app_data.fetch(
-            'allow_unencrypted_appfirewall_payloads',
-            @allow_payloads
-          )
-          @allow_payloads = app_data.fetch(
-            'allow_payloads',
-            @allow_payloads
-          )
-
-          data_exposure = app_data.fetch('data_exposure', {})
-          @max_data_ex_db_records_per_request = data_exposure.fetch('max_data_ex_db_records_per_request', @max_data_ex_db_records_per_request)
-
-          @enabled_instrumentations = app_data.fetch('enabled_instrumentations', @enabled_instrumentations)
-
-          @reverse_proxy = app_data.fetch('reverse_proxy', @reverse_proxy)
-          @reverse_proxy_ip_address_header = app_data.fetch('reverse_proxy_ip_address_header', @reverse_proxy_ip_address_header)
-
-          @host_identifier = app_data.fetch('host_identifier', @host_identifier)
-          @hmac_key = app_data.fetch('hmac_key', @hmac_key)
-
-          @password_hmac_key = app_data.fetch('password_hmac_key', @password_hmac_key)
-
-          @uuid = SecureRandom.uuid
-          @uuid = 'secure-random-failed' if @uuid.nil?
-
-          if app_data.key?('js_agent_api_base_url')
-            @js_agent_api_base_url = app_data['js_agent_api_base_url']
-          end
-          if app_data.key?('js_agent_url')
-            @js_agent_url = app_data['js_agent_url']
-          end
-
-          @demomode = app_data.fetch('demomode', @demomode)
-        else
-          puts ' ********* ********* ********* *********'
-          puts '* tCell.io                               *'
-          puts '* Unsupported config file version        *'
-          puts ' ********* ********* ********* *********'
+          app_data = config['applications'][0]
+          @disable_all = to_bool(app_data.fetch('disable_all', @disable_all))
         end
-      rescue StandardError => e
-        puts ' ********* ********* ********* *********'
-        puts '* tCell.io                               *'
-        puts '* Could not load config file             *'
-        puts ' ********* ********* ********* *********'
-        puts e
+      rescue StandardError # rubocop:disable Lint/HandleExceptions
       end
     end
 
-    # old value could be set via initializers, this makes sure those initializers still work
-    # properly
-    def allow_unencrypted_appfirewall_payloads=(val)
-      @allow_payloads = val
+    def to_bool(var)
+      { 'true' => true, 'false' => false }[var.to_s.downcase]
     end
 
-    # keep this around in case the value was read as well
-    def allow_unencrypted_appfirewall_payloads
-      @allow_payloads
-    end
+    def populate_configuration(native_agent_config_response)
+      # config
+      @enabled = native_agent_config_response['enabled']
+      @disabled_instrumentation = Set.new(native_agent_config_response['disabled_instrumentation'])
+      @fetch_policies_from_tcell = native_agent_config_response['update_policy']
+      @instrument = native_agent_config_response['instrument']
 
-    def log_filename
-      @agent_log_dir ||= File.join(@agent_home_dir, 'logs')
-      File.join(@agent_log_dir, @log_file_name)
-    end
+      # app config
+      apps = native_agent_config_response['applications'] ? native_agent_config_response['applications']['first'] : {}
+      @app_id = apps['app_id']
+      @api_key = apps['api_key']
+      @hmac_key = apps['hmac_key']
+      @password_hmac_key = apps['password_hmac_key']
+
+      # proxy config
+      proxy_config = apps['proxy_config'] || {}
+      @reverse_proxy = proxy_config['reverse_proxy']
+      @reverse_proxy_ip_address_header = proxy_config['reverse_proxy_ip_address_header']
+
+      # endpoint config
+      endpoint = native_agent_config_response['endpoint_config'] || {}
+      @tcell_api_url = endpoint['api_url']
 
-    def appfirewall_payloads_log_filename
-      @agent_log_dir ||= File.join(@agent_home_dir, 'logs')
-      File.join(@agent_log_dir, 'tcell_agent_payloads.log')
+      # ruby config
+      ruby_config = native_agent_config_response['ruby_config'] || {}
+      @enable_policy_polling = ruby_config['enable_policy_polling']
+      @enable_intercept_requests = !@disabled_instrumentation.include?('intercept_requests')
     end
   end