tcell_agent 1.1.12 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6334e154fc67ca343e771118111ed19b4961bcaa
-  data.tar.gz: a7acb414e6e0cb4506d6abafed4709818fcb720f
+SHA256:
+  metadata.gz: b0b98e0366f6de14a287a83e13f08a0b5fdc9f13d12d8c6d1f0a2fa7f1caad9c
+  data.tar.gz: 88cfc253d06f635d54ab72d83e4d3a28829526c3df47e2e1e7cad19fd47c9293
 SHA512:
-  metadata.gz: 21c687dafdccd3582369ea7aef360201d274f3cc2043479416a35f03af5efac90b649e7621569d83b0d90a6f72d88dab28be3c86c9c1cf27178c7a69ed99eb05
-  data.tar.gz: 4b176e6bf2625639647e3b82b8b52294e80bebafd73e2d6a6ba8844d4b41ba581a1882077397653e09944db4ebcccb76acbc6b2e74847d4931bd144e01b8358e
+  metadata.gz: 04c30f374634daf21590a07f94e3ea27cf09cd6931570b43a215f8d424983a86519c393faad76f7c809580dc9a02a905de00e6e28c4017d789b7baa6b86c5a3f
+  data.tar.gz: 39119777a798add18175e51c5935fb08d8ea85f2cde16d237e87f9175f477d9d3333cf91e4cecf089b39b355db31052a5047a444a420ea778e505631952be623

data/bin/tcell_agent

@@ -257,33 +257,45 @@ elsif command == 'test'
 
   printf '%-50s', 'Requiring configuration library... '
   require 'tcell_agent/configuration'
-  require 'tcell_agent/api'
   puts 'passed'
 
-  printf '%-50s', 'Make test API call for policies... '
-  api = TCellAgent::TCellApi.new
-  if api.poll_api
-    puts 'passed'
-  else
+  printf '%-50s', 'Loading native library... '
+  require 'tcell_agent/rust/native_library'
+  unless TCellAgent::Rust::NativeLibrary.common_lib_available?
     puts 'failed'
     Kernel.exit(1)
   end
+  puts 'passed'
 
-  printf '%-50s', 'Sending a Test event... '
-  send_succeeded = api.send_event_set([])
-  unless send_succeeded
+  printf '%-50s', 'Make test API call for policies... '
+  require 'tcell_agent/rust/native_agent'
+  errors = TCellAgent::Rust::NativeAgent.test_policies
+  if !errors.empty?
     puts 'failed'
+    puts errors
     Kernel.exit(1)
+  else
+    puts 'passed'
   end
-  puts 'passed'
 
-  printf '%-50s', 'Loading native library... '
-  require 'tcell_agent/rust/whisperer'
-  unless TCellAgent::Rust::Wrapper.common_lib_available?
+  printf '%-50s', 'Sending a Test event... '
+  require 'tcell_agent/logger'
+  require 'tcell_agent/sensor_events/server_agent'
+  errors = TCellAgent::Rust::NativeAgent.test_event_sender(
+    [
+      TCellAgent::SensorEvents::ServerAgentDetailsLanguageEvent.new(
+        'Ruby',
+        RUBY_VERSION
+      )
+    ]
+  )
+  if !errors.empty?
     puts 'failed'
+    puts errors
     Kernel.exit(1)
+  else
+    puts 'passed'
   end
-  puts 'passed'
 
   puts
   puts 'all tests passed, looks good.'

data/lib/tcell_agent.rb

@@ -1,20 +1,26 @@
 # See the file "LICENSE" for the full license governing this code.
 
-require 'tcell_agent/utils/passwords'
-require 'tcell_agent/utils/strings'
-require 'tcell_agent/utils/io'
 require 'tcell_agent/logger'
+require 'tcell_agent/utils/strings'
 require 'tcell_agent/configuration'
 
 require 'tcell_agent/agent'
 
-require 'tcell_agent/policies/http_tx_policy'
-require 'tcell_agent/policies/http_redirect_policy'
-require 'tcell_agent/policies/login_fraud_policy'
-require 'tcell_agent/policies/dataloss_policy'
-
-require 'tcell_agent/sensor_events/dlp'
 require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 
 require 'tcell_agent/instrumentation'
-require 'tcell_agent/start_background_thread'
+
+require 'tcell_agent/instrument_servers'
+
+if !TCellAgent.configuration.disable_all && TCellAgent.configuration.should_instrument?
+  require 'tcell_agent/instrumentation/cmdi'
+  require 'tcell_agent/instrumentation/lfi'
+  require 'tcell_agent/instrumentation/monkey_patches/io'
+  require 'tcell_agent/instrumentation/monkey_patches/file'
+  require 'tcell_agent/instrumentation/monkey_patches/kernel'
+
+  require 'tcell_agent/hooks/login_fraud'
+  require 'tcell_agent/rails/on_start' if defined?(Rails)
+  # sinatra used to be supported, but dropped support due to no customers using it
+  # require 'tcell_agent/sinatra' if defined?(Sinatra)
+end

data/lib/tcell_agent/agent.rb

@@ -2,120 +2,44 @@
 
 require 'tcell_agent/logger'
 require 'tcell_agent/version'
-require 'tcell_agent/api'
 require 'tcell_agent/configuration'
 
 require 'tcell_agent/sensor_events/server_agent'
-require 'tcell_agent/utils/queue_with_timeout'
 
-require 'tcell_agent/agent/event_processor'
-require 'tcell_agent/agent/policy_manager'
+require 'tcell_agent/policies/policy_types'
+require 'tcell_agent/policies/policies_manager'
+require 'tcell_agent/policies/policy_polling'
 require 'tcell_agent/agent/static_agent'
-require 'tcell_agent/agent/policy_types'
 require 'tcell_agent/agent/route_manager'
-require 'tcell_agent/agent/fork_pipe_manager'
 
 require 'tcell_agent/routes/table'
 
-require 'net/http'
-require 'thread'
-require 'logger'
+require 'tcell_agent/settings_reporter'
+require 'tcell_agent/rust/native_agent'
+
 require 'json'
-require 'monitor'
 
 module TCellAgent
   class Agent
-    attr_accessor :start_pid
-    attr_accessor :event_queue
-
-    attr_accessor :fork_event_queue
-    attr_accessor :fork_event_thread
-    attr_accessor :fork_event_thread_mutex
-
-    attr_accessor :metrics_event_queue
-    attr_accessor :metrics_event_thread
-    attr_accessor :metrics_event_thread_mutex
-
-    attr_accessor :policies
-    attr_accessor :eventProcessorThread
-    attr_accessor :response_time_table
-    attr_accessor :route_table
-
-    attr_accessor :event_processor_thread
-    attr_accessor :event_processor
-    attr_accessor :worker_mutex
-
-    attr_accessor :policy_polling_thread
-    attr_accessor :policy_polling_worker_mutex
-
-    attr_accessor :event_queue_monitor
-    attr_accessor :event_dispatch_monitor
-
-    attr_accessor :stop_agent
-    attr_accessor :complete_policy_cache
-
-    def initialize(start_pid = Process.pid)
-      @start_pid = start_pid
-      @dispatch_events_timeout = TCellAgent.configuration.event_time_limit_seconds || 55
-      @dispatch_events_limit = TCellAgent.configuration.event_batch_size_limit || 20
-      @worker_mutex = Mutex.new
-      @policy_polling_worker_mutex = Mutex.new
-      @@policy_tapi = TCellApi.new
-
-      # Agent request thread
-      @policies = {
-        TCellAgent::PolicyTypes::RUST => TCellAgent::Policies::RustPolicies.new
-      }
-      @lock = Monitor.new
-
-      initialize_processor_variables
-
-      if TCellAgent.configuration.preload_policy_filename
-        TCellAgent.logger.info('Preloading a policy file')
-        begin
-          policy_file = File.open(TCellAgent.configuration.preload_policy_filename).read
-          policy_jsons = JSON.parse(policy_file)
-          policy_jsons = policy_jsons['result'] if policy_jsons.key?('result')
-          process_policy_json(policy_jsons, false)
-        rescue StandardError => e
-          TCellAgent.logger.error(e.message)
-        end
-      end
-      cached_policies = policies_from_cachefile
-      process_policy_json(cached_policies, false) if cached_policies
-    end
+    include TCellAgent::ModuleLoggerAccess
 
-    def initialize_processor_variables
-      @complete_policy_cache = {}
+    attr_accessor :route_table,
+                  :stop_agent,
+                  :safe_to_check_cmdi
 
+    def initialize
       @stop_agent = false
+      @native_agent = nil
       @route_table = TCellAgent::Routes::RouteTable.new
-
-      @event_queue_monitor = Monitor.new
-      @event_dispatch_monitor = Monitor.new
-      @mutex = Monitor.new
-
-      @response_time_table = {}
-      @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
-      @sessions_metrics_mutex = Monitor.new
-
-      @dispatch_events = []
-      @event_queue = BoundedQueue.new(200)
-
-      @fork_event_queue = Queue.new
-      @fork_event_thread_mutex = Monitor.new
-
-      @metrics_event_queue = Queue.new
-      @metrics_event_thread_mutex = Monitor.new
+      @safe_to_check_cmdi = false
+      @policies_manager = PoliciesManager.new(nil)
     end
 
-    def parent_process?
-      @start_pid == Process.pid
-    end
-
-    def start
+    def validate_config
       if TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.api_key) ||
-         TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.app_id)
+         TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.app_id) ||
+         TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.tcell_input_url) ||
+         TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.tcell_api_url)
         puts ' ********* ********* ********* *********'
         puts '* tCell.io                               *'
         puts '* Configuration info is missing, you may  *'
@@ -123,13 +47,70 @@ module TCellAgent
         puts '* it in the config/ directory            *'
         puts ' ********* ********* ********* *********'
         TCellAgent.configuration.enabled = false
+      end
+    end
+
+    def start(server_name)
+      TCellAgent.thread_agent.validate_config
+      return unless TCellAgent.configuration.should_instrument?
+
+      @native_agent = TCellAgent::Rust::NativeAgent.create_agent(
+        TCellAgent.configuration
+      )
+      if @native_agent.nil?
+        TCellAgent.configuration.enabled = false
         return
       end
 
-      TCellAgent.logger.debug('Starting thread agent')
+      TCellAgent.native_agent = @native_agent
+      @policies_manager = PoliciesManager.new(@native_agent)
+      # if preload_policy_filename is used and policy polling is
+      # disabled, need to call poll policies to make sure
+      # ruby policies are in sync with native agent enablements
+      result = @native_agent.poll_new_policies
+      policies_and_enablements = result['new_policies_and_enablements'] || {}
+      @policies_manager.process_policy_json(
+        policies_and_enablements['enablements'],
+        policies_and_enablements['policies']
+      )
+
+      @policy_polling = PolicyPolling.new(@policies_manager, @native_agent)
+
+      module_logger.info("Starting thread agent: #{server_name}")
+
+      @safe_to_check_cmdi = true
+
+      TCellAgent.report_settings
+      TCellAgent::Instrumentation::Rails.send_framework_info
+      TCellAgent::Instrumentation::Rails.send_settings
+    rescue StandardError => standard_error
+      TCellAgent.configuration.enabled = false
+      module_logger.error("Error starting agent: (#{standard_error.class}) #{standard_error.message}")
+      module_logger.exception(standard_error)
+    end
+
+    def policies
+      @policies_manager.policies
+    end
+
+    def report_metrics(request_time, tcell_context)
+      @native_agent.report_metrics(
+        request_time, tcell_context
+      )
+    rescue StandardError => standard_error
+      module_logger.error("Error reporting metric: (#{standard_error.class}) #{standard_error.message}")
+      module_logger.exception(standard_error)
+    end
+
+    def queue_sensor_event(event)
+      return unless @native_agent
 
-      ensure_policy_polling_running
-      ensure_event_processor_running
+      @native_agent.send_sanitized_events(
+        [event]
+      )
+    rescue StandardError => standard_error
+      module_logger.error("Error sending event: (#{standard_error.class}) #{standard_error.message}")
+      module_logger.exception(standard_error)
     end
   end
 end

data/lib/tcell_agent/agent/route_manager.rb

@@ -1,13 +1,7 @@
 # See the file "LICENSE" for the full license governing this code.
 
-require 'tcell_agent/logger'
-require 'tcell_agent/version'
-require 'tcell_agent/api'
-require 'tcell_agent/configuration'
-
 require 'tcell_agent/routes/table'
 require 'tcell_agent/sensor_events/discovery'
-require 'tcell_agent'
 
 module TCellAgent
   class Agent
@@ -23,16 +17,6 @@ module TCellAgent
     def discover_database_fields(route_id, database, schema, table, fields)
       return if route_id.nil? || database.nil? || schema.nil? || table.nil? || fields.nil?
 
-      if TCellAgent::Agent.parent_process? == false
-        TCellAgent.queue_metric('_type' => 'discover_database_fields',
-                                'route_id' => route_id,
-                                'database' => database,
-                                'schema' => schema,
-                                'table' => table,
-                                'fields' => fields)
-        return
-      end
-
       query_hash = TCellAgent::Agent.get_database_discovery_identifier(database, schema, table, fields)
 
       return if @route_table.routes[route_id].database_queries_discovered.fetch(query_hash, false)

data/lib/tcell_agent/agent/static_agent.rb

@@ -1,24 +1,15 @@
-# See the file "LICENSE" for the full license governing this code.
-require 'tcell_agent/sensor_events/metrics'
-require 'monitor'
-
 module TCellAgent
   @@instance_lock = Mutex.new
   @@my_thread_agent = nil
 
   def self.thread_agent
-    if thread_agent_defined? == false
+    unless @@my_thread_agent
       @@instance_lock.synchronize do
-        if thread_agent_defined? == false
-          @@my_thread_agent = TCellAgent::Agent.new(Process.pid)
-        end
+        @@my_thread_agent ||= TCellAgent::Agent.new
       end
     end
-    @@my_thread_agent
-  end
 
-  def self.thread_agent_defined?
-    @@my_thread_agent != nil
+    @@my_thread_agent
   end
 
   def self.thread_agent=(some_agent)
@@ -31,35 +22,23 @@ module TCellAgent
     thread_agent.queue_sensor_event(event)
   end
 
-  def self.queue_metric(event)
-    thread_agent._queue_metric(event)
+  def self.report_metrics(response_time, tcell_context)
+    thread_agent.report_metrics(response_time, tcell_context)
   end
 
   def self.policy(policy_type)
     thread_agent.policies.fetch(policy_type, nil)
   end
 
-  def self.increment_session_info(hmac_session_id, user_id, ip_address, user_agent)
-    thread_agent.increment_session_info(hmac_session_id, user_id, ip_address, user_agent)
-  end
-
-  def self.increment_route(route_id, response_time)
-    thread_agent.increment_route(route_id, response_time)
-  end
-
   def self.discover_database_fields(route_id, database, schema, table, fields)
     thread_agent.discover_database_fields(route_id, database, schema, table, fields)
   end
 
-  def self.stop_agent
-    thread_agent.stop_agent = true
+  def self.safe_to_check_cmdi?
+    thread_agent && thread_agent.safe_to_check_cmdi
   end
 
-  def self.ensure_event_processor_running
-    thread_agent.ensure_event_processor_running
-  end
-
-  def self.safe_to_send_cmdi_events?
-    thread_agent.safe_to_send_cmdi_events?
+  def self.stop_agent
+    thread_agent.stop_agent = true
   end
 end

data/lib/tcell_agent/authlogic.rb

@@ -1,11 +1,8 @@
-# See the file "LICENSE" for the full license governing this code.
+if TCellAgent.configuration.should_instrument_authlogic? && defined?(Authlogic)
 
-require 'tcell_agent/userinfo'
-require 'tcell_agent/logger'
-require 'tcell_agent/sensor_events/honeytokens'
+  require 'tcell_agent/userinfo'
 
-module TCellAgent
-  if defined?(Authlogic)
+  module TCellAgent
     TCellAgent::UserInformation.class_eval do
       class << self
         alias_method :original_get_user_from_request, :get_user_from_request

data/lib/tcell_agent/config/unknown_options.rb

@@ -20,11 +20,10 @@ module TCellAgent
             'TCELL_AGENT_HOME',
             'TCELL_AGENT_LOG_DIR',
             'TCELL_AGENT_CONFIG',
-            'TCELL_AGENT_ALLOW_UNENCRYPTED_APPSENSOR_PAYLOADS',
-            'TCELL_AGENT_ALLOW_UNENCRYPTED_APPFIREWALL_PAYLOADS',
             'TCELL_AGENT_ALLOW_PAYLOADS',
-            'TCELL_AGENT_HOME_OWNER',
-            'TCELL_AGENT_ENABLED'
+            'TCELL_AGENT_LOG_LEVEL',
+            'TCELL_AGENT_LOG_FILENAME',
+            'TCELL_AGENT_LOG_ENABLED'
           ]
         )
 
@@ -68,8 +67,6 @@ module TCellAgent
                   js_agent_url
                   max_csp_header_bytes
                   event_batch_size_limit
-                  allow_unencrypted_appsensor_payloads
-                  allow_unencrypted_appfirewall_payloads
                   allow_payloads
                   reverse_proxy
                   reverse_proxy_ip_address_header
@@ -79,13 +76,12 @@ module TCellAgent
                   disable_all
                   enabled
                   enable_event_manager
-                  enable_event_consumer
                   enable_policy_polling
                   enable_instrumentation
                   enable_intercept_requests
                   instrument_for_events
-                  agent_home_owner
                   enabled_instrumentations
+                  stdout_logger
                 ]
 
                 key_differences += (application.keys - second_level_keys)