tcell_agent 1.1.12 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6334e154fc67ca343e771118111ed19b4961bcaa
-  data.tar.gz: a7acb414e6e0cb4506d6abafed4709818fcb720f
+SHA256:
+  metadata.gz: 4874ce28005d00849139d7ff1620e5eb0969a6a4626a2b005936ebb429be7064
+  data.tar.gz: bb009ff4896aecb87e92ea17c57273a4b9bee02f0ef211a51f45f72ed96dbc04
 SHA512:
-  metadata.gz: 21c687dafdccd3582369ea7aef360201d274f3cc2043479416a35f03af5efac90b649e7621569d83b0d90a6f72d88dab28be3c86c9c1cf27178c7a69ed99eb05
-  data.tar.gz: 4b176e6bf2625639647e3b82b8b52294e80bebafd73e2d6a6ba8844d4b41ba581a1882077397653e09944db4ebcccb76acbc6b2e74847d4931bd144e01b8358e
+  metadata.gz: 960aa5847e382e2ef19f3d5ef615113a3c042a1f567bdc0cbf9231d5662b7bc140275177103de97b7967408329f1ac90d7702e9606adb8c19117e4dee691eb4c
+  data.tar.gz: 8ff116f37231cd8d045ecd634b01446250293520318a0aedbe10c79adcb49ae318e966e67c3576df9fa364152c06b71546a6caf35583d82e9f7e0856908a6472

data/bin/tcell_agent

@@ -1,25 +1,11 @@
 #!/usr/bin/env ruby
 
-# TODO: so a small bit becames something, larger, rewrite as a real cmdline script
-
-require 'fileutils'
-require 'json'
+require 'tcell_agent'
 require 'optparse'
 
+CONFIG_DIR = 'config'.freeze
+CONFIG_FILE = 'config/tcell_agent.config'.freeze
 options = {}
-
-subtext = <<HELP
-Commonly used command are:
-   setup :     Setup new config file
-    test :     Run diagnostics classes and config
-loglevel :     Set the loglevel of the tcell agent
-  enable :     Enable the agent
- disable :     Disable the agent
-
-See 'tcell_agent COMMAND --help' for more information on a specific command.
-
-HELP
-
 def yesno(default = true)
   begin
     system('stty raw -echo')
@@ -34,74 +20,59 @@ def yesno(default = true)
   default
 end
 
-CONFIG_DIR = 'config'.freeze
-CONFIG_FILE = 'config/tcell_agent.config'.freeze
+subtext = <<HELP
+Commonly used commands are:
+    test    : Run diagnostics and test event sending
+HELP
 
 global = OptionParser.new do |opts|
-  opts.banner = 'Usage: tcell_agent [options] [subcommand [options]]'
+  opts.banner = 'Usage: tcell_agent [options] [subcommand]'
+
   opts.on('--version', 'Print version') do |_v|
-    require 'tcell_agent/version'
-    puts "TCell.io Ruby Agent (Version #{TCellAgent::VERSION})"
+    puts "tCell Ruby Agent Version #{TCellAgent::VERSION}"
     Kernel.exit(1)
   end
-  opts.on('-v', '--[no-]verbose', 'Run verbosely') do |v|
-    options[:verbose] = v
+
+  opts.on('test', 'Run diagnostics and test event sending') do |_v|
   end
   opts.separator ''
   opts.separator subtext
 end
 
 subcommands = {
-  'setup' => OptionParser.new do |opts|
-    opts.banner = 'Usage: setup'
+  'test' => OptionParser.new do |opts|
+    opts.banner = 'Usage: test'
   end,
   'loglevel' => OptionParser.new do |opts|
     opts.banner = 'Usage: loglevel [options] error|warn|info|debug'
     opts.on('-o', '--off', 'turn logging off ') do |v|
       options[:off] = v
     end
-  end,
-  'preload' => OptionParser.new do |opts|
-    opts.banner = 'Usage: loglevel [options] [preload_filename]'
-    opts.on('-o', '--off', 'turn preloading filename off ') do |v|
-      options[:off] = v
-    end
-  end,
-  'demomode' => OptionParser.new do |opts|
-    opts.banner = 'Usage: loglevel [options]'
-    opts.on('-o', '--off', 'turn preloading filename off ') do |v|
-      options[:off] = v
-    end
-  end,
-  'enable' => OptionParser.new do |opts|
-    opts.banner = 'Usage: enable'
-  end,
-  'disable' => OptionParser.new do |opts|
-    opts.banner = 'Usage: disable'
-  end,
-  'test' => OptionParser.new do |opts|
-              opts.banner = 'Usage: test'
-              # opts.on("-q", "--[no-]quiet", "quietly run ") do |v|
-              #  options[:quiet] = v
-              # end
-            end
+  end
 }
 
 global.order!
 command = ARGV.shift
-if command.nil? || subcommands[command].nil?
+deprecated_commands = %w[setup loglevel preload demomode enable disable]
+if command.nil?
+  puts global
+  Kernel.exit(1)
+elsif subcommands[command]
+  subcommands[command].order!
+elsif !deprecated_commands.include?(command)
   puts global
   Kernel.exit(1)
 end
-subcommands[command].order!
 
 if command == 'setup'
+  puts '[WARN] tcell_agent setup is deprecated and will be removed in the future.'
+  puts '       Please download the config file from the tCell dashboard'
   unless File.directory?(CONFIG_DIR)
     print "Directory 'config' not found, create? [Y/n]"
     answer = yesno
     print "\n"
     unless answer
-      puts 'ERROR: Could not create config'
+      puts '[ERROR] Could not create config'
       Kernel.exit(1)
     end
     FileUtils.mkdir_p CONFIG_DIR
@@ -132,6 +103,7 @@ if command == 'setup'
   puts 'done.'
 
 elsif command == 'loglevel'
+  puts '[WARN] tcell_agent loglevel is deprecated and will be removed in the future.'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
   logging_options = config_hash['applications'][0].fetch('logging_options', {})
@@ -158,6 +130,7 @@ elsif command == 'loglevel'
   puts 'done.'
 
 elsif command == 'preload'
+  puts '[WARN] tcell_agent preload is deprecated and will be removed in the future.'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
 
@@ -175,6 +148,7 @@ elsif command == 'preload'
   puts 'done.'
 
 elsif command == 'enable'
+  puts '[WARN] tcell_agent enable is deprecated and will be removed in the future.'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
   config_hash['applications'][0].delete('enabled')
@@ -182,6 +156,7 @@ elsif command == 'enable'
   puts 'Enabled, you will need to restart the server.'
 
 elsif command == 'disable'
+  puts '[WARN] tcell_agent disable is deprecated and will be removed in the future.'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
   config_hash['applications'][0]['enabled'] = false
@@ -189,6 +164,7 @@ elsif command == 'disable'
   puts 'Disabled, you will need to restart the server.'
 
 elsif command == 'demomode'
+  puts '[WARN] tcell_agent demomode is deprecated and will be removed in the future.'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
   if options[:off] == true
@@ -200,92 +176,24 @@ elsif command == 'demomode'
   puts 'done.'
 
 elsif command == 'test'
-  puts
-  printf '%-50s', 'Config file exists... '
-  unless File.exist?(CONFIG_FILE)
-    puts 'failed'
-    Kernel.exit(1)
-  end
-  puts 'passed'
-
-  printf '%-50s', 'Config valid json... '
-  file = File.read(CONFIG_FILE)
-  config_hash = JSON.parse(file)
-  puts 'passed'
-
-  printf '%-50s', 'Config file has valid version... '
-  if config_hash.fetch('version') != 1
-    puts 'failed'
-    Kernel.exit(1)
-  end
-  puts 'passed'
-
-  printf '%-50s', 'Config file has application...'
-  if config_hash.fetch('applications').empty?
-    puts 'failed'
-    Kernel.exit(1)
-  end
-  puts 'passed'
-
-  printf '%-50s', 'Application has app_id... '
-  tcell_application = config_hash.fetch('applications')[0]
-  if !tcell_application.key?('app_id') && !ENV['TCELL_AGENT_APP_ID']
-    puts 'failed'
-    Kernel.exit(1)
-  end
-  puts 'passed'
-
-  printf '%-50s', 'Application has api_key... '
-  tcell_application = config_hash.fetch('applications')[0]
-  if !tcell_application.key?('api_key') && !ENV['TCELL_AGENT_API_KEY']
-    puts 'failed'
-    Kernel.exit(1)
-  end
-  puts 'passed'
-
-  printf '%-50s', 'Check for unknown settings... '
-  require 'tcell_agent/config/unknown_options'
-  messages = TCellAgent::Config::Validate.get_unknown_options(config_hash)
-  unless messages.empty?
-    puts 'failed'
-    messages.each do |message|
-      puts message
+  Dir[Dir.pwd + '/config/initializers/*.rb'].sort.each do |f|
+    begin
+      require f
+    rescue NameError # rubocop: disable Lint/HandleExceptions
+    rescue StandardError => e
+      puts "[ERROR] Loading initializers failed. #{e}"
     end
-    Kernel.exit(1)
-  end
-  puts 'passed'
-
-  printf '%-50s', 'Requiring configuration library... '
-  require 'tcell_agent/configuration'
-  require 'tcell_agent/api'
-  puts 'passed'
-
-  printf '%-50s', 'Make test API call for policies... '
-  api = TCellAgent::TCellApi.new
-  if api.poll_api
-    puts 'passed'
-  else
-    puts 'failed'
-    Kernel.exit(1)
   end
 
-  printf '%-50s', 'Sending a Test event... '
-  send_succeeded = api.send_event_set([])
-  unless send_succeeded
-    puts 'failed'
-    Kernel.exit(1)
+  require 'tcell_agent/rust/native_library'
+  unless TCellAgent::Rust::NativeLibrary.common_lib_available?
+    puts '[ERROR] Native Library is unavailable'
+    return
   end
-  puts 'passed'
 
-  printf '%-50s', 'Loading native library... '
-  require 'tcell_agent/rust/whisperer'
-  unless TCellAgent::Rust::Wrapper.common_lib_available?
-    puts 'failed'
-    Kernel.exit(1)
-  end
-  puts 'passed'
-
-  puts
-  puts 'all tests passed, looks good.'
-  puts 'done.'
+  puts "tCell Ruby Agent Version #{TCellAgent::VERSION}"
+  TCellAgent::Rust::NativeAgent.test_agent(
+    TCellAgent.initializer_configuration ||
+    TCellAgent.configuration
+  )
 end

data/lib/tcell_agent.rb

@@ -1,20 +1,18 @@
 # See the file "LICENSE" for the full license governing this code.
-
-require 'tcell_agent/utils/passwords'
-require 'tcell_agent/utils/strings'
-require 'tcell_agent/utils/io'
-require 'tcell_agent/logger'
 require 'tcell_agent/configuration'
 
-require 'tcell_agent/agent'
+unless TCellAgent.configuration.disable_all
+  require 'tcell_agent/logger'
+  require 'tcell_agent/utils/strings'
+  require 'tcell_agent/agent'
 
-require 'tcell_agent/policies/http_tx_policy'
-require 'tcell_agent/policies/http_redirect_policy'
-require 'tcell_agent/policies/login_fraud_policy'
-require 'tcell_agent/policies/dataloss_policy'
+  require 'tcell_agent/sensor_events/util/sanitizer_utilities'
 
-require 'tcell_agent/sensor_events/dlp'
-require 'tcell_agent/sensor_events/util/sanitizer_utilities'
+  require 'tcell_agent/instrumentation'
 
-require 'tcell_agent/instrumentation'
-require 'tcell_agent/start_background_thread'
+  require 'tcell_agent/instrument_servers'
+  require 'tcell_agent/hooks/login_fraud'
+  require 'tcell_agent/rails/railties/tcell_agent_railties' if defined?(Rails)
+  # sinatra used to be supported, but dropped support due to no customers using it
+  # require 'tcell_agent/sinatra' if defined?(Sinatra)
+end

data/lib/tcell_agent/agent.rb

@@ -2,134 +2,145 @@
 
 require 'tcell_agent/logger'
 require 'tcell_agent/version'
-require 'tcell_agent/api'
 require 'tcell_agent/configuration'
 
 require 'tcell_agent/sensor_events/server_agent'
-require 'tcell_agent/utils/queue_with_timeout'
 
-require 'tcell_agent/agent/event_processor'
-require 'tcell_agent/agent/policy_manager'
+require 'tcell_agent/policies/policy_types'
+require 'tcell_agent/policies/policies_manager'
+require 'tcell_agent/policies/policy_polling'
 require 'tcell_agent/agent/static_agent'
-require 'tcell_agent/agent/policy_types'
 require 'tcell_agent/agent/route_manager'
-require 'tcell_agent/agent/fork_pipe_manager'
 
 require 'tcell_agent/routes/table'
 
-require 'net/http'
-require 'thread'
-require 'logger'
+require 'tcell_agent/settings_reporter'
+require 'tcell_agent/rust/native_agent'
+
 require 'json'
-require 'monitor'
 
 module TCellAgent
   class Agent
-    attr_accessor :start_pid
-    attr_accessor :event_queue
-
-    attr_accessor :fork_event_queue
-    attr_accessor :fork_event_thread
-    attr_accessor :fork_event_thread_mutex
-
-    attr_accessor :metrics_event_queue
-    attr_accessor :metrics_event_thread
-    attr_accessor :metrics_event_thread_mutex
-
-    attr_accessor :policies
-    attr_accessor :eventProcessorThread
-    attr_accessor :response_time_table
-    attr_accessor :route_table
-
-    attr_accessor :event_processor_thread
-    attr_accessor :event_processor
-    attr_accessor :worker_mutex
-
-    attr_accessor :policy_polling_thread
-    attr_accessor :policy_polling_worker_mutex
-
-    attr_accessor :event_queue_monitor
-    attr_accessor :event_dispatch_monitor
-
-    attr_accessor :stop_agent
-    attr_accessor :complete_policy_cache
-
-    def initialize(start_pid = Process.pid)
-      @start_pid = start_pid
-      @dispatch_events_timeout = TCellAgent.configuration.event_time_limit_seconds || 55
-      @dispatch_events_limit = TCellAgent.configuration.event_batch_size_limit || 20
-      @worker_mutex = Mutex.new
-      @policy_polling_worker_mutex = Mutex.new
-      @@policy_tapi = TCellApi.new
-
-      # Agent request thread
-      @policies = {
-        TCellAgent::PolicyTypes::RUST => TCellAgent::Policies::RustPolicies.new
-      }
-      @lock = Monitor.new
-
-      initialize_processor_variables
-
-      if TCellAgent.configuration.preload_policy_filename
-        TCellAgent.logger.info('Preloading a policy file')
-        begin
-          policy_file = File.open(TCellAgent.configuration.preload_policy_filename).read
-          policy_jsons = JSON.parse(policy_file)
-          policy_jsons = policy_jsons['result'] if policy_jsons.key?('result')
-          process_policy_json(policy_jsons, false)
-        rescue StandardError => e
-          TCellAgent.logger.error(e.message)
-        end
-      end
-      cached_policies = policies_from_cachefile
-      process_policy_json(cached_policies, false) if cached_policies
-    end
+    include TCellAgent::ModuleLoggerAccess
 
-    def initialize_processor_variables
-      @complete_policy_cache = {}
+    attr_accessor :route_table,
+                  :stop_agent
 
+    def initialize
       @stop_agent = false
+      @native_agent = nil
       @route_table = TCellAgent::Routes::RouteTable.new
+      @policies_manager = PoliciesManager.new(nil)
+    end
+
+    def instrument_auth_frameworks
+      if defined?(Devise) && TCellAgent.configuration.should_instrument?('devise')
+        module_logger.info('Instrumenting Devise authentication framework')
+        require 'tcell_agent/rails/auth/devise'
+        require 'tcell_agent/rails/auth/devise_helper'
+      end
+
+      if defined?(Authlogic) && TCellAgent.configuration.should_instrument?('authlogic')
+        module_logger.info('Instrumenting Authlogic authentication framework')
+        require 'tcell_agent/rails/auth/authlogic'
+        require 'tcell_agent/rails/auth/authlogic_helper'
+      end
+
+      if defined?(Doorkeeper) && TCellAgent.configuration.should_instrument?('doorkeeper') # rubocop:disable Style/GuardClause
+        module_logger.info('Instrumenting Doorkeeper authentication framework')
+        require 'tcell_agent/rails/auth/doorkeeper'
+      end
+    end
+
+    def instrument_built_ins
+      require 'tcell_agent/instrumentation/cmdi'
+      require 'tcell_agent/instrumentation/lfi'
+
+      if TCellAgent.configuration.should_instrument?('io')
+        module_logger.info('Instrumenting Ruby Class: IO')
+        require 'tcell_agent/instrumentation/monkey_patches/io'
+      end
 
-      @event_queue_monitor = Monitor.new
-      @event_dispatch_monitor = Monitor.new
-      @mutex = Monitor.new
+      if TCellAgent.configuration.should_instrument?('file')
+        module_logger.info('Instrumenting Ruby Class: File')
+        require 'tcell_agent/instrumentation/monkey_patches/file'
+      end
+
+      if TCellAgent.configuration.should_instrument?('kernel') # rubocop:disable Style/GuardClause
+        module_logger.info('Instrumenting Ruby Module: Kernel')
+        require 'tcell_agent/instrumentation/monkey_patches/kernel'
+      end
+    end
+
+    def manage_policies
+      @policies_manager = PoliciesManager.new(@native_agent)
+
+      result = {}
+      unless TCellAgent.configuration.should_start_policy_poll?
+        result = @native_agent.poll_new_policies
+      end
 
-      @response_time_table = {}
-      @sessions_metrics = TCellAgent::SensorEvents::SessionsMetric.new
-      @sessions_metrics_mutex = Monitor.new
+      policies_and_enablements = result['new_policies_and_enablements'] || {}
 
-      @dispatch_events = []
-      @event_queue = BoundedQueue.new(200)
+      @policies_manager.process_policy_json(
+        policies_and_enablements['enablements'],
+        policies_and_enablements['policies']
+      )
 
-      @fork_event_queue = Queue.new
-      @fork_event_thread_mutex = Monitor.new
+      @policy_polling = PolicyPolling.new(@policies_manager, @native_agent)
+    end
+
+    def policies
+      @policies_manager.policies
+    end
+
+    def queue_sensor_event(event)
+      return unless @native_agent
 
-      @metrics_event_queue = Queue.new
-      @metrics_event_thread_mutex = Monitor.new
+      @native_agent.send_sanitized_events(
+        [event]
+      )
+    rescue StandardError => standard_error
+      module_logger.error("Error sending event: (#{standard_error.class}) #{standard_error.message}")
+      module_logger.exception(standard_error)
     end
 
-    def parent_process?
-      @start_pid == Process.pid
+    def report_metrics(request_time, tcell_context)
+      @native_agent.report_metrics(
+        request_time, tcell_context
+      )
+    rescue StandardError => standard_error
+      module_logger.error("Error reporting metric: (#{standard_error.class}) #{standard_error.message}")
+      module_logger.exception(standard_error)
     end
 
-    def start
-      if TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.api_key) ||
-         TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.app_id)
-        puts ' ********* ********* ********* *********'
-        puts '* tCell.io                               *'
-        puts '* Configuration info is missing, you may  *'
-        puts '* need to download config file and place *'
-        puts '* it in the config/ directory            *'
-        puts ' ********* ********* ********* *********'
+    def start(server_name)
+      @native_agent = TCellAgent::Rust::NativeAgent.create_agent(
+        TCellAgent.initializer_configuration ||
+        TCellAgent.configuration
+      )
+
+      if @native_agent.nil?
         TCellAgent.configuration.enabled = false
         return
       end
 
-      TCellAgent.logger.debug('Starting thread agent')
+      TCellAgent.native_logger = @native_agent
+
+      module_logger.info('Rails initializer overriding default agent configuration') unless TCellAgent.initializer_configuration.nil?
+
+      instrument_auth_frameworks
+      instrument_built_ins
+      manage_policies
 
-      ensure_policy_polling_running
-      ensure_event_processor_running
+      module_logger.info("Started thread agent: #{server_name}")
+      TCellAgent.report_settings
+      TCellAgent::Instrumentation::Rails.send_framework_info
+      TCellAgent::Instrumentation::Rails.send_settings
+    rescue StandardError => standard_error
+      TCellAgent.configuration.enabled = false
+      module_logger.error("Error starting agent: (#{standard_error.class}) #{standard_error.message}")
+      module_logger.exception(standard_error)
     end
   end
 end