tcell_agent 0.4.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 521d6057316a3d03d8f1b69dab4419c22bd52f08
-  data.tar.gz: 8296679382ce95cbc764e8cddcc2f5a42b8ce87b
+  metadata.gz: e3bfdacf0f493764e58e984f770dc53d04787a98
+  data.tar.gz: 27c3464c78adda7a3224e5ea080e2615ff857975
 SHA512:
-  metadata.gz: b8c8463350c03e01fe5d0a3da334500df9c81047bf8b5ce20378c4840ee9659523aa1ee44233aa18f27ac8f51977d5f3a36c5cc4f566e5f641ab745ace18a5f1
-  data.tar.gz: 44fd7f23e32263bab9643e33a3befab20e1f3848258c80055e663911f4f1607a53672426fcfff066094bcc13e759bd6df8eaad41f2b777d54acf9b83f374df51
+  metadata.gz: ef098d1aeb0feb1398c0ce002d706d0cef7b1a82ce1f9dd600eda39230e2bd718e60800ee3c9d7bb23ed345e8728f8fefb4b6bd5bda59a379868aea5f63ae0c7
+  data.tar.gz: d320becd35d19b4e124dfa04d703264b538191ccaf27c699fbdfde939769c4b8e97ff9ca6ac0cca16a47fe4d71ccdba83feb28c29eae7ff4a218aadfd29a7994

data/Rakefile

@@ -1,31 +1,18 @@
-require 'rake/clean'
-require 'rake/extensiontask'
 require 'rspec/core/rake_task'
 
-CLEAN.include('ext/**/*{.o,.log,.so,.bundle}')
-CLEAN.include('ext/**/Makefile')
-CLOBBER.include('lib/*{.so,.bundle}')
-
-Rake::ExtensionTask.new("libinjection") do |ext|
-  ext.lib_dir = File.join("lib", "libinjection")
-end
-
-task :default => [:compile, :spec]
-
 RSpec::Core::RakeTask.new(:spec)
 
-desc "Run tests"
+desc 'Run tests'
+task :default => [:spec]
 task :test => :spec
 
-Rake::Task[:test].prerequisites << :compile
-
-task "init-integration-tests" do
-system("docker-compose run railsintegration224 bundle install")
-system("docker-compose run railsintegration224 bundle exec rake db:create db:setup")
-system("docker-compose stop")
+task 'init-integration-tests' do
+  system('docker-compose run railsintegration224 bundle install')
+  system('docker-compose run railsintegration224 bundle exec rake db:create db:setup')
+  system('docker-compose stop')
 end
 
-task "integration-test" do
-  system("docker-compose up railsintegration224")
-  system("docker-compose stop")
+task 'integration-test' do
+  system('docker-compose up railsintegration224')
+  system('docker-compose stop')
 end

data/bin/tcell_agent

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-#todo: so a small bit becames something, larger, rewrite as a real cmdline script
+# TODO: so a small bit becames something, larger, rewrite as a real cmdline script
 
 require 'fileutils'
 require 'json'
@@ -8,7 +8,6 @@ require 'optparse'
 
 options = {}
 
-
 subtext = <<HELP
 Commonly used command are:
    setup :     Setup new config file
@@ -21,264 +20,260 @@ See 'tcell_agent COMMAND --help' for more information on a specific command.
 
 HELP
 
-def yesno(default=true)
+def yesno(default = true)
   begin
-    system("stty raw -echo")
+    system('stty raw -echo')
     str = STDIN.getc
   ensure
-    system("stty -raw echo")
-  end
-  if str == "Y" || str == "y"
-    return true
-  elsif str == "N" || str == "n"
-    return false
-  else
-    return default
+    system('stty -raw echo')
   end
+
+  return true if %w[Y y].include?(str)
+  return false if %w[N n].include?(str)
+
+  default
 end
 
-CONFIG_DIR = 'config'
-CONFIG_FILE = 'config/tcell_agent.config'
+CONFIG_DIR = 'config'.freeze
+CONFIG_FILE = 'config/tcell_agent.config'.freeze
 
 global = OptionParser.new do |opts|
-  opts.banner = "Usage: tcell_agent [options] [subcommand [options]]"
-  opts.on("--version", "Print version") do |v|
+  opts.banner = 'Usage: tcell_agent [options] [subcommand [options]]'
+  opts.on('--version', 'Print version') do |_v|
     require 'tcell_agent/version'
     puts "TCell.io Ruby Agent (Version #{TCellAgent::VERSION})"
     Kernel.exit(1)
   end
-  opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+  opts.on('-v', '--[no-]verbose', 'Run verbosely') do |v|
     options[:verbose] = v
   end
-  opts.separator ""
+  opts.separator ''
   opts.separator subtext
 end
 
 subcommands = {
   'setup' => OptionParser.new do |opts|
-      opts.banner = "Usage: setup"
+    opts.banner = 'Usage: setup'
   end,
   'loglevel' => OptionParser.new do |opts|
-      opts.banner = "Usage: loglevel [options] error|warn|info|debug"
-      opts.on("-o", "--off", "turn logging off ") do |v|
-        options[:off] = v
-      end
+    opts.banner = 'Usage: loglevel [options] error|warn|info|debug'
+    opts.on('-o', '--off', 'turn logging off ') do |v|
+      options[:off] = v
+    end
   end,
   'preload' => OptionParser.new do |opts|
-      opts.banner = "Usage: loglevel [options] [preload_filename]"
-      opts.on("-o", "--off", "turn preloading filename off ") do |v|
-        options[:off] = v
-      end
+    opts.banner = 'Usage: loglevel [options] [preload_filename]'
+    opts.on('-o', '--off', 'turn preloading filename off ') do |v|
+      options[:off] = v
+    end
   end,
   'demomode' => OptionParser.new do |opts|
-      opts.banner = "Usage: loglevel [options]"
-      opts.on("-o", "--off", "turn preloading filename off ") do |v|
-        options[:off] = v
-      end
+    opts.banner = 'Usage: loglevel [options]'
+    opts.on('-o', '--off', 'turn preloading filename off ') do |v|
+      options[:off] = v
+    end
   end,
   'enable' => OptionParser.new do |opts|
-      opts.banner = "Usage: enable"
+    opts.banner = 'Usage: enable'
   end,
   'disable' => OptionParser.new do |opts|
-      opts.banner = "Usage: disable"
+    opts.banner = 'Usage: disable'
   end,
   'test' => OptionParser.new do |opts|
-      opts.banner = "Usage: test"
-      #opts.on("-q", "--[no-]quiet", "quietly run ") do |v|
-      #  options[:quiet] = v
-      #end
-   end
- }
+              opts.banner = 'Usage: test'
+              # opts.on("-q", "--[no-]quiet", "quietly run ") do |v|
+              #  options[:quiet] = v
+              # end
+            end
+}
 
 global.order!
 command = ARGV.shift
-if command == nil
- puts global
- Kernel.exit(1)
+if command.nil?
+  puts global
+  Kernel.exit(1)
 end
 subcommands[command].order!
 
-
-if (command == 'setup')
-  if !File.directory?(CONFIG_DIR)
+if command == 'setup'
+  unless File.directory?(CONFIG_DIR)
     print "Directory 'config' not found, create? [Y/n]"
-    answer = yesno()
+    answer = yesno
     print "\n"
-    if !answer
-      puts "ERROR: Could not create config"
+    unless answer
+      puts 'ERROR: Could not create config'
       Kernel.exit(1)
     end
-    FileUtils::mkdir_p CONFIG_DIR
+    FileUtils.mkdir_p CONFIG_DIR
   end
-  if File.exists?(CONFIG_FILE)
-    print "Config file already exists, overwrite? [y/N]"
+  if File.exist?(CONFIG_FILE)
+    print 'Config file already exists, overwrite? [y/N]'
     answer = yesno(false)
     print "\n"
-    if !answer
-      puts "Keeping existing config"
+    unless answer
+      puts 'Keeping existing config'
       Kernel.exit(1)
     end
   end
-  print "Enter your API Key (ie gAABAAAA...): "
+  print 'Enter your API Key (ie gAABAAAA...): '
   api_key = STDIN.gets.chomp
-  print "Enter your App ID (ie MyApp-Fdk4j): "
+  print 'Enter your App ID (ie MyApp-Fdk4j): '
   app_id = STDIN.gets.chomp
   config_hash = {
-    "version"=>1,
-    "applications"=>[
+    'version' => 1,
+    'applications' => [
       {
-        "app_id"=>app_id,
-        "api_key"=>api_key
+        'app_id' => app_id,
+        'api_key' => api_key
       }
     ]
   }
-  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
-  puts "done."
+  File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
+  puts 'done.'
 
-elsif (command == 'loglevel')
+elsif command == 'loglevel'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
-  logging_options = config_hash["applications"][0].fetch("logging_options",{})
+  logging_options = config_hash['applications'][0].fetch('logging_options', {})
 
   if options[:off] == true
-    logging_options["enabled"] = false
+    logging_options['enabled'] = false
   else
     loglevel = ARGV.pop
-    if (loglevel == nil)
+    if loglevel.nil?
       puts subcommands[command]
       Kernel.exit(1)
     end
     loglevel = loglevel.upcase
-    if ["ERROR","WARN","INFO","DEBUG"].include?(loglevel)
-      logging_options["enabled"] = true
-      logging_options["level"] = loglevel
+    if %w[ERROR WARN INFO DEBUG].include?(loglevel)
+      logging_options['enabled'] = true
+      logging_options['level'] = loglevel
     else
       puts subcommands[command]
       Kernel.exit(1)
     end
   end
-  config_hash["applications"][0]["logging_options"] = logging_options
-  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
-  puts "done."
+  config_hash['applications'][0]['logging_options'] = logging_options
+  File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
+  puts 'done.'
 
-elsif (command == 'preload')
+elsif command == 'preload'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
 
   if options[:off] == true
-    config_hash["applications"][0].delete("preload_policy_filename")
+    config_hash['applications'][0].delete('preload_policy_filename')
   else
     preload_policy_filename = ARGV.pop
-    if (preload_policy_filename == nil)
+    if preload_policy_filename.nil?
       puts subcommands[command]
       Kernel.exit(1)
     end
-    config_hash["applications"][0]["preload_policy_filename"] = preload_policy_filename
+    config_hash['applications'][0]['preload_policy_filename'] = preload_policy_filename
   end
-  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
-  puts "done."
+  File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
+  puts 'done.'
 
-elsif (command == 'enable')
+elsif command == 'enable'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
-  config_hash["applications"][0].delete("enabled")
-  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
-  puts "Enabled, you will need to restart the server."
+  config_hash['applications'][0].delete('enabled')
+  File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
+  puts 'Enabled, you will need to restart the server.'
 
-elsif (command == 'disable')
+elsif command == 'disable'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
-  config_hash["applications"][0]["enabled"] = false
-  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
-  puts "Disabled, you will need to restart the server."
+  config_hash['applications'][0]['enabled'] = false
+  File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
+  puts 'Disabled, you will need to restart the server.'
 
-elsif (command == 'demomode')
+elsif command == 'demomode'
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
   if options[:off] == true
-    config_hash["applications"][0].delete("demomode")
+    config_hash['applications'][0].delete('demomode')
   else
-    config_hash["applications"][0]["demomode"] = true
+    config_hash['applications'][0]['demomode'] = true
   end
-  File.open(CONFIG_FILE, 'w'){|f| f.puts JSON.pretty_generate(config_hash) }
-  puts "done."
+  File.open(CONFIG_FILE, 'w') { |f| f.puts JSON.pretty_generate(config_hash) }
+  puts 'done.'
 
-elsif (command == 'test')
+elsif command == 'test'
   puts
-  printf "%-50s", "Config file exists... "
-  if !File.exists?(CONFIG_FILE)
-    puts "failed"
+  printf '%-50s', 'Config file exists... '
+  unless File.exist?(CONFIG_FILE)
+    puts 'failed'
     Kernel.exit(1)
   end
-  puts "passed"
+  puts 'passed'
 
-  printf "%-50s", "Config valid json... "
+  printf '%-50s', 'Config valid json... '
   file = File.read(CONFIG_FILE)
   config_hash = JSON.parse(file)
-  puts "passed"
+  puts 'passed'
 
-  printf "%-50s", "Config file has valid version... "
-  if config_hash.fetch("version") != 1
-    puts "failed"
+  printf '%-50s', 'Config file has valid version... '
+  if config_hash.fetch('version') != 1
+    puts 'failed'
     Kernel.exit(1)
   end
-  puts "passed"
+  puts 'passed'
 
-  printf "%-50s", "Config file has application..."
-  if config_hash.fetch("applications").length == 0
-    puts "failed"
+  printf '%-50s', 'Config file has application...'
+  if config_hash.fetch('applications').empty?
+    puts 'failed'
     Kernel.exit(1)
   end
-  puts "passed"
+  puts 'passed'
 
-  printf "%-50s", "Application has api_key and app_id... "
-  tcell_application = config_hash.fetch("applications")[0]
-  if !tcell_application.key?("app_id") || !tcell_application.key?("api_key")
-    puts "failed"
+  printf '%-50s', 'Application has api_key and app_id... '
+  tcell_application = config_hash.fetch('applications')[0]
+  if !tcell_application.key?('app_id') || !tcell_application.key?('api_key')
+    puts 'failed'
     Kernel.exit(1)
   end
-  puts "passed"
+  puts 'passed'
 
-  printf "%-50s", "Check for unknown settings... "
-  require "tcell_agent/config/unknown_options"
+  printf '%-50s', 'Check for unknown settings... '
+  require 'tcell_agent/config/unknown_options'
   messages = TCellAgent::Config::Validate.get_unknown_options(config_hash)
-  if messages.size > 0
-    puts "failed"
+  unless messages.empty?
+    puts 'failed'
     messages.each do |message|
-        puts message
+      puts message
     end
     Kernel.exit(1)
   end
-  puts "passed"
+  puts 'passed'
 
-  printf "%-50s", "Requiring configuration library... "
+  printf '%-50s', 'Requiring configuration library... '
   require 'tcell_agent/configuration'
   require 'tcell_agent/api'
-  puts "passed"
+  puts 'passed'
 
-  printf "%-50s", "Make test API call for policies... "
+  printf '%-50s', 'Make test API call for policies... '
   api = TCellAgent::TCellApi.new
-  api.poll_api()
-  puts "passed"
+  api.poll_api
+  puts 'passed'
 
-  printf "%-50s", "Sending a Test event... "
+  printf '%-50s', 'Sending a Test event... '
   send_succeeded = api.send_event_set([])
-  if !send_succeeded
-    puts "failed"
+  unless send_succeeded
+    puts 'failed'
     Kernel.exit(1)
   end
-  puts "passed"
+  puts 'passed'
 
-  printf "%-50s", "Loading native library... "
+  printf '%-50s', 'Loading native library... '
   require 'tcell_agent/rust/whisperer'
-  if !TCellAgent::Rust::Wrapper.common_lib_available?
-    puts "failed"
+  unless TCellAgent::Rust::Wrapper.common_lib_available?
+    puts 'failed'
     Kernel.exit(1)
   end
-  puts "passed"
+  puts 'passed'
 
   puts
-  puts "all tests passed, looks good."
-  puts "done."
+  puts 'all tests passed, looks good.'
+  puts 'done.'
 end
-

data/lib/tcell_agent/agent/event_processor.rb

@@ -12,8 +12,6 @@ require "tcell_agent/policies/clickjacking_policy"
 require "tcell_agent/policies/http_tx_policy"
 require "tcell_agent/policies/http_redirect_policy"
 require "tcell_agent/policies/secure_headers_policy"
-require "tcell_agent/policies/honeytokens_policy"
-require "tcell_agent/policies/appsensor_policy"
 
 require "tcell_agent/sensor_events/server_agent"
 require "tcell_agent/sensor_events/metrics"
@@ -32,6 +30,14 @@ require 'json'
 module TCellAgent
   class Agent
 
+    # cmdi events are special because they can be triggered very easily by running any shell command.
+    # Startup scripts are likely to run shell commands. It's not a good idea to startup the event
+    # processor before worker processses are forked, so the safest thing to do is let a different
+    # event start the event processor to avoid deadlocking worker processes.
+    def is_it_safe_to_send_cmdi_events?()
+      event_processor_running?
+    end
+
     def ensure_event_processor_running
       return if event_processor_running?
       return if TCellAgent.configuration.should_start_event_manager? == false
@@ -124,7 +130,7 @@ module TCellAgent
                       @dispatchEvents.push(event)
                     }
                   end
-                  if (event.flush or @dispatchEvents.length >= @dispatchEventsLimit or wait_for < 0)
+                  if (event.flush || @dispatchEvents.length >= @dispatchEventsLimit || wait_for < 0)
                     last_run_time = Time.now
                     self.send_dispatch_events(tapi)
                   end
@@ -137,7 +143,7 @@ module TCellAgent
               end
             rescue TCellAgent::ConfigurationException
               Thread.exit
-            rescue Exception => e
+            rescue StandardError => e
               last_run_time = Time.now
               TCellAgent.logger.error("Exception while processing events: #{e.message}")
               TCellAgent.logger.debug(e.backtrace)
@@ -146,7 +152,7 @@ module TCellAgent
               }
             end
           end
-        rescue Exception => xyz
+        rescue StandardError => xyz
           TCellAgent.logger.error("Exception while starting processor: #{xyz.message}")
           TCellAgent.logger.debug(xyz.backtrace)
         end
@@ -181,12 +187,12 @@ module TCellAgent
               else
                 TCellAgent::Agent.send_to_event_pipe(event)
               end
-            rescue Exception => e
+            rescue StandardError => e
               TCellAgent.logger.error("Exception while processing (forked) events: #{e.message}")
               TCellAgent.logger.debug(e.backtrace)
             end
           end
-        rescue Exception => e
+        rescue StandardError => e
           TCellAgent.logger.error("Exception while running (forked) events: #{e.message}")
           TCellAgent.logger.debug(e.backtrace)
         end
@@ -201,7 +207,7 @@ module TCellAgent
           return
         end
         @fork_event_queue.push(event)
-      rescue Exception => queue_exception
+      rescue StandardError => queue_exception
         TCellAgent.logger.debug("Could not add (forked) event #{queue_exception.message}")
       end
     end
@@ -231,19 +237,14 @@ module TCellAgent
     def start_metrics_event_thread
       return if TCellAgent.configuration.should_consume_event? == false
       @metrics_event_thread = Thread.new do
-        begin
-          loop do
-            begin
-              event = @metrics_event_queue.pop
-              TCellAgent::Agent.send_to_metrics_pipe(event)
-            rescue Exception => e
-              TCellAgent.logger.error("Exception while processing (forked) metrics: #{e.message}")
-              TCellAgent.logger.debug(e.backtrace)
-            end
+        loop do
+          begin
+            event = @metrics_event_queue.pop
+            TCellAgent::Agent.send_to_metrics_pipe(event)
+          rescue StandardError => e
+            TCellAgent.logger.error("Exception while processing (forked) metrics: #{e.message}")
+            TCellAgent.logger.debug(e.backtrace)
           end
-        rescue Exception => e
-          TCellAgent.logger.error("Exception while running (forked) metrics: #{e.message}")
-          TCellAgent.logger.debug(e.backtrace)
         end
       end
     end
@@ -257,7 +258,7 @@ module TCellAgent
           return
         end
         @metrics_event_queue.push(event)
-      rescue Exception => queue_exception
+      rescue StandardError => queue_exception
         TCellAgent.logger.debug("Could not add (forked) metric #{queue_exception.message}")
       end
     end
@@ -280,7 +281,7 @@ module TCellAgent
           TCellAgent.logger.debug("Dropping event because queue full")
         end
 
-      rescue Exception => queue_exception
+      rescue StandardError => queue_exception
         TCellAgent.logger.debug("Could not add event #{queue_exception.message}")
       end
     end

data/lib/tcell_agent/agent/fork_pipe_manager.rb

@@ -12,14 +12,14 @@ module TCellAgent
       @@parent_id = Process.pid
       def initialize(&block)
         begin
-          @readp, @writep = IO.pipe('ASCII-8BIT', 'ASCII-8BIT', binmode: true)
+          @readp, @writep = IO.pipe('ASCII-8BIT', 'ASCII-8BIT', :binmode => true)
           if defined?(::Encoding::ASCII_8BIT)
             @writep.set_encoding(::Encoding::ASCII_8BIT)
           end
           if is_parent?
             self.start_listener(&block)
           end
-        rescue Exception => init_exception
+        rescue StandardError => init_exception
           TCellAgent.logger.error("Could not start listener for pipe to forks")
           TCellAgent.logger.error(init_exception.message)
           TCellAgent.logger.debug(init_exception.backtrace)
@@ -30,7 +30,7 @@ module TCellAgent
       end
       def start_listener(&block)
         Thread.new {
-          while true do
+          loop do
             begin
               packed_bytes = @readp.read(4)
               event_length = packed_bytes.unpack("L>").first
@@ -39,7 +39,7 @@ module TCellAgent
               if block
                 block.call(event)
               end
-            rescue Exception=>block_exception
+            rescue StandardError => block_exception
               TCellAgent.logger.error("Could not decode block")
               TCellAgent.logger.error(block_exception.message)
               TCellAgent.logger.debug(block_exception.backtrace)
@@ -57,7 +57,7 @@ module TCellAgent
             packed_event = Marshal.dump(event)
             packed_bytes = [packed_event.bytesize].pack("L>")
             @writep.write(packed_bytes+packed_event)
-          rescue Exception => block_exception
+          rescue StandardError => block_exception
             TCellAgent.logger.error("Could not write to pipe")
             TCellAgent.logger.error(block_exception.message)
             TCellAgent.logger.debug(block_exception.backtrace)
@@ -69,7 +69,7 @@ module TCellAgent
     @@event_pipe_manager = ForkPipeManager.new { |event|
       begin
         TCellAgent.send_event(event)
-      rescue Exception => block_exception
+      rescue StandardError => block_exception
         TCellAgent.logger.error("Could handle send_event_block")
         TCellAgent.logger.error(block_exception.message)
         TCellAgent.logger.debug(block_exception.backtrace)
@@ -100,7 +100,7 @@ module TCellAgent
             val.fetch("user_agent", nil)
           )
         else
-          raise Exception.new("Metrics Pipe Manager received unknown metric: #{val.fetch("_type","")}")
+          raise StandardError.new("Metrics Pipe Manager received unknown metric: #{val.fetch("_type","")}")
         end
       end
     }