tcell_agent 1.1.11 → 2.1.2

data/lib/tcell_agent/agent/fork_pipe_manager.rb

@@ -1,113 +0,0 @@
-require 'thread'
-require 'tcell_agent/logger'
-
-module TCellAgent
-  class Agent
-    class ForkPipeManager
-      attr_accessor :readp
-      attr_accessor :writep
-
-      @@parent_id = Process.pid
-      def initialize(&block)
-        @readp, @writep = IO.pipe('ASCII-8BIT', 'ASCII-8BIT', :binmode => true)
-        if defined?(::Encoding::ASCII_8BIT)
-          @writep.set_encoding(::Encoding::ASCII_8BIT)
-        end
-        start_listener(&block) if parent?
-      rescue StandardError => init_exception
-        TCellAgent.logger.error('Could not start listener for pipe to forks')
-        TCellAgent.logger.error(init_exception.message)
-        TCellAgent.logger.debug(init_exception.backtrace)
-      end
-
-      def parent?
-        @@parent_id == Process.pid
-      end
-
-      def start_listener(&block)
-        Thread.new do
-          loop do
-            begin
-              packed_bytes = @readp.read(4)
-              event_length = packed_bytes.unpack('L>').first
-              packed_event = @readp.read(event_length)
-              event = Marshal.load(packed_event) # rubocop:disable Security/MarshalLoad
-              yield(event) if block
-            rescue StandardError => block_exception
-              TCellAgent.logger.error('Could not decode block')
-              TCellAgent.logger.error(block_exception.message)
-              TCellAgent.logger.debug(block_exception.backtrace)
-              sleep 0.5
-            end
-          end
-        end
-      end
-
-      def send_to_parent(event)
-        return if parent?
-
-        begin
-          packed_event = Marshal.dump(event)
-          packed_bytes = [packed_event.bytesize].pack('L>')
-          @writep.write(packed_bytes + packed_event)
-        rescue StandardError => block_exception
-          TCellAgent.logger.error('Could not write to pipe')
-          TCellAgent.logger.error(block_exception.message)
-          TCellAgent.logger.debug(block_exception.backtrace)
-        end
-      end
-    end
-
-    @@event_pipe_manager = ForkPipeManager.new do |event|
-      begin
-        TCellAgent.send_event(event)
-      rescue StandardError => block_exception
-        TCellAgent.logger.error('Could handle send_event_block')
-        TCellAgent.logger.error(block_exception.message)
-        TCellAgent.logger.debug(block_exception.backtrace)
-      end
-    end
-
-    @@metrics_pipe_manager = ForkPipeManager.new do |val|
-      TCellAgent::Instrumentation.safe_block('Handling metrics_pipe_block') do
-        switch_on = val.fetch('_type', '')
-        case switch_on
-        when 'increment_route'
-          TCellAgent.increment_route(
-            val.fetch('route_id', nil),
-            val.fetch('response_time', nil)
-          )
-        when 'discover_database_fields'
-          TCellAgent.discover_database_fields(
-            val.fetch('route_id', nil),
-            val.fetch('database', nil),
-            val.fetch('schema', nil),
-            val.fetch('table', nil),
-            val.fetch('fields', nil)
-          )
-        when 'increment_session_info'
-          TCellAgent.increment_session_info(
-            val.fetch('hmac_session_id', nil),
-            val.fetch('user_id', nil),
-            val.fetch('ip_address', nil),
-            val.fetch('user_agent', nil)
-          )
-        else
-          raise StandardError, "Metrics Pipe Manager received unknown metric: #{val.fetch('_type', '')}"
-        end
-      end
-    end
-
-    def self.parent_process?
-      @@event_pipe_manager.parent?
-    end
-
-    def self.send_to_metrics_pipe(hash_value)
-      @@metrics_pipe_manager.send_to_parent(hash_value)
-    end
-
-    def self.send_to_event_pipe(event)
-      @@event_pipe_manager.send_to_parent(event)
-    end
-  end
-end

data/lib/tcell_agent/agent/policy_manager.rb

@@ -1,219 +0,0 @@
-# See the file "LICENSE" for the full license governing this code.
-
-require 'tcell_agent/logger'
-require 'tcell_agent/version'
-require 'tcell_agent/api'
-require 'tcell_agent/configuration'
-
-require 'tcell_agent/agent/policy_types'
-
-require 'tcell_agent/policies/http_tx_policy'
-require 'tcell_agent/policies/http_redirect_policy'
-
-require 'tcell_agent/sensor_events/server_agent'
-
-require 'tcell_agent/utils/queue_with_timeout'
-
-require 'net/http'
-require 'thread'
-require 'logger'
-require 'json'
-
-module TCellAgent
-  class Agent # rubocop:disable Metrics/ClassLength
-    def ensure_policy_polling_running
-      return if policy_polling_running?
-      return if TCellAgent.configuration.should_start_policy_poll? == false
-
-      @policy_polling_worker_mutex.synchronize do
-        return if policy_polling_running?
-        start_policy_polling_loop
-      end
-    end
-
-    def policy_polling_running?
-      @policy_polling_thread && @policy_polling_thread.alive?
-    end
-
-    def stop_policy_polling
-      TCellAgent.logger.debug('Stopping policy polling thread')
-      @policy_polling_thread.exit if @policy_polling_thread && @policy_polling_thread.alive?
-    end
-
-    def start_policy_polling_loop
-      return unless TCellAgent.configuration.should_start_policy_poll?
-
-      if TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.tcell_api_url)
-        TCellAgent.logger.error('tCell.io tcell_api_url is missing. Disabling policy polling.')
-        return
-      end
-
-      if TCellAgent::Utils::Strings.blank?(TCellAgent.configuration.app_id)
-        TCellAgent.logger.error('tCell.io app_id is missing. Disabling policy polling.')
-        return
-      end
-
-      TCellAgent.logger.debug('Starting policy polling thread')
-
-      @policy_polling_thread = Thread.new do
-        failure_sleep_time = 30
-        last_poll_time = 0
-        last_run = Time.now
-
-        loop do
-          failure_sleep_time, last_poll_time = policy_polling_iteration(failure_sleep_time, last_poll_time)
-
-          unless TCellAgent.configuration.demomode
-            if (Time.now - last_run) < 2
-              random = Random.new
-              sleeptime = sleep(random.rand(5..20))
-              TCellAgent.logger.debug("Rate limiting: sleeping #{sleeptime} seconds")
-              sleep(sleeptime)
-            end
-          end
-
-          last_run = Time.now
-        end
-      end
-    end
-
-    def policy_polling_iteration(failure_sleep_time, last_poll_time)
-      begin
-        policy_jsons = @@policy_tapi.poll_api(last_poll_time)
-
-        if policy_jsons.nil?
-          TCellAgent.logger.error("Policy was nil. Sleeping for #{failure_sleep_time}")
-
-          sleep(failure_sleep_time)
-
-          failure_sleep_time *= 2 if failure_sleep_time < 480
-
-          return [failure_sleep_time, last_poll_time]
-
-        elsif policy_jsons.key?('last_id')
-          if policy_jsons['last_id'] != 0
-            last_poll_time = policy_jsons['last_id']
-          end
-        end
-
-        failure_sleep_time = 30
-
-        process_policy_json(policy_jsons)
-      rescue TCellAgent::ConfigurationException
-        Thread.exit
-      rescue StandardError => e
-        TCellAgent.logger.error("exception while handling connection: #{e.message}")
-        TCellAgent.logger.debug(e.backtrace)
-        TCellAgent.logger.debug('Sleeping 30 seconds because the tCell.io request failed...')
-        sleep(failure_sleep_time)
-
-        failure_sleep_time *= 2 if failure_sleep_time < 480
-      end
-
-      [failure_sleep_time, last_poll_time]
-    end
-
-    def process_policy_json(policy_jsons, cache_the_policy = true)
-      return if policy_jsons.nil?
-
-      TCellAgent::PolicyTypes::CLASS_MAP.each do |policy_type, policy_class|
-        next unless policy_jsons.key?(policy_type)
-        new_policy = policy_class.from_json(policy_jsons[policy_type])
-        next unless new_policy
-        @lock.synchronize do
-          @policies[policy_type] = new_policy
-        end
-      end
-
-      @policies[TCellAgent::PolicyTypes::RUST].update_policies(policy_jsons)
-
-      return unless cache_the_policy
-
-      (TCellAgent::PolicyTypes::CLASS_MAP.keys +
-      [TCellAgent::PolicyTypes::CSP,
-       TCellAgent::PolicyTypes::CLICKJACKING,
-       TCellAgent::PolicyTypes::SECUREHEADERS,
-       TCellAgent::PolicyTypes::JSAGENTINJECTION,
-       TCellAgent::PolicyTypes::APPSENSOR,
-       TCellAgent::PolicyTypes::PATCHES,
-       TCellAgent::PolicyTypes::COMMANDINJECTION,
-       TCellAgent::PolicyTypes::REGEX]).each do |policy_type|
-         @lock.synchronize do
-           cache(policy_type, policy_jsons[policy_type]) if policy_jsons[policy_type]
-         end
-       end
-    end
-
-    def cache(policy_name, policy)
-      cache_filename = TCellAgent.configuration.cache_filename_with_app_id
-
-      begin
-        TCellAgent::Utils::IO.create_directory(
-          File.dirname(cache_filename),
-          TCellAgent.configuration.agent_home_owner
-        )
-
-        f1 = File.open(cache_filename, File::RDWR | File::CREAT)
-
-        Timeout.timeout(0.100) { f1.flock(File::LOCK_EX) }
-
-        policy_cache = {}
-        existing_policy = f1.read
-
-        begin
-          if !existing_policy.nil? && existing_policy != ''
-            policy_jsons = JSON.parse(existing_policy)
-            if policy_jsons
-              policy_cache = if policy_jsons.key?('result')
-                               policy_jsons['result']
-                             else
-                               policy_jsons
-                             end
-            end
-          end
-          policy_cache[policy_name] = policy
-          @complete_policy_cache = policy_cache
-        rescue StandardError => e
-          TCellAgent.logger.warn(e.message)
-          policy_cache = @complete_policy_cache if @complete_policy_cache
-        end
-
-        f1.rewind
-        f1.write(JSON.dump(policy_cache))
-        f1.flush
-        f1.truncate(f1.pos)
-
-        TCellAgent::Utils::IO.set_owner(
-          cache_filename,
-          TCellAgent.configuration.agent_home_owner
-        )
-      rescue StandardError => e
-        TCellAgent.logger.warn(e.message)
-      ensure
-        f1.close unless f1.nil?
-      end
-    end
-
-    def policies_from_cachefile
-      cache_filename = TCellAgent.configuration.cache_filename_with_app_id
-
-      return nil unless File.exist?(cache_filename)
-
-      begin
-        f1 = File.open(cache_filename, File::RDONLY)
-        Timeout.timeout(1) { f1.flock(File::LOCK_SH) }
-        policy_filedata = f1.read
-        f1.close
-
-        policy_jsons = JSON.parse(policy_filedata)
-        return policy_jsons['result'] if policy_jsons.key?('result')
-        @complete_policy_cache = policy_jsons
-        return policy_jsons
-      rescue StandardError => e
-        TCellAgent.logger.warn(e.message)
-      end
-
-      nil
-    end
-  end
-end

data/lib/tcell_agent/agent/policy_types.rb

@@ -1,30 +0,0 @@
-require 'tcell_agent/policies/http_tx_policy'
-require 'tcell_agent/policies/http_redirect_policy'
-require 'tcell_agent/policies/login_fraud_policy'
-require 'tcell_agent/policies/dataloss_policy'
-require 'tcell_agent/policies/rust_policies'
-
-module TCellAgent
-  class PolicyTypes
-    CSP = 'csp-headers'.freeze
-    CLICKJACKING = 'clickjacking'.freeze
-    SECUREHEADERS = 'secure-headers'.freeze
-    HTTPTX = 'http-tx'.freeze
-    HTTPREDIRECT = 'http-redirect'.freeze
-    LOGINFRAUD = 'login'.freeze
-    DATALOSS = 'dlp'.freeze
-    APPSENSOR = 'appsensor'.freeze
-    PATCHES = 'patches'.freeze
-    COMMANDINJECTION = 'cmdi'.freeze
-    REGEX = 'regex'.freeze
-    RUST = 'rust'.freeze
-    JSAGENTINJECTION = 'jsagentinjection'.freeze
-
-    CLASS_MAP = {
-      HTTPTX => TCellAgent::Policies::HttpTxPolicy,
-      HTTPREDIRECT => TCellAgent::Policies::HttpRedirectPolicy,
-      LOGINFRAUD => TCellAgent::Policies::LoginFraudPolicy,
-      DATALOSS => TCellAgent::Policies::DataLossPolicy
-    }.freeze
-  end
-end

data/lib/tcell_agent/api.rb

@@ -1,91 +0,0 @@
-# See the file "LICENSE" for the full license governing this code.
-require 'json'
-require 'tcell_agent/logger'
-require 'tcell_agent/configuration'
-require 'tcell_agent/version'
-require 'date'
-
-require 'net/http'
-
-module TCellAgent
-  class TCellApi
-    def initialize; end
-
-    def poll_api(last_id = nil)
-      if !TCellAgent.configuration || !TCellAgent.configuration.tcell_api_url || !TCellAgent.configuration.app_id
-        raise TCellAgent::ConfigurationException, "Config Information Not Found, can't poll for policy updates"
-      end
-
-      full_url = TCellAgent.configuration.tcell_api_url.sub(
-        '{app_id}',
-        TCellAgent.configuration.app_id
-      )
-      full_url += "&last_id=#{last_id}" if last_id
-
-      TCellAgent.logger.debug("tCell.io API Request: #{full_url}")
-
-      uri = URI(full_url)
-      req = Net::HTTP::Get.new(uri.request_uri)
-      req['Authorization'] = 'Bearer ' + TCellAgent.configuration.api_key
-      begin
-        req['TCellAgent'] = 'RubyAgent ' + TCellAgent::VERSION
-      rescue StandardError => e
-        TCellAgent.logger.debug("tCell.io Could not add agent string: #{e.message}")
-      end
-
-      res = Net::HTTP.start(uri.hostname, uri.port, :use_ssl => (uri.scheme == 'https')) { |http| http.request(req) }
-
-      if res.is_a?(Net::HTTPSuccess)
-        TCellAgent.logger.debug("tCell.io API Response: #{res.body}".force_encoding('UTF-8'))
-        return JSON.parse(res.body)
-      else
-        TCellAgent.logger.error("Received error response while contacting api: #{res.inspect}")
-      end
-
-      nil
-    end
-
-    def send_event_set(events)
-      if !TCellAgent.configuration || !TCellAgent.configuration.tcell_input_url || !TCellAgent.configuration.app_id
-        raise TCellAgent::ConfigurationException, "Config Information Not Found, can't send events"
-      end
-
-      return false if events.nil?
-
-      eventset = { 'uuid' => TCellAgent.configuration.uuid,
-                   'hostname' => TCellAgent.configuration.host_identifier,
-                   'events' => events }
-      TCellAgent.logger.debug("Sending #{JSON.dump(eventset)}")
-      full_url = TCellAgent.configuration.tcell_input_url +
-                 '/app/' +
-                 TCellAgent.configuration.app_id +
-                 '/server_agent'
-
-      TCellAgent.logger.debug("tCell.io SendEvents API Request: #{full_url}")
-
-      uri = URI(full_url)
-      req = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
-      req.body = JSON.dump(eventset)
-      req['Authorization'] = 'Bearer ' + TCellAgent.configuration.api_key
-      req['Content-Type'] = 'application/json'
-      req['Accept'] = 'application/json'
-
-      begin
-        req['TCellAgent'] = 'RubyAgent ' + TCellAgent::VERSION
-      rescue StandardError => e
-        TCellAgent.logger.debug('tCell.io Could not add agent string: ' + e.message)
-      end
-
-      res = Net::HTTP.start(uri.hostname, uri.port, :use_ssl => (uri.scheme == 'https')) { |http| http.request(req) }
-
-      TCellAgent.logger.debug("tCell.io SendEvents API Response: #{res.code}")
-
-      res.is_a?(Net::HTTPSuccess)
-    end
-
-    def valid_header?(str)
-      # TODO: test the unescaped backslash
-      str =~ %r{^[\p{L}\w\d\-_ :/,;.'\"%?@#=$]*$}
-    end
-  end
-end