threatstack-agent-ruby 0.2.1

data/ext/libinjection/libinjection_xss.h

@@ -0,0 +1,21 @@
+#ifndef LIBINJECTION_XSS
+#define LIBINJECTION_XSS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * HEY THIS ISN'T DONE
+ */
+
+/* pull in size_t */
+
+#include <string.h>
+
+  int libinjection_is_xss(const char* s, size_t len, int flags);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

data/lib/constants.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'securerandom'
+
+module Threatstack
+  module Constants
+    def self.env(name, default = nil)
+      ts_var = "THREATSTACK_#{name}"
+      bf_var = "BLUEFYRE_#{name}"
+      bf_or_default = ENV.has_key?(bf_var) ? ENV[bf_var] : default
+      ENV.has_key?(ts_var) ? ENV[ts_var] : bf_or_default
+    end
+
+    def self.is_truthy(name, default = false)
+      ts_var = "THREATSTACK_#{name}"
+      bf_var = "BLUEFYRE_#{name}"
+      bf_or_default = ENV.has_key?(bf_var) ? ENV[bf_var] : default
+      val = ENV.has_key?(ts_var) ? ENV[ts_var] : bf_or_default
+      TRUTHY.include?(val.to_s.downcase)
+    end
+
+    TRUTHY = ['true', '1', 'yes'].freeze
+
+    # AGENT
+    RUBY = 'ruby'
+    AGENT_NAME = 'threatstack-agent-ruby'
+    ## main agent id
+    AGENT_ID = self.env('AGENT_ID', '')
+    ## autogenerated Id for this agent instance
+    AGENT_INSTANCE_ID = SecureRandom.uuid
+    ## whether or not the agent is disabled
+    DISABLED = self.is_truthy('DISABLED')
+    ## whether or not initialization is done manually by calling
+    MANUAL_INIT = self.is_truthy('MANUAL_INIT')
+    ## whether or not requests containing XSS payloads should be blocked
+    BLOCK_XSS = self.is_truthy('BLOCK_XSS')
+    ## whether or not requests containing SQLI payloads should be blocked
+    BLOCK_SQLI = self.is_truthy('BLOCK_SQLI')
+    ## specifies which user fields should be omitted from event payloads
+    DROP_FIELDS = self.env('DROP_FIELDS', false) ? self.env('DROP_FIELDS').split(',').each_with_object({}) do |val, h|
+      h[val] = true
+    end : nil
+    ## string to use when redacting fields
+    REDACTED = self.env('REDACTED', '#REDACTED#')
+
+    # EVENT SUBMITTER
+    ## event reporting frequency
+    JOB_INTERVAL = Integer(self.env('SUBMISSION_INTERVAL', 10))
+    ## max number of events per request
+    EVENTS_PER_REQ = Integer(self.env('EVENTS_PER_REQ', 1000))
+    ## base url
+    APPSEC_BASE_URL = self.env('API_COLLECTOR_URL', 'https://appsec-sensors.threatstack.com')
+    ## event collector path
+    APPSEC_EVENTS_URL = '/api/events'
+
+    # LOGGING
+    ## logging level threshold
+    LOG_LEVEL = self.env('LOG_LEVEL', 'UNKNOWN')
+    ## toggle color output for logging
+    LOG_COLORS = self.is_truthy('LOG_COLORS')
+
+    # AWS
+    AWS_METADATA_URL = self.env('AWS_METADATA_BASE_URL', 'http://169.254.169.254/latest/dynamic/instance-identity/document')
+
+    # EVENTS
+    INSTRUMENTATION = 'instrumentation'
+    DEPENDENCIES = 'dependencies'
+    ENVIRONMENT = 'environment'
+    ATTACK = 'attack'
+
+    # IP
+    IPV4 = 'IPv4'
+    IPV6 = 'IPv6'
+
+    # Strings
+    XSS = 'xss'
+    SQLI = 'sqli'
+    REQUEST_BLOCKED = 'Request blocked'
+    DETECTED_NOT_BLOCKED = 'Detected not blocked'
+    CGI_VARIABLES = Set.new(%w[ AUTH_TYPE CONTENT_LENGTH CONTENT_TYPE GATEWAY_INTERFACE HTTPS PATH_INFO
+                                PATH_TRANSLATED REMOTE_ADDR REMOTE_HOST REMOTE_IDENT REMOTE_USER
+                                REQUEST_METHOD SCRIPT_NAME SERVER_NAMESERVER_PORT SERVER_PROTOCOL
+                                SERVER_SOFTWARE]).freeze
+  end
+end
+
+require_relative './utils/logger'
+
+module Threatstack
+  module Constants
+    spec = Gem.loaded_specs['threatstack-agent-ruby']
+    logger = Threatstack::Utils::TSLogger.create 'Constants'
+    logger.info """ Threatstack Ruby Agent Config
+                VERSION: #{spec.nil? || !spec.respond_to?(:version) ? 'N/A' : spec.version}
+                  STATE: #{DISABLED ? 'Disabled' : 'Enabled'}
+               AGENT ID: #{AGENT_ID}
+      AGENT INSTANCE ID: #{AGENT_INSTANCE_ID}
+      APPSEC SENSOR URL: #{APPSEC_BASE_URL}
+             BLOCK SQLI: #{BLOCK_SQLI}
+              BLOCK XSS: #{BLOCK_XSS}
+            DROP FIELDS: #{DROP_FIELDS}
+        SUBMIT INTERVAL: #{JOB_INTERVAL}
+         EVENTS PER REQ: #{EVENTS_PER_REQ}
+              LOG LEVEL: #{LOG_LEVEL}
+             LOG COLORS: #{LOG_COLORS}
+            MANUAL INIT: #{MANUAL_INIT}
+          REDACTED TEXT: #{REDACTED}"""
+  end
+end

data/lib/control.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'thread'
+
+require_relative './events/models/environment_event'
+require_relative './events/models/dependency_event'
+require_relative './instrumentation/rails'
+require_relative './instrumentation/kernel'
+require_relative './jobs/event_submitter'
+require_relative './jobs/delayed_job'
+require_relative './utils/logger'
+require_relative './constants'
+
+module Threatstack
+  module Control
+    include Threatstack::Constants
+
+    @@agent_initialized = false
+    @@agent_init_mutex = Mutex.new
+
+    def self.init
+      @@agent_init_mutex.synchronize do
+        return if @@agent_initialized
+        @@agent_initialized = true
+      end
+      logger = Threatstack::Utils::TSLogger.create 'MainAgent'
+      logger.info 'Initializing Threatstack Ruby agent'
+
+      # patch Rails ActionController
+      logger.info 'Instrumenting Rails...'
+      Threatstack::Instrumentation::TSRails.patch_action_controller
+      logger.info 'Done instrumenting Rails'
+
+      # patch Kernel methods
+      logger.info 'Instrumenting Kernel methods...'
+      Threatstack::Instrumentation::TSKernel.wrap_methods
+      logger.info 'Done instrumenting Kernel methods'
+
+      # Start EventSubmitter asynchronously
+      logger.info 'Starting Event Submitter...'
+      Threatstack::Jobs::EventSubmitter.instance.start
+      logger.info 'Started Event Submitter'
+
+      # Gather environment and dependency info asynchronously
+      Threatstack::Jobs::DelayedJob.new(logger, 5) do
+        dep_event = Threatstack::Events::DependencyEvent.new
+        # submit dependency event
+        Threatstack::Jobs::EventSubmitter.instance.queue_event dep_event
+        # submit environment event
+        Threatstack::Jobs::EventSubmitter.instance.queue_event Threatstack::Events::EnvironmentEvent.new
+      end
+
+      logger.info 'Initialization done for agent'
+    end
+
+    def self._setup_agent
+      # initialize agent unless DISABLED or set to MANUAL_INIT
+      self.init unless DISABLED || MANUAL_INIT
+    end
+  end
+end

data/lib/events/event_accumulator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+require_relative '../utils/logger'
+
+module Threatstack
+  module Events
+    # Singleton class that handles temporarily caching events until they're sent to the back end
+    class EventAccumulator
+      include Singleton
+
+      attr_reader :events
+
+      def initialize
+        @events = []
+        @logger = Threatstack::Utils::TSLogger.create 'EventAccumulator'
+      end
+
+      def add_event(event)
+        @logger.debug "Adding event - New Total: #{@events.length + 1}"
+        @events.push(event)
+      end
+
+      def remove_events(num = 1)
+        @logger.debug "Removing #{num} event(s) - New Total: #{@events.length - num}"
+        @events.shift(num)
+      end
+
+      def clear_events
+        @logger.debug "Clearing events - Total: #{@events.length}"
+        @events.clear
+      end
+    end
+  end
+end

data/lib/events/models/attack_event.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative './base_event'
+require_relative '../../constants'
+
+module Threatstack
+  module Events
+    # Instrumentation event model that inherits the common attributes and adds its own specifics
+    class AttackEvent < BaseEvent
+      attr_accessor :request_ip
+      attr_accessor :request_headers
+      attr_accessor :request_url
+      attr_accessor :request_method
+      attr_accessor :module_name
+      attr_accessor :attack_message
+      attr_accessor :attack_stack
+      attr_accessor :attack_details
+
+      # @param [Hash]    args
+      #        [String]  args.event_id
+      #        [String]  args.timestamp
+      #        [String]  args.request_ip
+      #        [Hash]    args.request_headers
+      #        [String]  args.request_url
+      #        [String]  args.request_method
+      #        [String]  args.module_name
+      #        [String]  args.attack_message
+      #        [String]  args.attack_stack
+      #        [Hash]    args.attack_details
+      def initialize(args)
+        args[:event_type] = Threatstack::Constants::ATTACK
+        @request_ip = args[:request_ip]
+        @request_headers = args[:request_headers]
+        @request_url = args[:request_url]
+        @request_method = args[:request_method]
+        @module_name = args[:module_name]
+        @attack_message = args[:attack_message]
+        @attack_stack = args[:attack_stack]
+        @attack_details = args[:attack_details]
+        super args
+      end
+
+      def to_hash
+        hash = to_core_hash
+        hash[:module_name] = @module_name
+        hash[:payload] = {
+          :attack => {
+            :message => @attack_message, :stack => @attack_stack, :details => @attack_details
+          },
+          :req => {
+            :headers => @request_headers, :ip_address => @request_ip, :url => @request_url, :method => @request_method
+          }
+        }
+        hash
+      end
+    end
+  end
+end

data/lib/events/models/base_event.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+
+require_relative '../../constants'
+
+module Threatstack
+  module Events
+    # Base event model containing the common attributes
+    class BaseEvent
+      attr_accessor :event_id
+      attr_accessor :event_type
+      attr_accessor :timestamp
+      attr_reader :agent_type
+
+      # @param [Hash]    args
+      #        [String]  args.event_id
+      #        [String]  args.event_type
+      #        [String]  args.timestamp
+      def initialize(args)
+        @event_id = args[:event_id].nil? ? SecureRandom.uuid : args[:event_id]
+        @timestamp = args[:timestamp].nil? ? Time.now.utc.strftime('%FT%T.%3NZ') : args[:timestamp]
+        @event_type = args[:event_type]
+        @agent_type = Threatstack::Constants::RUBY
+      end
+
+      def to_hash
+        Hash[instance_variables.map { |name| [name[1..-1], instance_variable_get(name)] }]
+      end
+
+      def to_core_hash
+        { :event_id => @event_id, :event_type => @event_type, :agent_type => @agent_type, :timestamp => @timestamp }
+      end
+
+      def to_json_string
+        to_hash.to_json
+      end
+    end
+  end
+end

data/lib/events/models/dependency_event.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'bundler'
+
+require_relative './base_event'
+require_relative '../../constants'
+require_relative '../../utils/logger'
+
+module Threatstack
+  module Events
+    # Dependency event model that inherits the common attributes and adds its own specifics
+    class DependencyEvent < BaseEvent
+      include Threatstack::Constants
+
+      attr_accessor :name
+      attr_accessor :dependencies
+      attr_accessor :graph
+
+      # @param [Hash]    args
+      #        [String]  args.event_id
+      #        [String]  args.timestamp
+      def initialize(args = {})
+        logger = Threatstack::Utils::TSLogger.create 'DependencyEvent'
+        logger.debug 'Creating dependency event...'
+        args[:event_type] = DEPENDENCIES
+        begin
+          root_dir = self.app_root_dir
+          gemfile_path = File.join(root_dir, 'Gemfile')
+          lockfile_path = File.join(root_dir, 'Gemfile.lock')
+          logger.debug "Root Dir: #{root_dir}"
+          @name = File.basename root_dir
+          # build dependency list
+          @dependencies = Bundler::Definition.build(gemfile_path, lockfile_path, nil).
+            dependencies.each_with_object({}) do |dep, obj|
+            dep.groups.each do
+              begin
+                obj[dep.name] = dep.to_spec.version.to_s
+              rescue ScriptError => sce
+                logger.error "ScriptError: #{sce.inspect}"
+              rescue StandardError => ste
+                logger.error "StandardError: #{ste.inspect}"
+              end
+            end
+          end
+          # build dependency graph
+          @graph = Gem.loaded_specs.each_with_object({}) do |(k, v), obj|
+            requires = v.dependencies.each_with_object({}) do |dep, h|
+              h[dep.name] = dep.requirement.requirements.join
+            end
+            begin
+              obj[k] = { :version => v.version.to_s, :requires => requires }
+            rescue ScriptError => sce
+              logger.error "ScriptError: #{sce.inspect}"
+            rescue StandardError => ste
+              logger.error "StandardError: #{ste.inspect}"
+            end
+          end if Gem.respond_to? :loaded_specs
+          logger.debug "Name: #{@name}"
+          logger.debug "Dependencies: #{@dependencies}"
+          logger.debug "Graph: #{@graph}"
+        rescue ScriptError => sce
+          logger.error "General ScriptError: #{sce.inspect}"
+        rescue StandardError => ste
+          logger.error "General StandardError: #{ste.inspect}"
+        end
+        super args
+      end
+
+      # Returns the root directory of the currently running app
+      def app_root_dir
+        return Bundler.root if defined?(Bundler)
+
+        return ENV['RAILS_ROOT'] if defined?(ENV['RAILS_ROOT']) && ENV['RAILS_ROOT'].to_s.strip.length != 0
+
+        return Rails.root if defined?(Rails) && Rails.root.to_s.strip.length != 0
+
+        Dir.pwd
+      end
+
+      def to_hash
+        hash = to_core_hash
+        hash[:module_name] = AGENT_NAME
+        hash[:package_type] = RUBY
+        hash[:payload] = {
+          :name => @name,
+          :dependencies => @dependencies,
+          :graph => @graph
+        }
+        hash
+      end
+    end
+  end
+end

data/lib/events/models/environment_event.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'socket'
+require 'platform'
+require 'network_interface'
+
+require_relative './base_event'
+require_relative '../../constants'
+require_relative '../../utils/aws_utils'
+require_relative '../../utils/logger'
+
+module Threatstack
+  module Events
+
+    # Environment event model that inherits the common attributes and adds its own specifics
+    class EnvironmentEvent < BaseEvent
+      include Threatstack::Constants
+
+      attr_accessor :os_type
+      attr_accessor :os_platform
+      attr_accessor :os_arch
+      attr_accessor :hostname
+      attr_accessor :metadata
+      attr_accessor :interfaces
+      attr_accessor :versions
+
+      # @param [Hash]    args
+      #        [String]  args.event_id
+      #        [String]  args.timestamp
+      def initialize(args = {})
+        logger = Threatstack::Utils::TSLogger.create 'EnvironmentEvent'
+        logger.debug 'Creating environment event...'
+        args[:event_type] = ENVIRONMENT
+        @os_type = Platform::IMPL.to_s
+        @os_platform = Platform::OS.to_s
+        @os_arch = Platform::ARCH.to_s
+        @hostname = Socket.gethostname
+        @versions = { :ruby => RUBY_VERSION }
+        @metadata = Threatstack::Utils::Aws.instance.get_aws_metadata
+        # interface type constants
+        af_link = NetworkInterface::AF_LINK
+        af_inet = NetworkInterface::AF_INET
+        af_inet6 = NetworkInterface::AF_INET6
+        # build interface hash
+        @interfaces = NetworkInterface.interfaces.each_with_object({}) do |iname, hash|
+          addresses = NetworkInterface.addresses(iname)
+          to_keep = []
+          mac = nil
+          # get mac address if any
+          if addresses[af_link] && !addresses[af_link].empty?
+            addr = addresses[af_link][0]
+            mac = addr['addr'] if !addr['addr'].nil? && !addr['addr'].empty?
+          end
+          # get IPv4 addresses if any
+          if addresses[af_inet] && !addresses[af_inet].empty?
+            addresses[af_inet].each do |addr|
+              next unless addr['addr'] && !addr['addr'].empty?
+
+              to_keep.push(:address => addr['addr'], :netmask => addr['netmask'], :family => IPV4, :mac => mac)
+            end
+          end
+          # get IPv6 addresses if any
+          if addresses[af_inet6] && !addresses[af_inet6].empty?
+            addresses[af_inet6].each do |addr|
+              next unless addr['addr'] && !addr['addr'].empty?
+
+              to_keep.push(:address => addr['addr'], :netmask => addr['netmask'], :family => IPV6, :mac => mac)
+            end
+          end
+          # add interface entry unless no addresses were found
+          hash[iname] = to_keep unless to_keep.empty?
+        end
+        super args
+      end
+
+      def to_hash
+        hash = to_core_hash
+        hash[:module_name] = AGENT_NAME
+        hash[:payload] = {
+          :hostname => @hostname,
+          :os => {
+            :platform => @os_platform, :type => @os_type, :arch => @os_arch
+          },
+          :aws => @metadata,
+          :versions => @versions,
+          :network => @interfaces
+        }
+        hash
+      end
+    end
+
+  end
+end