threatstack-agent-ruby 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +3 -0
  3. data/LICENSE +6 -0
  4. data/ext/libinjection/extconf.rb +4 -0
  5. data/ext/libinjection/libinjection.h +65 -0
  6. data/ext/libinjection/libinjection.i +13 -0
  7. data/ext/libinjection/libinjection_html5.c +850 -0
  8. data/ext/libinjection/libinjection_html5.h +54 -0
  9. data/ext/libinjection/libinjection_sqli.c +2325 -0
  10. data/ext/libinjection/libinjection_sqli.h +298 -0
  11. data/ext/libinjection/libinjection_sqli_data.h +9654 -0
  12. data/ext/libinjection/libinjection_wrap.c +2393 -0
  13. data/ext/libinjection/libinjection_xss.c +532 -0
  14. data/ext/libinjection/libinjection_xss.h +21 -0
  15. data/lib/constants.rb +110 -0
  16. data/lib/control.rb +61 -0
  17. data/lib/events/event_accumulator.rb +36 -0
  18. data/lib/events/models/attack_event.rb +58 -0
  19. data/lib/events/models/base_event.rb +41 -0
  20. data/lib/events/models/dependency_event.rb +93 -0
  21. data/lib/events/models/environment_event.rb +93 -0
  22. data/lib/events/models/instrumentation_event.rb +46 -0
  23. data/lib/exceptions/request_blocked_error.rb +11 -0
  24. data/lib/instrumentation/common.rb +172 -0
  25. data/lib/instrumentation/instrumenter.rb +144 -0
  26. data/lib/instrumentation/kernel.rb +45 -0
  27. data/lib/instrumentation/rails.rb +61 -0
  28. data/lib/jobs/delayed_job.rb +26 -0
  29. data/lib/jobs/event_submitter.rb +101 -0
  30. data/lib/jobs/job_queue.rb +38 -0
  31. data/lib/jobs/recurrent_job.rb +61 -0
  32. data/lib/threatstack-agent-ruby.rb +7 -0
  33. data/lib/utils/aws_utils.rb +46 -0
  34. data/lib/utils/formatter.rb +47 -0
  35. data/lib/utils/logger.rb +43 -0
  36. data/threatstack-agent-ruby.gemspec +35 -0
  37. metadata +221 -0
@@ -0,0 +1,21 @@
1
+ #ifndef LIBINJECTION_XSS
2
+ #define LIBINJECTION_XSS
3
+
4
+ #ifdef __cplusplus
5
+ extern "C" {
6
+ #endif
7
+
8
+ /**
9
+ * HEY THIS ISN'T DONE
10
+ */
11
+
12
+ /* pull in size_t */
13
+
14
+ #include <string.h>
15
+
16
+ int libinjection_is_xss(const char* s, size_t len, int flags);
17
+
18
+ #ifdef __cplusplus
19
+ }
20
+ #endif
21
+ #endif
data/lib/constants.rb ADDED
@@ -0,0 +1,110 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'set'
4
+ require 'securerandom'
5
+
6
+ module Threatstack
7
+ module Constants
8
+ def self.env(name, default = nil)
9
+ ts_var = "THREATSTACK_#{name}"
10
+ bf_var = "BLUEFYRE_#{name}"
11
+ bf_or_default = ENV.has_key?(bf_var) ? ENV[bf_var] : default
12
+ ENV.has_key?(ts_var) ? ENV[ts_var] : bf_or_default
13
+ end
14
+
15
+ def self.is_truthy(name, default = false)
16
+ ts_var = "THREATSTACK_#{name}"
17
+ bf_var = "BLUEFYRE_#{name}"
18
+ bf_or_default = ENV.has_key?(bf_var) ? ENV[bf_var] : default
19
+ val = ENV.has_key?(ts_var) ? ENV[ts_var] : bf_or_default
20
+ TRUTHY.include?(val.to_s.downcase)
21
+ end
22
+
23
+ TRUTHY = ['true', '1', 'yes'].freeze
24
+
25
+ # AGENT
26
+ RUBY = 'ruby'
27
+ AGENT_NAME = 'threatstack-agent-ruby'
28
+ ## main agent id
29
+ AGENT_ID = self.env('AGENT_ID', '')
30
+ ## autogenerated Id for this agent instance
31
+ AGENT_INSTANCE_ID = SecureRandom.uuid
32
+ ## whether or not the agent is disabled
33
+ DISABLED = self.is_truthy('DISABLED')
34
+ ## whether or not initialization is done manually by calling
35
+ MANUAL_INIT = self.is_truthy('MANUAL_INIT')
36
+ ## whether or not requests containing XSS payloads should be blocked
37
+ BLOCK_XSS = self.is_truthy('BLOCK_XSS')
38
+ ## whether or not requests containing SQLI payloads should be blocked
39
+ BLOCK_SQLI = self.is_truthy('BLOCK_SQLI')
40
+ ## specifies which user fields should be omitted from event payloads
41
+ DROP_FIELDS = self.env('DROP_FIELDS', false) ? self.env('DROP_FIELDS').split(',').each_with_object({}) do |val, h|
42
+ h[val] = true
43
+ end : nil
44
+ ## string to use when redacting fields
45
+ REDACTED = self.env('REDACTED', '#REDACTED#')
46
+
47
+ # EVENT SUBMITTER
48
+ ## event reporting frequency
49
+ JOB_INTERVAL = Integer(self.env('SUBMISSION_INTERVAL', 10))
50
+ ## max number of events per request
51
+ EVENTS_PER_REQ = Integer(self.env('EVENTS_PER_REQ', 1000))
52
+ ## base url
53
+ APPSEC_BASE_URL = self.env('API_COLLECTOR_URL', 'https://appsec-sensors.threatstack.com')
54
+ ## event collector path
55
+ APPSEC_EVENTS_URL = '/api/events'
56
+
57
+ # LOGGING
58
+ ## logging level threshold
59
+ LOG_LEVEL = self.env('LOG_LEVEL', 'UNKNOWN')
60
+ ## toggle color output for logging
61
+ LOG_COLORS = self.is_truthy('LOG_COLORS')
62
+
63
+ # AWS
64
+ AWS_METADATA_URL = self.env('AWS_METADATA_BASE_URL', 'http://169.254.169.254/latest/dynamic/instance-identity/document')
65
+
66
+ # EVENTS
67
+ INSTRUMENTATION = 'instrumentation'
68
+ DEPENDENCIES = 'dependencies'
69
+ ENVIRONMENT = 'environment'
70
+ ATTACK = 'attack'
71
+
72
+ # IP
73
+ IPV4 = 'IPv4'
74
+ IPV6 = 'IPv6'
75
+
76
+ # Strings
77
+ XSS = 'xss'
78
+ SQLI = 'sqli'
79
+ REQUEST_BLOCKED = 'Request blocked'
80
+ DETECTED_NOT_BLOCKED = 'Detected not blocked'
81
+ CGI_VARIABLES = Set.new(%w[ AUTH_TYPE CONTENT_LENGTH CONTENT_TYPE GATEWAY_INTERFACE HTTPS PATH_INFO
82
+ PATH_TRANSLATED REMOTE_ADDR REMOTE_HOST REMOTE_IDENT REMOTE_USER
83
+ REQUEST_METHOD SCRIPT_NAME SERVER_NAMESERVER_PORT SERVER_PROTOCOL
84
+ SERVER_SOFTWARE]).freeze
85
+ end
86
+ end
87
+
88
+ require_relative './utils/logger'
89
+
90
+ module Threatstack
91
+ module Constants
92
+ spec = Gem.loaded_specs['threatstack-agent-ruby']
93
+ logger = Threatstack::Utils::TSLogger.create 'Constants'
94
+ logger.info """ Threatstack Ruby Agent Config
95
+ VERSION: #{spec.nil? || !spec.respond_to?(:version) ? 'N/A' : spec.version}
96
+ STATE: #{DISABLED ? 'Disabled' : 'Enabled'}
97
+ AGENT ID: #{AGENT_ID}
98
+ AGENT INSTANCE ID: #{AGENT_INSTANCE_ID}
99
+ APPSEC SENSOR URL: #{APPSEC_BASE_URL}
100
+ BLOCK SQLI: #{BLOCK_SQLI}
101
+ BLOCK XSS: #{BLOCK_XSS}
102
+ DROP FIELDS: #{DROP_FIELDS}
103
+ SUBMIT INTERVAL: #{JOB_INTERVAL}
104
+ EVENTS PER REQ: #{EVENTS_PER_REQ}
105
+ LOG LEVEL: #{LOG_LEVEL}
106
+ LOG COLORS: #{LOG_COLORS}
107
+ MANUAL INIT: #{MANUAL_INIT}
108
+ REDACTED TEXT: #{REDACTED}"""
109
+ end
110
+ end
data/lib/control.rb ADDED
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'thread'
4
+
5
+ require_relative './events/models/environment_event'
6
+ require_relative './events/models/dependency_event'
7
+ require_relative './instrumentation/rails'
8
+ require_relative './instrumentation/kernel'
9
+ require_relative './jobs/event_submitter'
10
+ require_relative './jobs/delayed_job'
11
+ require_relative './utils/logger'
12
+ require_relative './constants'
13
+
14
+ module Threatstack
15
+ module Control
16
+ include Threatstack::Constants
17
+
18
+ @@agent_initialized = false
19
+ @@agent_init_mutex = Mutex.new
20
+
21
+ def self.init
22
+ @@agent_init_mutex.synchronize do
23
+ return if @@agent_initialized
24
+ @@agent_initialized = true
25
+ end
26
+ logger = Threatstack::Utils::TSLogger.create 'MainAgent'
27
+ logger.info 'Initializing Threatstack Ruby agent'
28
+
29
+ # patch Rails ActionController
30
+ logger.info 'Instrumenting Rails...'
31
+ Threatstack::Instrumentation::TSRails.patch_action_controller
32
+ logger.info 'Done instrumenting Rails'
33
+
34
+ # patch Kernel methods
35
+ logger.info 'Instrumenting Kernel methods...'
36
+ Threatstack::Instrumentation::TSKernel.wrap_methods
37
+ logger.info 'Done instrumenting Kernel methods'
38
+
39
+ # Start EventSubmitter asynchronously
40
+ logger.info 'Starting Event Submitter...'
41
+ Threatstack::Jobs::EventSubmitter.instance.start
42
+ logger.info 'Started Event Submitter'
43
+
44
+ # Gather environment and dependency info asynchronously
45
+ Threatstack::Jobs::DelayedJob.new(logger, 5) do
46
+ dep_event = Threatstack::Events::DependencyEvent.new
47
+ # submit dependency event
48
+ Threatstack::Jobs::EventSubmitter.instance.queue_event dep_event
49
+ # submit environment event
50
+ Threatstack::Jobs::EventSubmitter.instance.queue_event Threatstack::Events::EnvironmentEvent.new
51
+ end
52
+
53
+ logger.info 'Initialization done for agent'
54
+ end
55
+
56
+ def self._setup_agent
57
+ # initialize agent unless DISABLED or set to MANUAL_INIT
58
+ self.init unless DISABLED || MANUAL_INIT
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'singleton'
4
+
5
+ require_relative '../utils/logger'
6
+
7
+ module Threatstack
8
+ module Events
9
+ # Singleton class that handles temporarily caching events until they're sent to the back end
10
+ class EventAccumulator
11
+ include Singleton
12
+
13
+ attr_reader :events
14
+
15
+ def initialize
16
+ @events = []
17
+ @logger = Threatstack::Utils::TSLogger.create 'EventAccumulator'
18
+ end
19
+
20
+ def add_event(event)
21
+ @logger.debug "Adding event - New Total: #{@events.length + 1}"
22
+ @events.push(event)
23
+ end
24
+
25
+ def remove_events(num = 1)
26
+ @logger.debug "Removing #{num} event(s) - New Total: #{@events.length - num}"
27
+ @events.shift(num)
28
+ end
29
+
30
+ def clear_events
31
+ @logger.debug "Clearing events - Total: #{@events.length}"
32
+ @events.clear
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative './base_event'
4
+ require_relative '../../constants'
5
+
6
+ module Threatstack
7
+ module Events
8
+ # Instrumentation event model that inherits the common attributes and adds its own specifics
9
+ class AttackEvent < BaseEvent
10
+ attr_accessor :request_ip
11
+ attr_accessor :request_headers
12
+ attr_accessor :request_url
13
+ attr_accessor :request_method
14
+ attr_accessor :module_name
15
+ attr_accessor :attack_message
16
+ attr_accessor :attack_stack
17
+ attr_accessor :attack_details
18
+
19
+ # @param [Hash] args
20
+ # [String] args.event_id
21
+ # [String] args.timestamp
22
+ # [String] args.request_ip
23
+ # [Hash] args.request_headers
24
+ # [String] args.request_url
25
+ # [String] args.request_method
26
+ # [String] args.module_name
27
+ # [String] args.attack_message
28
+ # [String] args.attack_stack
29
+ # [Hash] args.attack_details
30
+ def initialize(args)
31
+ args[:event_type] = Threatstack::Constants::ATTACK
32
+ @request_ip = args[:request_ip]
33
+ @request_headers = args[:request_headers]
34
+ @request_url = args[:request_url]
35
+ @request_method = args[:request_method]
36
+ @module_name = args[:module_name]
37
+ @attack_message = args[:attack_message]
38
+ @attack_stack = args[:attack_stack]
39
+ @attack_details = args[:attack_details]
40
+ super args
41
+ end
42
+
43
+ def to_hash
44
+ hash = to_core_hash
45
+ hash[:module_name] = @module_name
46
+ hash[:payload] = {
47
+ :attack => {
48
+ :message => @attack_message, :stack => @attack_stack, :details => @attack_details
49
+ },
50
+ :req => {
51
+ :headers => @request_headers, :ip_address => @request_ip, :url => @request_url, :method => @request_method
52
+ }
53
+ }
54
+ hash
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'securerandom'
5
+
6
+ require_relative '../../constants'
7
+
8
+ module Threatstack
9
+ module Events
10
+ # Base event model containing the common attributes
11
+ class BaseEvent
12
+ attr_accessor :event_id
13
+ attr_accessor :event_type
14
+ attr_accessor :timestamp
15
+ attr_reader :agent_type
16
+
17
+ # @param [Hash] args
18
+ # [String] args.event_id
19
+ # [String] args.event_type
20
+ # [String] args.timestamp
21
+ def initialize(args)
22
+ @event_id = args[:event_id].nil? ? SecureRandom.uuid : args[:event_id]
23
+ @timestamp = args[:timestamp].nil? ? Time.now.utc.strftime('%FT%T.%3NZ') : args[:timestamp]
24
+ @event_type = args[:event_type]
25
+ @agent_type = Threatstack::Constants::RUBY
26
+ end
27
+
28
+ def to_hash
29
+ Hash[instance_variables.map { |name| [name[1..-1], instance_variable_get(name)] }]
30
+ end
31
+
32
+ def to_core_hash
33
+ { :event_id => @event_id, :event_type => @event_type, :agent_type => @agent_type, :timestamp => @timestamp }
34
+ end
35
+
36
+ def to_json_string
37
+ to_hash.to_json
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler'
4
+
5
+ require_relative './base_event'
6
+ require_relative '../../constants'
7
+ require_relative '../../utils/logger'
8
+
9
+ module Threatstack
10
+ module Events
11
+ # Dependency event model that inherits the common attributes and adds its own specifics
12
+ class DependencyEvent < BaseEvent
13
+ include Threatstack::Constants
14
+
15
+ attr_accessor :name
16
+ attr_accessor :dependencies
17
+ attr_accessor :graph
18
+
19
+ # @param [Hash] args
20
+ # [String] args.event_id
21
+ # [String] args.timestamp
22
+ def initialize(args = {})
23
+ logger = Threatstack::Utils::TSLogger.create 'DependencyEvent'
24
+ logger.debug 'Creating dependency event...'
25
+ args[:event_type] = DEPENDENCIES
26
+ begin
27
+ root_dir = self.app_root_dir
28
+ gemfile_path = File.join(root_dir, 'Gemfile')
29
+ lockfile_path = File.join(root_dir, 'Gemfile.lock')
30
+ logger.debug "Root Dir: #{root_dir}"
31
+ @name = File.basename root_dir
32
+ # build dependency list
33
+ @dependencies = Bundler::Definition.build(gemfile_path, lockfile_path, nil).
34
+ dependencies.each_with_object({}) do |dep, obj|
35
+ dep.groups.each do
36
+ begin
37
+ obj[dep.name] = dep.to_spec.version.to_s
38
+ rescue ScriptError => sce
39
+ logger.error "ScriptError: #{sce.inspect}"
40
+ rescue StandardError => ste
41
+ logger.error "StandardError: #{ste.inspect}"
42
+ end
43
+ end
44
+ end
45
+ # build dependency graph
46
+ @graph = Gem.loaded_specs.each_with_object({}) do |(k, v), obj|
47
+ requires = v.dependencies.each_with_object({}) do |dep, h|
48
+ h[dep.name] = dep.requirement.requirements.join
49
+ end
50
+ begin
51
+ obj[k] = { :version => v.version.to_s, :requires => requires }
52
+ rescue ScriptError => sce
53
+ logger.error "ScriptError: #{sce.inspect}"
54
+ rescue StandardError => ste
55
+ logger.error "StandardError: #{ste.inspect}"
56
+ end
57
+ end if Gem.respond_to? :loaded_specs
58
+ logger.debug "Name: #{@name}"
59
+ logger.debug "Dependencies: #{@dependencies}"
60
+ logger.debug "Graph: #{@graph}"
61
+ rescue ScriptError => sce
62
+ logger.error "General ScriptError: #{sce.inspect}"
63
+ rescue StandardError => ste
64
+ logger.error "General StandardError: #{ste.inspect}"
65
+ end
66
+ super args
67
+ end
68
+
69
+ # Returns the root directory of the currently running app
70
+ def app_root_dir
71
+ return Bundler.root if defined?(Bundler)
72
+
73
+ return ENV['RAILS_ROOT'] if defined?(ENV['RAILS_ROOT']) && ENV['RAILS_ROOT'].to_s.strip.length != 0
74
+
75
+ return Rails.root if defined?(Rails) && Rails.root.to_s.strip.length != 0
76
+
77
+ Dir.pwd
78
+ end
79
+
80
+ def to_hash
81
+ hash = to_core_hash
82
+ hash[:module_name] = AGENT_NAME
83
+ hash[:package_type] = RUBY
84
+ hash[:payload] = {
85
+ :name => @name,
86
+ :dependencies => @dependencies,
87
+ :graph => @graph
88
+ }
89
+ hash
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'socket'
4
+ require 'platform'
5
+ require 'network_interface'
6
+
7
+ require_relative './base_event'
8
+ require_relative '../../constants'
9
+ require_relative '../../utils/aws_utils'
10
+ require_relative '../../utils/logger'
11
+
12
+ module Threatstack
13
+ module Events
14
+
15
+ # Environment event model that inherits the common attributes and adds its own specifics
16
+ class EnvironmentEvent < BaseEvent
17
+ include Threatstack::Constants
18
+
19
+ attr_accessor :os_type
20
+ attr_accessor :os_platform
21
+ attr_accessor :os_arch
22
+ attr_accessor :hostname
23
+ attr_accessor :metadata
24
+ attr_accessor :interfaces
25
+ attr_accessor :versions
26
+
27
+ # @param [Hash] args
28
+ # [String] args.event_id
29
+ # [String] args.timestamp
30
+ def initialize(args = {})
31
+ logger = Threatstack::Utils::TSLogger.create 'EnvironmentEvent'
32
+ logger.debug 'Creating environment event...'
33
+ args[:event_type] = ENVIRONMENT
34
+ @os_type = Platform::IMPL.to_s
35
+ @os_platform = Platform::OS.to_s
36
+ @os_arch = Platform::ARCH.to_s
37
+ @hostname = Socket.gethostname
38
+ @versions = { :ruby => RUBY_VERSION }
39
+ @metadata = Threatstack::Utils::Aws.instance.get_aws_metadata
40
+ # interface type constants
41
+ af_link = NetworkInterface::AF_LINK
42
+ af_inet = NetworkInterface::AF_INET
43
+ af_inet6 = NetworkInterface::AF_INET6
44
+ # build interface hash
45
+ @interfaces = NetworkInterface.interfaces.each_with_object({}) do |iname, hash|
46
+ addresses = NetworkInterface.addresses(iname)
47
+ to_keep = []
48
+ mac = nil
49
+ # get mac address if any
50
+ if addresses[af_link] && !addresses[af_link].empty?
51
+ addr = addresses[af_link][0]
52
+ mac = addr['addr'] if !addr['addr'].nil? && !addr['addr'].empty?
53
+ end
54
+ # get IPv4 addresses if any
55
+ if addresses[af_inet] && !addresses[af_inet].empty?
56
+ addresses[af_inet].each do |addr|
57
+ next unless addr['addr'] && !addr['addr'].empty?
58
+
59
+ to_keep.push(:address => addr['addr'], :netmask => addr['netmask'], :family => IPV4, :mac => mac)
60
+ end
61
+ end
62
+ # get IPv6 addresses if any
63
+ if addresses[af_inet6] && !addresses[af_inet6].empty?
64
+ addresses[af_inet6].each do |addr|
65
+ next unless addr['addr'] && !addr['addr'].empty?
66
+
67
+ to_keep.push(:address => addr['addr'], :netmask => addr['netmask'], :family => IPV6, :mac => mac)
68
+ end
69
+ end
70
+ # add interface entry unless no addresses were found
71
+ hash[iname] = to_keep unless to_keep.empty?
72
+ end
73
+ super args
74
+ end
75
+
76
+ def to_hash
77
+ hash = to_core_hash
78
+ hash[:module_name] = AGENT_NAME
79
+ hash[:payload] = {
80
+ :hostname => @hostname,
81
+ :os => {
82
+ :platform => @os_platform, :type => @os_type, :arch => @os_arch
83
+ },
84
+ :aws => @metadata,
85
+ :versions => @versions,
86
+ :network => @interfaces
87
+ }
88
+ hash
89
+ end
90
+ end
91
+
92
+ end
93
+ end