newrelic_security 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/pr_ci.yml +2 -2
  3. data/CHANGELOG.md +82 -0
  4. data/THIRD_PARTY_NOTICES.md +8 -0
  5. data/lib/newrelic_security/agent/agent.rb +24 -4
  6. data/lib/newrelic_security/agent/configuration/manager.rb +51 -8
  7. data/lib/newrelic_security/agent/control/app_info.rb +2 -0
  8. data/lib/newrelic_security/agent/control/application_runtime_error.rb +95 -0
  9. data/lib/newrelic_security/agent/control/application_url_mappings.rb +2 -0
  10. data/lib/newrelic_security/agent/control/collector.rb +34 -3
  11. data/lib/newrelic_security/agent/control/control_command.rb +2 -3
  12. data/lib/newrelic_security/agent/control/critical_message.rb +2 -0
  13. data/lib/newrelic_security/agent/control/error_reporting.rb +74 -0
  14. data/lib/newrelic_security/agent/control/event.rb +26 -4
  15. data/lib/newrelic_security/agent/control/event_processor.rb +26 -0
  16. data/lib/newrelic_security/agent/control/event_subscriber.rb +2 -8
  17. data/lib/newrelic_security/agent/control/exit_event.rb +2 -0
  18. data/lib/newrelic_security/agent/control/grpc_context.rb +2 -1
  19. data/lib/newrelic_security/agent/control/health_check.rb +5 -0
  20. data/lib/newrelic_security/agent/control/http_context.rb +16 -7
  21. data/lib/newrelic_security/agent/control/iast_client.rb +24 -11
  22. data/lib/newrelic_security/agent/control/iast_data_transfer_request.rb +2 -0
  23. data/lib/newrelic_security/agent/control/scan_scheduler.rb +77 -0
  24. data/lib/newrelic_security/agent/control/websocket_client.rb +23 -0
  25. data/lib/newrelic_security/agent/utils/agent_utils.rb +14 -9
  26. data/lib/newrelic_security/constants.rb +2 -2
  27. data/lib/newrelic_security/instrumentation-security/async-http/instrumentation.rb +2 -13
  28. data/lib/newrelic_security/instrumentation-security/curb/instrumentation.rb +1 -14
  29. data/lib/newrelic_security/instrumentation-security/ethon/chain.rb +0 -6
  30. data/lib/newrelic_security/instrumentation-security/ethon/instrumentation.rb +7 -42
  31. data/lib/newrelic_security/instrumentation-security/ethon/prepend.rb +0 -4
  32. data/lib/newrelic_security/instrumentation-security/excon/instrumentation.rb +3 -13
  33. data/lib/newrelic_security/instrumentation-security/grape/chain.rb +7 -2
  34. data/lib/newrelic_security/instrumentation-security/grape/instrumentation.rb +3 -1
  35. data/lib/newrelic_security/instrumentation-security/grape/prepend.rb +7 -1
  36. data/lib/newrelic_security/instrumentation-security/grpc/server/instrumentation.rb +3 -2
  37. data/lib/newrelic_security/instrumentation-security/httpclient/instrumentation.rb +4 -28
  38. data/lib/newrelic_security/instrumentation-security/httprb/instrumentation.rb +1 -12
  39. data/lib/newrelic_security/instrumentation-security/httpx/instrumentation.rb +1 -15
  40. data/lib/newrelic_security/instrumentation-security/instrumentation_utils.rb +0 -17
  41. data/lib/newrelic_security/instrumentation-security/net_http/instrumentation.rb +6 -23
  42. data/lib/newrelic_security/instrumentation-security/net_ldap/instrumentation.rb +1 -1
  43. data/lib/newrelic_security/instrumentation-security/padrino/chain.rb +17 -0
  44. data/lib/newrelic_security/instrumentation-security/padrino/instrumentation.rb +15 -2
  45. data/lib/newrelic_security/instrumentation-security/padrino/prepend.rb +12 -0
  46. data/lib/newrelic_security/instrumentation-security/patron/instrumentation.rb +2 -15
  47. data/lib/newrelic_security/instrumentation-security/rails/chain.rb +26 -2
  48. data/lib/newrelic_security/instrumentation-security/rails/instrumentation.rb +29 -3
  49. data/lib/newrelic_security/instrumentation-security/rails/prepend.rb +18 -0
  50. data/lib/newrelic_security/instrumentation-security/roda/chain.rb +7 -2
  51. data/lib/newrelic_security/instrumentation-security/roda/instrumentation.rb +3 -1
  52. data/lib/newrelic_security/instrumentation-security/roda/prepend.rb +7 -1
  53. data/lib/newrelic_security/instrumentation-security/sinatra/chain.rb +6 -0
  54. data/lib/newrelic_security/instrumentation-security/sinatra/instrumentation.rb +9 -0
  55. data/lib/newrelic_security/instrumentation-security/sinatra/prepend.rb +4 -0
  56. data/lib/newrelic_security/instrumentation-security/sqlite3/instrumentation.rb +4 -4
  57. data/lib/newrelic_security/newrelic-security-api/api.rb +1 -1
  58. data/lib/newrelic_security/parse-cron/cron_parser.rb +294 -0
  59. data/lib/newrelic_security/version.rb +1 -1
  60. data/newrelic_security.gemspec +1 -1
  61. metadata +8 -4
@@ -5,6 +5,7 @@ module NewRelic::Security
5
5
  module Control
6
6
  EVENT_QUEUE_SIZE = 10_000
7
7
  HEALTH_INTERVAL = 300
8
+ ERROR_REPORTING_INTERVAL = 30
8
9
 
9
10
  class EventProcessor
10
11
 
@@ -15,6 +16,7 @@ module NewRelic::Security
15
16
  @eventQ = ::SizedQueue.new(EVENT_QUEUE_SIZE)
16
17
  create_dequeue_threads
17
18
  create_keep_alive_thread
19
+ create_error_reporting_thread
18
20
  NewRelic::Security::Agent.init_logger.info "[STEP-5] => Security agent components started"
19
21
  end
20
22
 
@@ -46,6 +48,7 @@ module NewRelic::Security
46
48
  enqueue(event)
47
49
  if @first_event
48
50
  NewRelic::Security::Agent.init_logger.info "[STEP-8] => First event sent for validation. Security agent started successfully : #{event.to_json}"
51
+ NewRelic::Security::Agent.config.traffic_start_time = current_time_millis unless NewRelic::Security::Agent.config[:traffic_start_time]
49
52
  @first_event = false
50
53
  end
51
54
  event = nil
@@ -128,6 +131,29 @@ module NewRelic::Security
128
131
  NewRelic::Security::Agent.logger.error "Exception in health check thread, #{exception.inspect}"
129
132
  end
130
133
 
134
+ def current_time_millis
135
+ (Time.now.to_f * 1000).to_i
136
+ end
137
+
138
+ def create_error_reporting_thread
139
+ @error_reporting_thread = Thread.new {
140
+ Thread.current.name = "newrelic_security_error_reporting_thread"
141
+ while true do
142
+ sleep ERROR_REPORTING_INTERVAL
143
+ send_error_reporting_event if NewRelic::Security::Agent::Control::WebsocketClient.instance.is_open?
144
+ end
145
+ }
146
+ rescue Exception => exception
147
+ NewRelic::Security::Agent.logger.error "Exception in error_reporting thread, #{exception.inspect}"
148
+ end
149
+
150
+ def send_error_reporting_event
151
+ NewRelic::Security::Agent.agent.error_reporting&.exceptions_map&.each_value do |exception|
152
+ NewRelic::Security::Agent::Control::WebsocketClient.instance.send(exception)
153
+ end
154
+ NewRelic::Security::Agent.agent.error_reporting&.exceptions_map&.clear
155
+ end
156
+
131
157
  end
132
158
  end
133
159
  end
@@ -7,19 +7,13 @@ module NewRelic::Security
7
7
  NewRelic::Security::Agent.logger.info "NewRelic server_source_configuration_added for pid : #{Process.pid}, Parent Pid : #{Process.ppid}"
8
8
  NewRelic::Security::Agent.init_logger.info "NewRelic server_source_configuration_added for pid : #{Process.pid}, Parent Pid : #{Process.ppid}"
9
9
  NewRelic::Security::Agent.config.update_server_config
10
- if NewRelic::Security::Agent.config[:enabled] && !NewRelic::Security::Agent.config[:high_security]
11
- NewRelic::Security::Agent.agent.init
10
+ if NewRelic::Security::Agent.config[:'security.enabled'] && !NewRelic::Security::Agent.config[:high_security]
11
+ Thread.new { NewRelic::Security::Agent.agent.scan_scheduler.init_via_scan_scheduler }
12
12
  else
13
13
  NewRelic::Security::Agent.logger.info "New Relic Security is disabled by one of the user provided config `security.enabled` or `high_security`."
14
14
  NewRelic::Security::Agent.init_logger.info "New Relic Security is disabled by one of the user provided config `security.enabled` or `high_security`."
15
15
  end
16
16
  }
17
- ::NewRelic::Agent.instance.events.subscribe(:security_policy_received) { |received_policy|
18
- NewRelic::Security::Agent.logger.info "security_policy_received pid ::::::: #{Process.pid} #{Process.ppid}, #{received_policy}"
19
- NewRelic::Security::Agent.config[:policy].merge!(received_policy)
20
- NewRelic::Security::Agent.init_logger.info "[STEP-7] => Received and applied policy/configuration : #{received_policy}"
21
- NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
22
- }
23
17
  end
24
18
  end
25
19
  end
@@ -14,6 +14,8 @@ module NewRelic::Security
14
14
  @collectorVersion = NewRelic::Security::VERSION
15
15
  @buildNumber = nil
16
16
  @applicationUUID = NewRelic::Security::Agent.config[:uuid]
17
+ @appAccountId = NewRelic::Security::Agent.config[:account_id]
18
+ @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
17
19
  @groupName = NewRelic::Security::Agent.config[:mode]
18
20
  @jsonVersion = NewRelic::Security::Agent.config[:json_version]
19
21
  @policyVersion = nil
@@ -8,7 +8,7 @@ module NewRelic::Security
8
8
 
9
9
  class GRPCContext
10
10
 
11
- attr_accessor :time_stamp, :method, :headers, :body, :route, :cache, :fuzz_files, :url, :server_name, :server_port, :client_ip, :client_port, :is_grpc, :metadata, :event_counter
11
+ attr_accessor :time_stamp, :method, :headers, :body, :route, :cache, :fuzz_files, :url, :server_name, :server_port, :client_ip, :client_port, :is_grpc, :metadata, :event_counter, :mutex
12
12
 
13
13
  def initialize(grpc_request)
14
14
  @time_stamp = current_time_millis
@@ -32,6 +32,7 @@ module NewRelic::Security
32
32
  @cache = {}
33
33
  @fuzz_files = ::Set.new
34
34
  @event_counter = 0
35
+ @mutex = Mutex.new
35
36
  NewRelic::Security::Agent.agent.http_request_count.increment
36
37
  end
37
38
 
@@ -18,6 +18,8 @@ module NewRelic::Security
18
18
  @framework = NewRelic::Security::Agent.config[:framework]
19
19
  @protectedServer = nil
20
20
  @applicationUUID = NewRelic::Security::Agent.config[:uuid]
21
+ @appAccountId = NewRelic::Security::Agent.config[:account_id]
22
+ @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
21
23
  @collectorVersion = NewRelic::Security::VERSION
22
24
  @buildNumber = nil
23
25
  @jsonVersion = NewRelic::Security::Agent.config[:json_version]
@@ -33,6 +35,9 @@ module NewRelic::Security
33
35
  @iastEventStats = {}
34
36
  @raspEventStats = {}
35
37
  @exitEventStats = {}
38
+ @procStartTime = NewRelic::Security::Agent.config[:process_start_time]
39
+ @trafficStartedTime = NewRelic::Security::Agent.config[:traffic_start_time]
40
+ @scanStartTime = NewRelic::Security::Agent.config[:scan_start_time]
36
41
  end
37
42
 
38
43
  def as_json
@@ -10,18 +10,22 @@ module NewRelic::Security
10
10
  UNDERSCORE = '_'
11
11
  HYPHEN = '-'
12
12
  REQUEST_METHOD = 'REQUEST_METHOD'
13
+ HTTP_HOST = 'HTTP_HOST'
13
14
  PATH_INFO = 'PATH_INFO'
15
+ QUERY_STRING = 'QUERY_STRING'
14
16
  RACK_INPUT = 'rack.input'
15
- CGI_VARIABLES = ::Set.new(%W[ AUTH_TYPE CONTENT_LENGTH CONTENT_TYPE GATEWAY_INTERFACE HTTPS PATH_INFO PATH_TRANSLATED REQUEST_URI QUERY_STRING REMOTE_ADDR REMOTE_HOST REMOTE_IDENT REMOTE_USER REQUEST_METHOD SCRIPT_NAME SERVER_NAME SERVER_PORT SERVER_PROTOCOL SERVER_SOFTWARE rack.url_scheme ])
17
+ CGI_VARIABLES = ::Set.new(%W[ AUTH_TYPE CONTENT_LENGTH CONTENT_TYPE GATEWAY_INTERFACE HTTPS HTTP_HOST PATH_INFO PATH_TRANSLATED REQUEST_URI QUERY_STRING REMOTE_ADDR REMOTE_HOST REMOTE_IDENT REMOTE_USER REQUEST_METHOD SCRIPT_NAME SERVER_NAME SERVER_PORT SERVER_PROTOCOL SERVER_SOFTWARE rack.url_scheme ])
18
+ REQUEST_BODY_LIMIT = 500 #KB
16
19
 
17
20
  class HTTPContext
18
21
 
19
- attr_accessor :time_stamp, :req, :method, :headers, :params, :body, :data_truncated, :route, :cache, :fuzz_files, :event_counter
22
+ attr_accessor :time_stamp, :req, :method, :headers, :params, :body, :data_truncated, :route, :cache, :fuzz_files, :event_counter, :mutex, :url
20
23
 
21
24
  def initialize(env)
22
25
  @time_stamp = current_time_millis
23
26
  @req = env.select { |key, _| CGI_VARIABLES.include? key}
24
27
  @method = @req[REQUEST_METHOD]
28
+ @url = "#{@req[PATH_INFO]}?#{@req[QUERY_STRING]}"
25
29
  @headers = env.select { |key, _| key.include?(HTTP_) }
26
30
  @headers = @headers.transform_keys{ |key| key[5..-1].gsub(UNDERSCORE, HYPHEN).downcase }
27
31
  request = Rack::Request.new(env) unless env.empty?
@@ -30,23 +34,24 @@ module NewRelic::Security
30
34
  strio = env[RACK_INPUT]
31
35
  if strio.instance_of?(::StringIO)
32
36
  offset = strio.tell
33
- @body = strio.read(NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024) #after read, offset changes
37
+ @body = strio.read(REQUEST_BODY_LIMIT * 1024) #after read, offset changes
34
38
  strio.seek(offset)
35
39
  # In case of Grape and Roda strio.read giving empty result, added below approach to handle such cases
36
40
  @body = strio.string if @body.nil? && strio.size > 0
37
41
  elsif defined?(::Rack) && defined?(::Rack::Lint::InputWrapper) && strio.instance_of?(::Rack::Lint::InputWrapper)
38
- @body = strio.read(NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024)
42
+ @body = strio.read(REQUEST_BODY_LIMIT * 1024)
39
43
  elsif defined?(::Protocol::Rack::Input) && defined?(::Protocol::Rack::Input) && strio.instance_of?(::Protocol::Rack::Input)
40
- @body = strio.read(NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024)
44
+ @body = strio.read(REQUEST_BODY_LIMIT * 1024)
41
45
  elsif defined?(::PhusionPassenger::Utils::TeeInput) && strio.instance_of?(::PhusionPassenger::Utils::TeeInput)
42
- @body = strio.read(NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024)
46
+ @body = strio.read(REQUEST_BODY_LIMIT * 1024)
43
47
  end
44
- @data_truncated = @body && @body.size >= NewRelic::Security::Agent.config[:'security.request.body_limit'] * 1024
48
+ @data_truncated = @body && @body.size >= REQUEST_BODY_LIMIT * 1024
45
49
  strio&.rewind
46
50
  @body = @body.force_encoding(Encoding::UTF_8) if @body.is_a?(String)
47
51
  @cache = Hash.new
48
52
  @fuzz_files = ::Set.new
49
53
  @event_counter = 0
54
+ @mutex = Mutex.new
50
55
  NewRelic::Security::Agent.agent.http_request_count.increment
51
56
  NewRelic::Security::Agent.agent.iast_client.completed_requests[@headers[NR_CSEC_PARENT_ID]] = [] if @headers.key?(NR_CSEC_PARENT_ID)
52
57
  end
@@ -66,6 +71,10 @@ module NewRelic::Security
66
71
  def self.reset_context
67
72
  ::NewRelic::Agent::Tracer.current_transaction.remove_instance_variable(:@security_context_data) if ::NewRelic::Agent::Tracer.current_transaction.instance_variable_defined?(:@security_context_data)
68
73
  end
74
+
75
+ def self.get_current_transaction
76
+ ::NewRelic::Agent::Tracer.current_transaction
77
+ end
69
78
  end
70
79
 
71
80
  end
@@ -17,9 +17,8 @@ module NewRelic::Security
17
17
  IS_GRPC = 'isGrpc'
18
18
  INPUT_CLASS = 'inputClass'
19
19
  SERVER_PORT_1 = 'serverPort'
20
- PROBING = 'probing'
21
- INTERVAL = 'interval'
22
20
  IS_GRPC_CLIENT_STREAM = 'isGrpcClientStream'
21
+ PROBING_INTERVAL = 5
23
22
 
24
23
  class IASTClient
25
24
 
@@ -54,6 +53,7 @@ module NewRelic::Security
54
53
  Thread.current.name = "newrelic_security_iast_thread-#{t}"
55
54
  loop do
56
55
  fuzz_request = @fuzzQ.deq #thread blocks when the queue is empty
56
+ NewRelic::Security::Agent.config.scan_start_time = current_time_millis unless NewRelic::Security::Agent.config[:scan_start_time]
57
57
  if fuzz_request.request[IS_GRPC]
58
58
  fire_grpc_request(fuzz_request.id, fuzz_request.request, fuzz_request.reflected_metadata)
59
59
  else
@@ -71,7 +71,8 @@ module NewRelic::Security
71
71
  @iast_data_transfer_request_processor_thread = Thread.new do
72
72
  Thread.current.name = "newrelic_security_iast_data_transfer_request_processor"
73
73
  loop do
74
- sleep NewRelic::Security::Agent.config[:policy][VULNERABILITY_SCAN][IAST_SCAN][PROBING][INTERVAL]
74
+ # TODO: Check & remove this probing interval if not required, earlier this was used from policy sent by SE.
75
+ sleep PROBING_INTERVAL
75
76
  current_timestamp = current_time_millis
76
77
  cooldown_sleep_time = @cooldown_till_timestamp - current_timestamp
77
78
  sleep cooldown_sleep_time/1000 if cooldown_sleep_time > 0
@@ -84,9 +85,21 @@ module NewRelic::Security
84
85
  if batch_size > 100 && remaining_record_capacity > batch_size
85
86
  iast_data_transfer_request = NewRelic::Security::Agent::Control::IASTDataTransferRequest.new
86
87
  iast_data_transfer_request.batchSize = batch_size * 2
88
+ # TODO: Below calculation of batch_size overrides above logic and can be removed once below one is stablises or rate limit feature is released.
89
+ if NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit']
90
+ batch_size =
91
+ if NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] < 12
92
+ 1
93
+ elsif NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] > 3600
94
+ 300
95
+ else
96
+ NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] / 12
97
+ end
98
+ iast_data_transfer_request.batchSize = batch_size
99
+ end
87
100
  iast_data_transfer_request.pendingRequestIds = pending_request_ids.to_a
88
101
  iast_data_transfer_request.completedRequests = completed_requests
89
- NewRelic::Security::Agent.agent.event_processor.send_iast_data_transfer_request(iast_data_transfer_request)
102
+ NewRelic::Security::Agent.agent.event_processor&.send_iast_data_transfer_request(iast_data_transfer_request)
90
103
  end
91
104
  end
92
105
  end
@@ -122,18 +135,18 @@ module NewRelic::Security
122
135
  def fire_grpc_request(fuzz_request_id, request, reflected_metadata)
123
136
  service = Object.const_get(request[METHOD].split(SLASH)[0]).superclass
124
137
  method = request[METHOD].split(SLASH)[1]
125
- @stub = service.rpc_stub_class.new("localhost:#{request[SERVER_PORT_1]}", :this_channel_is_insecure) unless @stub
138
+ @stub ||= service.rpc_stub_class.new("localhost:#{request[SERVER_PORT_1]}", :this_channel_is_insecure)
126
139
 
127
- parsed_body = request[BODY][1..-2].split(',')
128
- if reflected_metadata[IS_GRPC_CLIENT_STREAM]
129
- chunks_enum = Enumerator.new do |y|
140
+ parsed_body = request[BODY][1..-2].split(',')
141
+ chunks_enum = if reflected_metadata[IS_GRPC_CLIENT_STREAM]
142
+ Enumerator.new do |y|
130
143
  parsed_body.each do |b|
131
144
  y << Object.const_get(reflected_metadata[INPUT_CLASS]).decode_json(b)
132
145
  end
133
146
  end
134
- else
135
- chunks_enum = Object.const_get(reflected_metadata[INPUT_CLASS]).decode_json(request[BODY])
136
- end
147
+ else
148
+ Object.const_get(reflected_metadata[INPUT_CLASS]).decode_json(request[BODY])
149
+ end
137
150
  response = @stub.public_send(method, chunks_enum, metadata: request[HEADERS])
138
151
  # response = @stub.send(method, JSON.parse(request['body'], object_class: OpenStruct))
139
152
  # request[HEADERS].delete(VERSION) if request[HEADERS].key?(VERSION)
@@ -11,6 +11,8 @@ module NewRelic::Security
11
11
  def initialize
12
12
  @jsonName = :'iast-data-request'
13
13
  @applicationUUID = NewRelic::Security::Agent.config[:uuid]
14
+ @appAccountId = NewRelic::Security::Agent.config[:account_id]
15
+ @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
14
16
  @batchSize = 10
15
17
  @pendingRequestIds = []
16
18
  @completedRequests = Hash.new
@@ -0,0 +1,77 @@
1
+ require 'newrelic_security/parse-cron/cron_parser'
2
+
3
+ module NewRelic::Security
4
+ module Agent
5
+ module Control
6
+ class ScanScheduler
7
+ def init_via_scan_scheduler
8
+ if NewRelic::Security::Agent.config[:'security.scan_schedule.delay'].positive?
9
+ NewRelic::Security::Agent.logger.info "IAST delay is set to: #{NewRelic::Security::Agent.config[:'security.scan_schedule.delay']}, current time: #{Time.now}"
10
+ start_agent_with_delay(NewRelic::Security::Agent.config[:'security.scan_schedule.delay']*60)
11
+ elsif !NewRelic::Security::Agent.config[:'security.scan_schedule.schedule'].to_s.empty?
12
+ cron_expression_task(NewRelic::Security::Agent.config[:'security.scan_schedule.schedule'], NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
13
+ else
14
+ NewRelic::Security::Agent.agent.init
15
+ NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
16
+ shutdown_at_duration_reached(NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
17
+ end
18
+ rescue StandardError => exception
19
+ NewRelic::Security::Agent.logger.error "Exception in IAST scan scheduler: #{exception.inspect} #{exception.backtrace}"
20
+ ::NewRelic::Agent.notice_error(exception)
21
+ end
22
+
23
+ def start_agent_with_delay(delay)
24
+ NewRelic::Security::Agent.logger.info "Security Agent delay scan time is set to: #{(delay/60).ceil} minutes when always_sample_traces is #{NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']}, current time: #{Time.now}"
25
+ if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
26
+ NewRelic::Security::Agent.agent.init unless NewRelic::Security::Agent.config[:enabled]
27
+ sleep delay if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
28
+ else
29
+ sleep delay
30
+ NewRelic::Security::Agent.agent.init
31
+ end
32
+ NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
33
+ shutdown_at_duration_reached(NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
34
+ end
35
+
36
+ def shutdown_at_duration_reached(duration)
37
+ shutdown_at = Time.now.to_i + duration
38
+ shut_down_time = (Time.now + duration).strftime("%a %d %b %Y %H:%M:%S")
39
+ return if duration <= 0
40
+ NewRelic::Security::Agent.logger.info "IAST Duration is set to: #{duration/60} minutes, timestamp: #{shut_down_time} time, current time: #{Time.now}"
41
+ @shutdown_monitor_thread = Thread.new do
42
+ Thread.current.name = "newrelic_security_shutdown_monitor_thread"
43
+ loop do
44
+ sleep 1
45
+ next if Time.now.to_i < shutdown_at
46
+ if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
47
+ NewRelic::Security::Agent.logger.info "Shutdown IAST Data transfer request processor only as 'security.scan_schedule.always_sample_traces' is #{NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']} now at current time: #{Time.now}"
48
+ NewRelic::Security::Agent.agent.iast_client&.iast_data_transfer_request_processor_thread&.kill
49
+ else
50
+ NewRelic::Security::Agent.logger.info "Shutdown IAST agent now at current time: #{Time.now}"
51
+ ::NewRelic::Agent.notice_error(StandardError.new("WS Connection closed by local"))
52
+ NewRelic::Security::Agent.agent.shutdown_security_agent
53
+ end
54
+ break
55
+ end
56
+ end
57
+ end
58
+
59
+ def cron_expression_task(schedule, duration)
60
+ @cron_parser = NewRelic::Security::ParseCron::CronParser.new(schedule)
61
+ loop do
62
+ next_run = @cron_parser.next(Time.now)
63
+ NewRelic::Security::Agent.logger.info "Next init via cron exp: #{schedule}, is scheduled at : #{next_run}"
64
+ delay = next_run - Time.now
65
+ if NewRelic::Security::Agent.agent.iast_client&.iast_data_transfer_request_processor_thread&.alive?
66
+ sleep delay > 2 ? delay - 2 : delay
67
+ else
68
+ start_agent_with_delay(delay)
69
+ end
70
+ return if duration <= 0
71
+ end
72
+ end
73
+
74
+ end
75
+ end
76
+ end
77
+ end
@@ -21,6 +21,8 @@ module NewRelic::Security
21
21
  NR_CSEC_ENTITY_NAME = 'NR-CSEC-ENTITY-NAME'
22
22
  NR_CSEC_ENTITY_GUID = 'NR-CSEC-ENTITY-GUID'
23
23
  NR_CSEC_IAST_DATA_TRANSFER_MODE = 'NR-CSEC-IAST-DATA-TRANSFER-MODE'
24
+ NR_CSEC_IGNORED_VUL_CATEGORIES = 'NR-CSEC-IGNORED-VUL-CATEGORIES'
25
+ NR_CSEC_PROCESS_START_TIME = 'NR-CSEC-PROCESS-START-TIME'
24
26
 
25
27
  class WebsocketClient
26
28
  include Singleton
@@ -43,6 +45,8 @@ module NewRelic::Security
43
45
  headers[NR_CSEC_ENTITY_NAME] = NewRelic::Security::Agent.config[:app_name]
44
46
  headers[NR_CSEC_ENTITY_GUID] = NewRelic::Security::Agent.config[:entity_guid]
45
47
  headers[NR_CSEC_IAST_DATA_TRANSFER_MODE] = PULL
48
+ headers[NR_CSEC_IGNORED_VUL_CATEGORIES] = ingnored_vul_categories.join(COMMA)
49
+ headers[NR_CSEC_PROCESS_START_TIME] = NewRelic::Security::Agent.config[:process_start_time]
46
50
 
47
51
  begin
48
52
  cert_store = ::OpenSSL::X509::Store.new
@@ -76,21 +80,26 @@ module NewRelic::Security
76
80
 
77
81
  connection.on :error do |e|
78
82
  NewRelic::Security::Agent.logger.error "Error in websocket connection : #{e.inspect} #{e.backtrace}"
83
+ ::NewRelic::Agent.notice_error(e)
79
84
  Thread.new { NewRelic::Security::Agent::Control::WebsocketClient.instance.close(true) }
80
85
  end
81
86
  rescue Errno::EPIPE => exception
82
87
  NewRelic::Security::Agent.logger.error "Unable to connect to validator_service: #{exception.inspect}"
88
+ ::NewRelic::Agent.notice_error(exception)
83
89
  NewRelic::Security::Agent.config.disable_security
84
90
  rescue Errno::ECONNRESET => exception
85
91
  NewRelic::Security::Agent.logger.error "Unable to connect to validator_service: #{exception.inspect}"
92
+ ::NewRelic::Agent.notice_error(exception)
86
93
  NewRelic::Security::Agent.config.disable_security
87
94
  Thread.new { NewRelic::Security::Agent.agent.reconnect(15) }
88
95
  rescue Errno::ECONNREFUSED => exception
89
96
  NewRelic::Security::Agent.logger.error "Unable to connect to validator_service: #{exception.inspect}"
97
+ ::NewRelic::Agent.notice_error(exception)
90
98
  NewRelic::Security::Agent.config.disable_security
91
99
  Thread.new { NewRelic::Security::Agent.agent.reconnect(15) }
92
100
  rescue => exception
93
101
  NewRelic::Security::Agent.logger.error "Exception in websocket init: #{exception.inspect} #{exception.backtrace}"
102
+ ::NewRelic::Agent.notice_error(exception)
94
103
  NewRelic::Security::Agent.config.disable_security
95
104
  Thread.new { NewRelic::Security::Agent.agent.reconnect(15) }
96
105
  end
@@ -125,6 +134,20 @@ module NewRelic::Security
125
134
  false
126
135
  end
127
136
 
137
+ private
138
+
139
+ def ingnored_vul_categories
140
+ list = []
141
+ list << FILE_OPERATION << FILE_INTEGRITY if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access']
142
+ list << SQL_DB_COMMAND if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.sql_injection']
143
+ list << NOSQL_DB_COMMAND if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.nosql_injection']
144
+ list << LDAP if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ldap_injection']
145
+ list << SYSTEM_COMMAND if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.command_injection']
146
+ list << XPATH if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.xpath_injection']
147
+ list << HTTP_REQUEST if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ssrf']
148
+ list << REFLECTED_XSS if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.rxss']
149
+ list
150
+ end
128
151
  end
129
152
  end
130
153
  end
@@ -15,8 +15,7 @@ module NewRelic::Security
15
15
  ASTERISK = '*'
16
16
 
17
17
  def is_IAST?
18
- return false if NewRelic::Security::Agent.config[:policy].empty?
19
- return NewRelic::Security::Agent.config[:policy][VULNERABILITY_SCAN][IAST_SCAN][ENABLED] if NewRelic::Security::Agent.config[:policy][VULNERABILITY_SCAN][ENABLED]
18
+ return true if NewRelic::Security::Agent.config[:mode] == IAST
20
19
  false
21
20
  end
22
21
 
@@ -36,10 +35,11 @@ module NewRelic::Security
36
35
  decrypted_data.split(COMMA).each do |filename|
37
36
  begin
38
37
  filename.gsub!(NR_CSEC_VALIDATOR_HOME_TMP, NewRelic::Security::Agent.config[:fuzz_dir_path])
38
+ filename.gsub!(NR_CSEC_VALIDATOR_HOME_TMP_URL_ENCODED, NewRelic::Security::Agent.config[:fuzz_dir_path])
39
39
  filename.gsub!(NR_CSEC_VALIDATOR_FILE_SEPARATOR, ::File::SEPARATOR)
40
40
  dirname = ::File.dirname(filename)
41
41
  ::FileUtils.mkdir_p(dirname, :mode => 0770) unless ::File.directory?(dirname)
42
- ctxt&.fuzz_files << filename
42
+ ctxt&.fuzz_files&.<< filename
43
43
  ::File.open(filename, ::File::WRONLY | ::File::CREAT | ::File::EXCL) do |fd|
44
44
  # puts "Ownership acquired by : #{Process.pid}"
45
45
  end unless ::File.exist?(filename)
@@ -95,7 +95,8 @@ module NewRelic::Security
95
95
 
96
96
  def get_app_routes(framework, router = nil)
97
97
  enable_object_space_in_jruby
98
- if framework == :rails
98
+ case framework
99
+ when :rails
99
100
  ::Rails.application.routes.routes.each do |route|
100
101
  if route.verb.is_a?(::Regexp)
101
102
  method = route.verb.inspect.match(/[a-zA-Z]+/)
@@ -106,27 +107,31 @@ module NewRelic::Security
106
107
  }
107
108
  end
108
109
  end
109
- elsif framework == :sinatra
110
+ when :sinatra
110
111
  ::Sinatra::Application.routes.each do |method, routes|
111
112
  routes.map { |r| r.first.to_s }.map do |route|
112
113
  NewRelic::Security::Agent.agent.route_map << "#{method}@#{route}"
113
114
  end
114
115
  end
115
- elsif framework == :grape
116
+ when :grape
116
117
  ObjectSpace.each_object(::Grape::Endpoint) { |z|
117
118
  z.instance_variable_get(:@routes)&.each { |route|
118
- http_method = route.instance_variable_get(:@request_method) ? route.instance_variable_get(:@request_method) : route.instance_variable_get(:@options)[:method]
119
+ http_method = route.instance_variable_get(:@request_method) || route.instance_variable_get(:@options)[:method]
119
120
  NewRelic::Security::Agent.agent.route_map << "#{http_method}@#{route.pattern.origin}"
120
121
  }
121
122
  }
122
- elsif framework == :padrino
123
+ when :padrino
123
124
  if router.instance_of?(::Padrino::PathRouter::Router)
124
125
  router.instance_variable_get(:@routes).each do |route|
125
126
  NewRelic::Security::Agent.agent.route_map << "#{route.instance_variable_get(:@verb)}@#{route.matcher.instance_variable_get(:@path)}"
126
127
  end
127
128
  end
128
- elsif framework == :roda
129
+ when :roda
129
130
  NewRelic::Security::Agent.logger.warn "TODO: Roda is a routing tree web toolkit, which generates route dynamically, hence route extraction is not possible."
131
+ when :grpc
132
+ router.owner.superclass.public_instance_methods(false).each do |m|
133
+ NewRelic::Security::Agent.agent.route_map << "*@/#{router.owner}/#{m}"
134
+ end
130
135
  else
131
136
  NewRelic::Security::Agent.logger.error "Unable to get app routes as Framework not detected"
132
137
  end
@@ -7,6 +7,7 @@ module NewRelic::Security
7
7
  LANGUAGE_COLLECTOR = 'LANGUAGE_COLLECTOR'
8
8
  STANDARD_OUT = 'STDOUT'
9
9
  NR_CSEC_VALIDATOR_HOME_TMP = '{{NR_CSEC_VALIDATOR_HOME_TMP}}'
10
+ NR_CSEC_VALIDATOR_HOME_TMP_URL_ENCODED = '%7B%7BNR_CSEC_VALIDATOR_HOME_TMP%7D%7D'
10
11
  NR_CSEC_VALIDATOR_FILE_SEPARATOR = '{{NR_CSEC_VALIDATOR_FILE_SEPARATOR}}'
11
12
  SEC_HOME_PATH = 'nr-security-home'
12
13
  LOGS_DIR = 'logs'
@@ -16,6 +17,7 @@ module NewRelic::Security
16
17
  NR_CSEC_FUZZ_REQUEST_ID = 'nr-csec-fuzz-request-id'
17
18
  NR_CSEC_TRACING_DATA = 'nr-csec-tracing-data'
18
19
  NR_CSEC_PARENT_ID = 'nr-csec-parent-id'
20
+ IAST = 'IAST'
19
21
  COLON_IAST_COLON = ':IAST:'
20
22
  NOSQL_DB_COMMAND = 'NOSQL_DB_COMMAND'
21
23
  SQL_DB_COMMAND = 'SQL_DB_COMMAND'
@@ -62,6 +64,4 @@ module NewRelic::Security
62
64
  CONTENT_TYPE1 = 'content-Type'
63
65
  PULL = 'PULL'
64
66
  SHA1 = 'sha1'
65
- VULNERABILITY_SCAN = 'vulnerabilityScan'
66
- IAST_SCAN = 'iastScan'
67
67
  end
@@ -6,22 +6,11 @@ module NewRelic::Security
6
6
  module Instrumentation
7
7
  module AsyncHttp
8
8
 
9
- def call_on_enter(method, url, headers, body)
9
+ def call_on_enter(_method, url, headers, _body)
10
10
  event = nil
11
11
  NewRelic::Security::Agent.logger.debug "OnEnter : #{self.class}.#{__method__}"
12
- ob = {}
13
- ob[:Method] = method
14
12
  uri = ::URI.parse url
15
- ob[:scheme] = uri.scheme
16
- ob[:host] = uri.host
17
- ob[:port] = uri.port
18
- ob[:URI] = uri.to_s
19
- ob[:path] = uri.path
20
- ob[:query] = uri.query
21
- ob[:Body] = body.respond_to?(:join) ? body.join.to_s : body.to_s
22
- ob[:Headers] = headers.to_h
23
- ob.each { |_, value| value.dup.force_encoding(ISO_8859_1).encode(UTF_8) if value.is_a?(String) }
24
- event = NewRelic::Security::Agent::Control::Collector.collect(HTTP_REQUEST, [ob])
13
+ event = NewRelic::Security::Agent::Control::Collector.collect(HTTP_REQUEST, [uri.to_s])
25
14
  NewRelic::Security::Instrumentation::InstrumentationUtils.append_tracing_data(headers, event) if event
26
15
  event
27
16
  rescue => exception
@@ -12,20 +12,7 @@ module NewRelic::Security
12
12
  self.requests.each {
13
13
  |key, req|
14
14
  uri = NewRelic::Security::Instrumentation::InstrumentationUtils.parse_uri(req.url)
15
- ob = {}
16
- if uri
17
- ob[:Method] = nil
18
- ob[:scheme] = uri.scheme
19
- ob[:host] = uri.host
20
- ob[:port] = uri.port
21
- ob[:URI] = uri.to_s
22
- ob[:path] = uri.path
23
- ob[:query] = uri.query
24
- ob[:Body] = req.post_body
25
- ob[:Headers] = req.headers
26
- ob.each { |_, value| value.dup.force_encoding(ISO_8859_1).encode(UTF_8) if value.is_a?(String) }
27
- ic_args.push(ob)
28
- end
15
+ ic_args.push(uri.to_s) if uri
29
16
  }
30
17
  event = NewRelic::Security::Agent::Control::Collector.collect(HTTP_REQUEST, ic_args)
31
18
  self.requests.each { |key, req| NewRelic::Security::Instrumentation::InstrumentationUtils.add_tracing_data(req.headers, event) } if event
@@ -7,12 +7,6 @@ module NewRelic::Security
7
7
  ::Ethon::Easy.class_eval do
8
8
  include NewRelic::Security::Instrumentation::Ethon::Easy
9
9
 
10
- alias_method :fabricate_without_security, :fabricate
11
-
12
- def fabricate(url, action_name, options)
13
- fabricate_on_enter(url, action_name, options) { return fabricate_without_security(url, action_name, options) }
14
- end
15
-
16
10
  alias_method(:headers_equals_without_security, :headers=)
17
11
 
18
12
  def headers=(headers)