newrelic_security 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/pr_ci.yml +2 -2
- data/CHANGELOG.md +82 -0
- data/THIRD_PARTY_NOTICES.md +8 -0
- data/lib/newrelic_security/agent/agent.rb +24 -4
- data/lib/newrelic_security/agent/configuration/manager.rb +51 -8
- data/lib/newrelic_security/agent/control/app_info.rb +2 -0
- data/lib/newrelic_security/agent/control/application_runtime_error.rb +95 -0
- data/lib/newrelic_security/agent/control/application_url_mappings.rb +2 -0
- data/lib/newrelic_security/agent/control/collector.rb +34 -3
- data/lib/newrelic_security/agent/control/control_command.rb +2 -3
- data/lib/newrelic_security/agent/control/critical_message.rb +2 -0
- data/lib/newrelic_security/agent/control/error_reporting.rb +74 -0
- data/lib/newrelic_security/agent/control/event.rb +26 -4
- data/lib/newrelic_security/agent/control/event_processor.rb +26 -0
- data/lib/newrelic_security/agent/control/event_subscriber.rb +2 -8
- data/lib/newrelic_security/agent/control/exit_event.rb +2 -0
- data/lib/newrelic_security/agent/control/grpc_context.rb +2 -1
- data/lib/newrelic_security/agent/control/health_check.rb +5 -0
- data/lib/newrelic_security/agent/control/http_context.rb +16 -7
- data/lib/newrelic_security/agent/control/iast_client.rb +24 -11
- data/lib/newrelic_security/agent/control/iast_data_transfer_request.rb +2 -0
- data/lib/newrelic_security/agent/control/scan_scheduler.rb +77 -0
- data/lib/newrelic_security/agent/control/websocket_client.rb +23 -0
- data/lib/newrelic_security/agent/utils/agent_utils.rb +14 -9
- data/lib/newrelic_security/constants.rb +2 -2
- data/lib/newrelic_security/instrumentation-security/async-http/instrumentation.rb +2 -13
- data/lib/newrelic_security/instrumentation-security/curb/instrumentation.rb +1 -14
- data/lib/newrelic_security/instrumentation-security/ethon/chain.rb +0 -6
- data/lib/newrelic_security/instrumentation-security/ethon/instrumentation.rb +7 -42
- data/lib/newrelic_security/instrumentation-security/ethon/prepend.rb +0 -4
- data/lib/newrelic_security/instrumentation-security/excon/instrumentation.rb +3 -13
- data/lib/newrelic_security/instrumentation-security/grape/chain.rb +7 -2
- data/lib/newrelic_security/instrumentation-security/grape/instrumentation.rb +3 -1
- data/lib/newrelic_security/instrumentation-security/grape/prepend.rb +7 -1
- data/lib/newrelic_security/instrumentation-security/grpc/server/instrumentation.rb +3 -2
- data/lib/newrelic_security/instrumentation-security/httpclient/instrumentation.rb +4 -28
- data/lib/newrelic_security/instrumentation-security/httprb/instrumentation.rb +1 -12
- data/lib/newrelic_security/instrumentation-security/httpx/instrumentation.rb +1 -15
- data/lib/newrelic_security/instrumentation-security/instrumentation_utils.rb +0 -17
- data/lib/newrelic_security/instrumentation-security/net_http/instrumentation.rb +6 -23
- data/lib/newrelic_security/instrumentation-security/net_ldap/instrumentation.rb +1 -1
- data/lib/newrelic_security/instrumentation-security/padrino/chain.rb +17 -0
- data/lib/newrelic_security/instrumentation-security/padrino/instrumentation.rb +15 -2
- data/lib/newrelic_security/instrumentation-security/padrino/prepend.rb +12 -0
- data/lib/newrelic_security/instrumentation-security/patron/instrumentation.rb +2 -15
- data/lib/newrelic_security/instrumentation-security/rails/chain.rb +26 -2
- data/lib/newrelic_security/instrumentation-security/rails/instrumentation.rb +29 -3
- data/lib/newrelic_security/instrumentation-security/rails/prepend.rb +18 -0
- data/lib/newrelic_security/instrumentation-security/roda/chain.rb +7 -2
- data/lib/newrelic_security/instrumentation-security/roda/instrumentation.rb +3 -1
- data/lib/newrelic_security/instrumentation-security/roda/prepend.rb +7 -1
- data/lib/newrelic_security/instrumentation-security/sinatra/chain.rb +6 -0
- data/lib/newrelic_security/instrumentation-security/sinatra/instrumentation.rb +9 -0
- data/lib/newrelic_security/instrumentation-security/sinatra/prepend.rb +4 -0
- data/lib/newrelic_security/instrumentation-security/sqlite3/instrumentation.rb +4 -4
- data/lib/newrelic_security/newrelic-security-api/api.rb +1 -1
- data/lib/newrelic_security/parse-cron/cron_parser.rb +294 -0
- data/lib/newrelic_security/version.rb +1 -1
- data/newrelic_security.gemspec +1 -1
- metadata +8 -4
@@ -5,6 +5,7 @@ module NewRelic::Security
|
|
5
5
|
module Control
|
6
6
|
EVENT_QUEUE_SIZE = 10_000
|
7
7
|
HEALTH_INTERVAL = 300
|
8
|
+
ERROR_REPORTING_INTERVAL = 30
|
8
9
|
|
9
10
|
class EventProcessor
|
10
11
|
|
@@ -15,6 +16,7 @@ module NewRelic::Security
|
|
15
16
|
@eventQ = ::SizedQueue.new(EVENT_QUEUE_SIZE)
|
16
17
|
create_dequeue_threads
|
17
18
|
create_keep_alive_thread
|
19
|
+
create_error_reporting_thread
|
18
20
|
NewRelic::Security::Agent.init_logger.info "[STEP-5] => Security agent components started"
|
19
21
|
end
|
20
22
|
|
@@ -46,6 +48,7 @@ module NewRelic::Security
|
|
46
48
|
enqueue(event)
|
47
49
|
if @first_event
|
48
50
|
NewRelic::Security::Agent.init_logger.info "[STEP-8] => First event sent for validation. Security agent started successfully : #{event.to_json}"
|
51
|
+
NewRelic::Security::Agent.config.traffic_start_time = current_time_millis unless NewRelic::Security::Agent.config[:traffic_start_time]
|
49
52
|
@first_event = false
|
50
53
|
end
|
51
54
|
event = nil
|
@@ -128,6 +131,29 @@ module NewRelic::Security
|
|
128
131
|
NewRelic::Security::Agent.logger.error "Exception in health check thread, #{exception.inspect}"
|
129
132
|
end
|
130
133
|
|
134
|
+
def current_time_millis
|
135
|
+
(Time.now.to_f * 1000).to_i
|
136
|
+
end
|
137
|
+
|
138
|
+
def create_error_reporting_thread
|
139
|
+
@error_reporting_thread = Thread.new {
|
140
|
+
Thread.current.name = "newrelic_security_error_reporting_thread"
|
141
|
+
while true do
|
142
|
+
sleep ERROR_REPORTING_INTERVAL
|
143
|
+
send_error_reporting_event if NewRelic::Security::Agent::Control::WebsocketClient.instance.is_open?
|
144
|
+
end
|
145
|
+
}
|
146
|
+
rescue Exception => exception
|
147
|
+
NewRelic::Security::Agent.logger.error "Exception in error_reporting thread, #{exception.inspect}"
|
148
|
+
end
|
149
|
+
|
150
|
+
def send_error_reporting_event
|
151
|
+
NewRelic::Security::Agent.agent.error_reporting&.exceptions_map&.each_value do |exception|
|
152
|
+
NewRelic::Security::Agent::Control::WebsocketClient.instance.send(exception)
|
153
|
+
end
|
154
|
+
NewRelic::Security::Agent.agent.error_reporting&.exceptions_map&.clear
|
155
|
+
end
|
156
|
+
|
131
157
|
end
|
132
158
|
end
|
133
159
|
end
|
@@ -7,19 +7,13 @@ module NewRelic::Security
|
|
7
7
|
NewRelic::Security::Agent.logger.info "NewRelic server_source_configuration_added for pid : #{Process.pid}, Parent Pid : #{Process.ppid}"
|
8
8
|
NewRelic::Security::Agent.init_logger.info "NewRelic server_source_configuration_added for pid : #{Process.pid}, Parent Pid : #{Process.ppid}"
|
9
9
|
NewRelic::Security::Agent.config.update_server_config
|
10
|
-
if NewRelic::Security::Agent.config[:enabled] && !NewRelic::Security::Agent.config[:high_security]
|
11
|
-
NewRelic::Security::Agent.agent.
|
10
|
+
if NewRelic::Security::Agent.config[:'security.enabled'] && !NewRelic::Security::Agent.config[:high_security]
|
11
|
+
Thread.new { NewRelic::Security::Agent.agent.scan_scheduler.init_via_scan_scheduler }
|
12
12
|
else
|
13
13
|
NewRelic::Security::Agent.logger.info "New Relic Security is disabled by one of the user provided config `security.enabled` or `high_security`."
|
14
14
|
NewRelic::Security::Agent.init_logger.info "New Relic Security is disabled by one of the user provided config `security.enabled` or `high_security`."
|
15
15
|
end
|
16
16
|
}
|
17
|
-
::NewRelic::Agent.instance.events.subscribe(:security_policy_received) { |received_policy|
|
18
|
-
NewRelic::Security::Agent.logger.info "security_policy_received pid ::::::: #{Process.pid} #{Process.ppid}, #{received_policy}"
|
19
|
-
NewRelic::Security::Agent.config[:policy].merge!(received_policy)
|
20
|
-
NewRelic::Security::Agent.init_logger.info "[STEP-7] => Received and applied policy/configuration : #{received_policy}"
|
21
|
-
NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
|
22
|
-
}
|
23
17
|
end
|
24
18
|
end
|
25
19
|
end
|
@@ -14,6 +14,8 @@ module NewRelic::Security
|
|
14
14
|
@collectorVersion = NewRelic::Security::VERSION
|
15
15
|
@buildNumber = nil
|
16
16
|
@applicationUUID = NewRelic::Security::Agent.config[:uuid]
|
17
|
+
@appAccountId = NewRelic::Security::Agent.config[:account_id]
|
18
|
+
@appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
|
17
19
|
@groupName = NewRelic::Security::Agent.config[:mode]
|
18
20
|
@jsonVersion = NewRelic::Security::Agent.config[:json_version]
|
19
21
|
@policyVersion = nil
|
@@ -8,7 +8,7 @@ module NewRelic::Security
|
|
8
8
|
|
9
9
|
class GRPCContext
|
10
10
|
|
11
|
-
attr_accessor :time_stamp, :method, :headers, :body, :route, :cache, :fuzz_files, :url, :server_name, :server_port, :client_ip, :client_port, :is_grpc, :metadata, :event_counter
|
11
|
+
attr_accessor :time_stamp, :method, :headers, :body, :route, :cache, :fuzz_files, :url, :server_name, :server_port, :client_ip, :client_port, :is_grpc, :metadata, :event_counter, :mutex
|
12
12
|
|
13
13
|
def initialize(grpc_request)
|
14
14
|
@time_stamp = current_time_millis
|
@@ -32,6 +32,7 @@ module NewRelic::Security
|
|
32
32
|
@cache = {}
|
33
33
|
@fuzz_files = ::Set.new
|
34
34
|
@event_counter = 0
|
35
|
+
@mutex = Mutex.new
|
35
36
|
NewRelic::Security::Agent.agent.http_request_count.increment
|
36
37
|
end
|
37
38
|
|
@@ -18,6 +18,8 @@ module NewRelic::Security
|
|
18
18
|
@framework = NewRelic::Security::Agent.config[:framework]
|
19
19
|
@protectedServer = nil
|
20
20
|
@applicationUUID = NewRelic::Security::Agent.config[:uuid]
|
21
|
+
@appAccountId = NewRelic::Security::Agent.config[:account_id]
|
22
|
+
@appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
|
21
23
|
@collectorVersion = NewRelic::Security::VERSION
|
22
24
|
@buildNumber = nil
|
23
25
|
@jsonVersion = NewRelic::Security::Agent.config[:json_version]
|
@@ -33,6 +35,9 @@ module NewRelic::Security
|
|
33
35
|
@iastEventStats = {}
|
34
36
|
@raspEventStats = {}
|
35
37
|
@exitEventStats = {}
|
38
|
+
@procStartTime = NewRelic::Security::Agent.config[:process_start_time]
|
39
|
+
@trafficStartedTime = NewRelic::Security::Agent.config[:traffic_start_time]
|
40
|
+
@scanStartTime = NewRelic::Security::Agent.config[:scan_start_time]
|
36
41
|
end
|
37
42
|
|
38
43
|
def as_json
|
@@ -10,18 +10,22 @@ module NewRelic::Security
|
|
10
10
|
UNDERSCORE = '_'
|
11
11
|
HYPHEN = '-'
|
12
12
|
REQUEST_METHOD = 'REQUEST_METHOD'
|
13
|
+
HTTP_HOST = 'HTTP_HOST'
|
13
14
|
PATH_INFO = 'PATH_INFO'
|
15
|
+
QUERY_STRING = 'QUERY_STRING'
|
14
16
|
RACK_INPUT = 'rack.input'
|
15
|
-
CGI_VARIABLES = ::Set.new(%W[ AUTH_TYPE CONTENT_LENGTH CONTENT_TYPE GATEWAY_INTERFACE HTTPS PATH_INFO PATH_TRANSLATED REQUEST_URI QUERY_STRING REMOTE_ADDR REMOTE_HOST REMOTE_IDENT REMOTE_USER REQUEST_METHOD SCRIPT_NAME SERVER_NAME SERVER_PORT SERVER_PROTOCOL SERVER_SOFTWARE rack.url_scheme ])
|
17
|
+
CGI_VARIABLES = ::Set.new(%W[ AUTH_TYPE CONTENT_LENGTH CONTENT_TYPE GATEWAY_INTERFACE HTTPS HTTP_HOST PATH_INFO PATH_TRANSLATED REQUEST_URI QUERY_STRING REMOTE_ADDR REMOTE_HOST REMOTE_IDENT REMOTE_USER REQUEST_METHOD SCRIPT_NAME SERVER_NAME SERVER_PORT SERVER_PROTOCOL SERVER_SOFTWARE rack.url_scheme ])
|
18
|
+
REQUEST_BODY_LIMIT = 500 #KB
|
16
19
|
|
17
20
|
class HTTPContext
|
18
21
|
|
19
|
-
attr_accessor :time_stamp, :req, :method, :headers, :params, :body, :data_truncated, :route, :cache, :fuzz_files, :event_counter
|
22
|
+
attr_accessor :time_stamp, :req, :method, :headers, :params, :body, :data_truncated, :route, :cache, :fuzz_files, :event_counter, :mutex, :url
|
20
23
|
|
21
24
|
def initialize(env)
|
22
25
|
@time_stamp = current_time_millis
|
23
26
|
@req = env.select { |key, _| CGI_VARIABLES.include? key}
|
24
27
|
@method = @req[REQUEST_METHOD]
|
28
|
+
@url = "#{@req[PATH_INFO]}?#{@req[QUERY_STRING]}"
|
25
29
|
@headers = env.select { |key, _| key.include?(HTTP_) }
|
26
30
|
@headers = @headers.transform_keys{ |key| key[5..-1].gsub(UNDERSCORE, HYPHEN).downcase }
|
27
31
|
request = Rack::Request.new(env) unless env.empty?
|
@@ -30,23 +34,24 @@ module NewRelic::Security
|
|
30
34
|
strio = env[RACK_INPUT]
|
31
35
|
if strio.instance_of?(::StringIO)
|
32
36
|
offset = strio.tell
|
33
|
-
@body = strio.read(
|
37
|
+
@body = strio.read(REQUEST_BODY_LIMIT * 1024) #after read, offset changes
|
34
38
|
strio.seek(offset)
|
35
39
|
# In case of Grape and Roda strio.read giving empty result, added below approach to handle such cases
|
36
40
|
@body = strio.string if @body.nil? && strio.size > 0
|
37
41
|
elsif defined?(::Rack) && defined?(::Rack::Lint::InputWrapper) && strio.instance_of?(::Rack::Lint::InputWrapper)
|
38
|
-
@body = strio.read(
|
42
|
+
@body = strio.read(REQUEST_BODY_LIMIT * 1024)
|
39
43
|
elsif defined?(::Protocol::Rack::Input) && defined?(::Protocol::Rack::Input) && strio.instance_of?(::Protocol::Rack::Input)
|
40
|
-
@body = strio.read(
|
44
|
+
@body = strio.read(REQUEST_BODY_LIMIT * 1024)
|
41
45
|
elsif defined?(::PhusionPassenger::Utils::TeeInput) && strio.instance_of?(::PhusionPassenger::Utils::TeeInput)
|
42
|
-
@body = strio.read(
|
46
|
+
@body = strio.read(REQUEST_BODY_LIMIT * 1024)
|
43
47
|
end
|
44
|
-
@data_truncated = @body && @body.size >=
|
48
|
+
@data_truncated = @body && @body.size >= REQUEST_BODY_LIMIT * 1024
|
45
49
|
strio&.rewind
|
46
50
|
@body = @body.force_encoding(Encoding::UTF_8) if @body.is_a?(String)
|
47
51
|
@cache = Hash.new
|
48
52
|
@fuzz_files = ::Set.new
|
49
53
|
@event_counter = 0
|
54
|
+
@mutex = Mutex.new
|
50
55
|
NewRelic::Security::Agent.agent.http_request_count.increment
|
51
56
|
NewRelic::Security::Agent.agent.iast_client.completed_requests[@headers[NR_CSEC_PARENT_ID]] = [] if @headers.key?(NR_CSEC_PARENT_ID)
|
52
57
|
end
|
@@ -66,6 +71,10 @@ module NewRelic::Security
|
|
66
71
|
def self.reset_context
|
67
72
|
::NewRelic::Agent::Tracer.current_transaction.remove_instance_variable(:@security_context_data) if ::NewRelic::Agent::Tracer.current_transaction.instance_variable_defined?(:@security_context_data)
|
68
73
|
end
|
74
|
+
|
75
|
+
def self.get_current_transaction
|
76
|
+
::NewRelic::Agent::Tracer.current_transaction
|
77
|
+
end
|
69
78
|
end
|
70
79
|
|
71
80
|
end
|
@@ -17,9 +17,8 @@ module NewRelic::Security
|
|
17
17
|
IS_GRPC = 'isGrpc'
|
18
18
|
INPUT_CLASS = 'inputClass'
|
19
19
|
SERVER_PORT_1 = 'serverPort'
|
20
|
-
PROBING = 'probing'
|
21
|
-
INTERVAL = 'interval'
|
22
20
|
IS_GRPC_CLIENT_STREAM = 'isGrpcClientStream'
|
21
|
+
PROBING_INTERVAL = 5
|
23
22
|
|
24
23
|
class IASTClient
|
25
24
|
|
@@ -54,6 +53,7 @@ module NewRelic::Security
|
|
54
53
|
Thread.current.name = "newrelic_security_iast_thread-#{t}"
|
55
54
|
loop do
|
56
55
|
fuzz_request = @fuzzQ.deq #thread blocks when the queue is empty
|
56
|
+
NewRelic::Security::Agent.config.scan_start_time = current_time_millis unless NewRelic::Security::Agent.config[:scan_start_time]
|
57
57
|
if fuzz_request.request[IS_GRPC]
|
58
58
|
fire_grpc_request(fuzz_request.id, fuzz_request.request, fuzz_request.reflected_metadata)
|
59
59
|
else
|
@@ -71,7 +71,8 @@ module NewRelic::Security
|
|
71
71
|
@iast_data_transfer_request_processor_thread = Thread.new do
|
72
72
|
Thread.current.name = "newrelic_security_iast_data_transfer_request_processor"
|
73
73
|
loop do
|
74
|
-
|
74
|
+
# TODO: Check & remove this probing interval if not required, earlier this was used from policy sent by SE.
|
75
|
+
sleep PROBING_INTERVAL
|
75
76
|
current_timestamp = current_time_millis
|
76
77
|
cooldown_sleep_time = @cooldown_till_timestamp - current_timestamp
|
77
78
|
sleep cooldown_sleep_time/1000 if cooldown_sleep_time > 0
|
@@ -84,9 +85,21 @@ module NewRelic::Security
|
|
84
85
|
if batch_size > 100 && remaining_record_capacity > batch_size
|
85
86
|
iast_data_transfer_request = NewRelic::Security::Agent::Control::IASTDataTransferRequest.new
|
86
87
|
iast_data_transfer_request.batchSize = batch_size * 2
|
88
|
+
# TODO: Below calculation of batch_size overrides above logic and can be removed once below one is stablises or rate limit feature is released.
|
89
|
+
if NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit']
|
90
|
+
batch_size =
|
91
|
+
if NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] < 12
|
92
|
+
1
|
93
|
+
elsif NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] > 3600
|
94
|
+
300
|
95
|
+
else
|
96
|
+
NewRelic::Security::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'] / 12
|
97
|
+
end
|
98
|
+
iast_data_transfer_request.batchSize = batch_size
|
99
|
+
end
|
87
100
|
iast_data_transfer_request.pendingRequestIds = pending_request_ids.to_a
|
88
101
|
iast_data_transfer_request.completedRequests = completed_requests
|
89
|
-
NewRelic::Security::Agent.agent.event_processor
|
102
|
+
NewRelic::Security::Agent.agent.event_processor&.send_iast_data_transfer_request(iast_data_transfer_request)
|
90
103
|
end
|
91
104
|
end
|
92
105
|
end
|
@@ -122,18 +135,18 @@ module NewRelic::Security
|
|
122
135
|
def fire_grpc_request(fuzz_request_id, request, reflected_metadata)
|
123
136
|
service = Object.const_get(request[METHOD].split(SLASH)[0]).superclass
|
124
137
|
method = request[METHOD].split(SLASH)[1]
|
125
|
-
@stub
|
138
|
+
@stub ||= service.rpc_stub_class.new("localhost:#{request[SERVER_PORT_1]}", :this_channel_is_insecure)
|
126
139
|
|
127
|
-
parsed_body =
|
128
|
-
if reflected_metadata[IS_GRPC_CLIENT_STREAM]
|
129
|
-
|
140
|
+
parsed_body = request[BODY][1..-2].split(',')
|
141
|
+
chunks_enum = if reflected_metadata[IS_GRPC_CLIENT_STREAM]
|
142
|
+
Enumerator.new do |y|
|
130
143
|
parsed_body.each do |b|
|
131
144
|
y << Object.const_get(reflected_metadata[INPUT_CLASS]).decode_json(b)
|
132
145
|
end
|
133
146
|
end
|
134
|
-
|
135
|
-
|
136
|
-
|
147
|
+
else
|
148
|
+
Object.const_get(reflected_metadata[INPUT_CLASS]).decode_json(request[BODY])
|
149
|
+
end
|
137
150
|
response = @stub.public_send(method, chunks_enum, metadata: request[HEADERS])
|
138
151
|
# response = @stub.send(method, JSON.parse(request['body'], object_class: OpenStruct))
|
139
152
|
# request[HEADERS].delete(VERSION) if request[HEADERS].key?(VERSION)
|
@@ -11,6 +11,8 @@ module NewRelic::Security
|
|
11
11
|
def initialize
|
12
12
|
@jsonName = :'iast-data-request'
|
13
13
|
@applicationUUID = NewRelic::Security::Agent.config[:uuid]
|
14
|
+
@appAccountId = NewRelic::Security::Agent.config[:account_id]
|
15
|
+
@appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
|
14
16
|
@batchSize = 10
|
15
17
|
@pendingRequestIds = []
|
16
18
|
@completedRequests = Hash.new
|
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'newrelic_security/parse-cron/cron_parser'
|
2
|
+
|
3
|
+
module NewRelic::Security
|
4
|
+
module Agent
|
5
|
+
module Control
|
6
|
+
class ScanScheduler
|
7
|
+
def init_via_scan_scheduler
|
8
|
+
if NewRelic::Security::Agent.config[:'security.scan_schedule.delay'].positive?
|
9
|
+
NewRelic::Security::Agent.logger.info "IAST delay is set to: #{NewRelic::Security::Agent.config[:'security.scan_schedule.delay']}, current time: #{Time.now}"
|
10
|
+
start_agent_with_delay(NewRelic::Security::Agent.config[:'security.scan_schedule.delay']*60)
|
11
|
+
elsif !NewRelic::Security::Agent.config[:'security.scan_schedule.schedule'].to_s.empty?
|
12
|
+
cron_expression_task(NewRelic::Security::Agent.config[:'security.scan_schedule.schedule'], NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
|
13
|
+
else
|
14
|
+
NewRelic::Security::Agent.agent.init
|
15
|
+
NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
|
16
|
+
shutdown_at_duration_reached(NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
|
17
|
+
end
|
18
|
+
rescue StandardError => exception
|
19
|
+
NewRelic::Security::Agent.logger.error "Exception in IAST scan scheduler: #{exception.inspect} #{exception.backtrace}"
|
20
|
+
::NewRelic::Agent.notice_error(exception)
|
21
|
+
end
|
22
|
+
|
23
|
+
def start_agent_with_delay(delay)
|
24
|
+
NewRelic::Security::Agent.logger.info "Security Agent delay scan time is set to: #{(delay/60).ceil} minutes when always_sample_traces is #{NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']}, current time: #{Time.now}"
|
25
|
+
if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
|
26
|
+
NewRelic::Security::Agent.agent.init unless NewRelic::Security::Agent.config[:enabled]
|
27
|
+
sleep delay if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
|
28
|
+
else
|
29
|
+
sleep delay
|
30
|
+
NewRelic::Security::Agent.agent.init
|
31
|
+
end
|
32
|
+
NewRelic::Security::Agent.agent.start_iast_client if NewRelic::Security::Agent::Utils.is_IAST?
|
33
|
+
shutdown_at_duration_reached(NewRelic::Security::Agent.config[:'security.scan_schedule.duration']*60)
|
34
|
+
end
|
35
|
+
|
36
|
+
def shutdown_at_duration_reached(duration)
|
37
|
+
shutdown_at = Time.now.to_i + duration
|
38
|
+
shut_down_time = (Time.now + duration).strftime("%a %d %b %Y %H:%M:%S")
|
39
|
+
return if duration <= 0
|
40
|
+
NewRelic::Security::Agent.logger.info "IAST Duration is set to: #{duration/60} minutes, timestamp: #{shut_down_time} time, current time: #{Time.now}"
|
41
|
+
@shutdown_monitor_thread = Thread.new do
|
42
|
+
Thread.current.name = "newrelic_security_shutdown_monitor_thread"
|
43
|
+
loop do
|
44
|
+
sleep 1
|
45
|
+
next if Time.now.to_i < shutdown_at
|
46
|
+
if NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']
|
47
|
+
NewRelic::Security::Agent.logger.info "Shutdown IAST Data transfer request processor only as 'security.scan_schedule.always_sample_traces' is #{NewRelic::Security::Agent.config[:'security.scan_schedule.always_sample_traces']} now at current time: #{Time.now}"
|
48
|
+
NewRelic::Security::Agent.agent.iast_client&.iast_data_transfer_request_processor_thread&.kill
|
49
|
+
else
|
50
|
+
NewRelic::Security::Agent.logger.info "Shutdown IAST agent now at current time: #{Time.now}"
|
51
|
+
::NewRelic::Agent.notice_error(StandardError.new("WS Connection closed by local"))
|
52
|
+
NewRelic::Security::Agent.agent.shutdown_security_agent
|
53
|
+
end
|
54
|
+
break
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def cron_expression_task(schedule, duration)
|
60
|
+
@cron_parser = NewRelic::Security::ParseCron::CronParser.new(schedule)
|
61
|
+
loop do
|
62
|
+
next_run = @cron_parser.next(Time.now)
|
63
|
+
NewRelic::Security::Agent.logger.info "Next init via cron exp: #{schedule}, is scheduled at : #{next_run}"
|
64
|
+
delay = next_run - Time.now
|
65
|
+
if NewRelic::Security::Agent.agent.iast_client&.iast_data_transfer_request_processor_thread&.alive?
|
66
|
+
sleep delay > 2 ? delay - 2 : delay
|
67
|
+
else
|
68
|
+
start_agent_with_delay(delay)
|
69
|
+
end
|
70
|
+
return if duration <= 0
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -21,6 +21,8 @@ module NewRelic::Security
|
|
21
21
|
NR_CSEC_ENTITY_NAME = 'NR-CSEC-ENTITY-NAME'
|
22
22
|
NR_CSEC_ENTITY_GUID = 'NR-CSEC-ENTITY-GUID'
|
23
23
|
NR_CSEC_IAST_DATA_TRANSFER_MODE = 'NR-CSEC-IAST-DATA-TRANSFER-MODE'
|
24
|
+
NR_CSEC_IGNORED_VUL_CATEGORIES = 'NR-CSEC-IGNORED-VUL-CATEGORIES'
|
25
|
+
NR_CSEC_PROCESS_START_TIME = 'NR-CSEC-PROCESS-START-TIME'
|
24
26
|
|
25
27
|
class WebsocketClient
|
26
28
|
include Singleton
|
@@ -43,6 +45,8 @@ module NewRelic::Security
|
|
43
45
|
headers[NR_CSEC_ENTITY_NAME] = NewRelic::Security::Agent.config[:app_name]
|
44
46
|
headers[NR_CSEC_ENTITY_GUID] = NewRelic::Security::Agent.config[:entity_guid]
|
45
47
|
headers[NR_CSEC_IAST_DATA_TRANSFER_MODE] = PULL
|
48
|
+
headers[NR_CSEC_IGNORED_VUL_CATEGORIES] = ingnored_vul_categories.join(COMMA)
|
49
|
+
headers[NR_CSEC_PROCESS_START_TIME] = NewRelic::Security::Agent.config[:process_start_time]
|
46
50
|
|
47
51
|
begin
|
48
52
|
cert_store = ::OpenSSL::X509::Store.new
|
@@ -76,21 +80,26 @@ module NewRelic::Security
|
|
76
80
|
|
77
81
|
connection.on :error do |e|
|
78
82
|
NewRelic::Security::Agent.logger.error "Error in websocket connection : #{e.inspect} #{e.backtrace}"
|
83
|
+
::NewRelic::Agent.notice_error(e)
|
79
84
|
Thread.new { NewRelic::Security::Agent::Control::WebsocketClient.instance.close(true) }
|
80
85
|
end
|
81
86
|
rescue Errno::EPIPE => exception
|
82
87
|
NewRelic::Security::Agent.logger.error "Unable to connect to validator_service: #{exception.inspect}"
|
88
|
+
::NewRelic::Agent.notice_error(exception)
|
83
89
|
NewRelic::Security::Agent.config.disable_security
|
84
90
|
rescue Errno::ECONNRESET => exception
|
85
91
|
NewRelic::Security::Agent.logger.error "Unable to connect to validator_service: #{exception.inspect}"
|
92
|
+
::NewRelic::Agent.notice_error(exception)
|
86
93
|
NewRelic::Security::Agent.config.disable_security
|
87
94
|
Thread.new { NewRelic::Security::Agent.agent.reconnect(15) }
|
88
95
|
rescue Errno::ECONNREFUSED => exception
|
89
96
|
NewRelic::Security::Agent.logger.error "Unable to connect to validator_service: #{exception.inspect}"
|
97
|
+
::NewRelic::Agent.notice_error(exception)
|
90
98
|
NewRelic::Security::Agent.config.disable_security
|
91
99
|
Thread.new { NewRelic::Security::Agent.agent.reconnect(15) }
|
92
100
|
rescue => exception
|
93
101
|
NewRelic::Security::Agent.logger.error "Exception in websocket init: #{exception.inspect} #{exception.backtrace}"
|
102
|
+
::NewRelic::Agent.notice_error(exception)
|
94
103
|
NewRelic::Security::Agent.config.disable_security
|
95
104
|
Thread.new { NewRelic::Security::Agent.agent.reconnect(15) }
|
96
105
|
end
|
@@ -125,6 +134,20 @@ module NewRelic::Security
|
|
125
134
|
false
|
126
135
|
end
|
127
136
|
|
137
|
+
private
|
138
|
+
|
139
|
+
def ingnored_vul_categories
|
140
|
+
list = []
|
141
|
+
list << FILE_OPERATION << FILE_INTEGRITY if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access']
|
142
|
+
list << SQL_DB_COMMAND if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.sql_injection']
|
143
|
+
list << NOSQL_DB_COMMAND if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.nosql_injection']
|
144
|
+
list << LDAP if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ldap_injection']
|
145
|
+
list << SYSTEM_COMMAND if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.command_injection']
|
146
|
+
list << XPATH if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.xpath_injection']
|
147
|
+
list << HTTP_REQUEST if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ssrf']
|
148
|
+
list << REFLECTED_XSS if NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.rxss']
|
149
|
+
list
|
150
|
+
end
|
128
151
|
end
|
129
152
|
end
|
130
153
|
end
|
@@ -15,8 +15,7 @@ module NewRelic::Security
|
|
15
15
|
ASTERISK = '*'
|
16
16
|
|
17
17
|
def is_IAST?
|
18
|
-
return
|
19
|
-
return NewRelic::Security::Agent.config[:policy][VULNERABILITY_SCAN][IAST_SCAN][ENABLED] if NewRelic::Security::Agent.config[:policy][VULNERABILITY_SCAN][ENABLED]
|
18
|
+
return true if NewRelic::Security::Agent.config[:mode] == IAST
|
20
19
|
false
|
21
20
|
end
|
22
21
|
|
@@ -36,10 +35,11 @@ module NewRelic::Security
|
|
36
35
|
decrypted_data.split(COMMA).each do |filename|
|
37
36
|
begin
|
38
37
|
filename.gsub!(NR_CSEC_VALIDATOR_HOME_TMP, NewRelic::Security::Agent.config[:fuzz_dir_path])
|
38
|
+
filename.gsub!(NR_CSEC_VALIDATOR_HOME_TMP_URL_ENCODED, NewRelic::Security::Agent.config[:fuzz_dir_path])
|
39
39
|
filename.gsub!(NR_CSEC_VALIDATOR_FILE_SEPARATOR, ::File::SEPARATOR)
|
40
40
|
dirname = ::File.dirname(filename)
|
41
41
|
::FileUtils.mkdir_p(dirname, :mode => 0770) unless ::File.directory?(dirname)
|
42
|
-
ctxt&.fuzz_files
|
42
|
+
ctxt&.fuzz_files&.<< filename
|
43
43
|
::File.open(filename, ::File::WRONLY | ::File::CREAT | ::File::EXCL) do |fd|
|
44
44
|
# puts "Ownership acquired by : #{Process.pid}"
|
45
45
|
end unless ::File.exist?(filename)
|
@@ -95,7 +95,8 @@ module NewRelic::Security
|
|
95
95
|
|
96
96
|
def get_app_routes(framework, router = nil)
|
97
97
|
enable_object_space_in_jruby
|
98
|
-
|
98
|
+
case framework
|
99
|
+
when :rails
|
99
100
|
::Rails.application.routes.routes.each do |route|
|
100
101
|
if route.verb.is_a?(::Regexp)
|
101
102
|
method = route.verb.inspect.match(/[a-zA-Z]+/)
|
@@ -106,27 +107,31 @@ module NewRelic::Security
|
|
106
107
|
}
|
107
108
|
end
|
108
109
|
end
|
109
|
-
|
110
|
+
when :sinatra
|
110
111
|
::Sinatra::Application.routes.each do |method, routes|
|
111
112
|
routes.map { |r| r.first.to_s }.map do |route|
|
112
113
|
NewRelic::Security::Agent.agent.route_map << "#{method}@#{route}"
|
113
114
|
end
|
114
115
|
end
|
115
|
-
|
116
|
+
when :grape
|
116
117
|
ObjectSpace.each_object(::Grape::Endpoint) { |z|
|
117
118
|
z.instance_variable_get(:@routes)&.each { |route|
|
118
|
-
http_method = route.instance_variable_get(:@request_method)
|
119
|
+
http_method = route.instance_variable_get(:@request_method) || route.instance_variable_get(:@options)[:method]
|
119
120
|
NewRelic::Security::Agent.agent.route_map << "#{http_method}@#{route.pattern.origin}"
|
120
121
|
}
|
121
122
|
}
|
122
|
-
|
123
|
+
when :padrino
|
123
124
|
if router.instance_of?(::Padrino::PathRouter::Router)
|
124
125
|
router.instance_variable_get(:@routes).each do |route|
|
125
126
|
NewRelic::Security::Agent.agent.route_map << "#{route.instance_variable_get(:@verb)}@#{route.matcher.instance_variable_get(:@path)}"
|
126
127
|
end
|
127
128
|
end
|
128
|
-
|
129
|
+
when :roda
|
129
130
|
NewRelic::Security::Agent.logger.warn "TODO: Roda is a routing tree web toolkit, which generates route dynamically, hence route extraction is not possible."
|
131
|
+
when :grpc
|
132
|
+
router.owner.superclass.public_instance_methods(false).each do |m|
|
133
|
+
NewRelic::Security::Agent.agent.route_map << "*@/#{router.owner}/#{m}"
|
134
|
+
end
|
130
135
|
else
|
131
136
|
NewRelic::Security::Agent.logger.error "Unable to get app routes as Framework not detected"
|
132
137
|
end
|
@@ -7,6 +7,7 @@ module NewRelic::Security
|
|
7
7
|
LANGUAGE_COLLECTOR = 'LANGUAGE_COLLECTOR'
|
8
8
|
STANDARD_OUT = 'STDOUT'
|
9
9
|
NR_CSEC_VALIDATOR_HOME_TMP = '{{NR_CSEC_VALIDATOR_HOME_TMP}}'
|
10
|
+
NR_CSEC_VALIDATOR_HOME_TMP_URL_ENCODED = '%7B%7BNR_CSEC_VALIDATOR_HOME_TMP%7D%7D'
|
10
11
|
NR_CSEC_VALIDATOR_FILE_SEPARATOR = '{{NR_CSEC_VALIDATOR_FILE_SEPARATOR}}'
|
11
12
|
SEC_HOME_PATH = 'nr-security-home'
|
12
13
|
LOGS_DIR = 'logs'
|
@@ -16,6 +17,7 @@ module NewRelic::Security
|
|
16
17
|
NR_CSEC_FUZZ_REQUEST_ID = 'nr-csec-fuzz-request-id'
|
17
18
|
NR_CSEC_TRACING_DATA = 'nr-csec-tracing-data'
|
18
19
|
NR_CSEC_PARENT_ID = 'nr-csec-parent-id'
|
20
|
+
IAST = 'IAST'
|
19
21
|
COLON_IAST_COLON = ':IAST:'
|
20
22
|
NOSQL_DB_COMMAND = 'NOSQL_DB_COMMAND'
|
21
23
|
SQL_DB_COMMAND = 'SQL_DB_COMMAND'
|
@@ -62,6 +64,4 @@ module NewRelic::Security
|
|
62
64
|
CONTENT_TYPE1 = 'content-Type'
|
63
65
|
PULL = 'PULL'
|
64
66
|
SHA1 = 'sha1'
|
65
|
-
VULNERABILITY_SCAN = 'vulnerabilityScan'
|
66
|
-
IAST_SCAN = 'iastScan'
|
67
67
|
end
|
@@ -6,22 +6,11 @@ module NewRelic::Security
|
|
6
6
|
module Instrumentation
|
7
7
|
module AsyncHttp
|
8
8
|
|
9
|
-
def call_on_enter(
|
9
|
+
def call_on_enter(_method, url, headers, _body)
|
10
10
|
event = nil
|
11
11
|
NewRelic::Security::Agent.logger.debug "OnEnter : #{self.class}.#{__method__}"
|
12
|
-
ob = {}
|
13
|
-
ob[:Method] = method
|
14
12
|
uri = ::URI.parse url
|
15
|
-
|
16
|
-
ob[:host] = uri.host
|
17
|
-
ob[:port] = uri.port
|
18
|
-
ob[:URI] = uri.to_s
|
19
|
-
ob[:path] = uri.path
|
20
|
-
ob[:query] = uri.query
|
21
|
-
ob[:Body] = body.respond_to?(:join) ? body.join.to_s : body.to_s
|
22
|
-
ob[:Headers] = headers.to_h
|
23
|
-
ob.each { |_, value| value.dup.force_encoding(ISO_8859_1).encode(UTF_8) if value.is_a?(String) }
|
24
|
-
event = NewRelic::Security::Agent::Control::Collector.collect(HTTP_REQUEST, [ob])
|
13
|
+
event = NewRelic::Security::Agent::Control::Collector.collect(HTTP_REQUEST, [uri.to_s])
|
25
14
|
NewRelic::Security::Instrumentation::InstrumentationUtils.append_tracing_data(headers, event) if event
|
26
15
|
event
|
27
16
|
rescue => exception
|
@@ -12,20 +12,7 @@ module NewRelic::Security
|
|
12
12
|
self.requests.each {
|
13
13
|
|key, req|
|
14
14
|
uri = NewRelic::Security::Instrumentation::InstrumentationUtils.parse_uri(req.url)
|
15
|
-
|
16
|
-
if uri
|
17
|
-
ob[:Method] = nil
|
18
|
-
ob[:scheme] = uri.scheme
|
19
|
-
ob[:host] = uri.host
|
20
|
-
ob[:port] = uri.port
|
21
|
-
ob[:URI] = uri.to_s
|
22
|
-
ob[:path] = uri.path
|
23
|
-
ob[:query] = uri.query
|
24
|
-
ob[:Body] = req.post_body
|
25
|
-
ob[:Headers] = req.headers
|
26
|
-
ob.each { |_, value| value.dup.force_encoding(ISO_8859_1).encode(UTF_8) if value.is_a?(String) }
|
27
|
-
ic_args.push(ob)
|
28
|
-
end
|
15
|
+
ic_args.push(uri.to_s) if uri
|
29
16
|
}
|
30
17
|
event = NewRelic::Security::Agent::Control::Collector.collect(HTTP_REQUEST, ic_args)
|
31
18
|
self.requests.each { |key, req| NewRelic::Security::Instrumentation::InstrumentationUtils.add_tracing_data(req.headers, event) } if event
|
@@ -7,12 +7,6 @@ module NewRelic::Security
|
|
7
7
|
::Ethon::Easy.class_eval do
|
8
8
|
include NewRelic::Security::Instrumentation::Ethon::Easy
|
9
9
|
|
10
|
-
alias_method :fabricate_without_security, :fabricate
|
11
|
-
|
12
|
-
def fabricate(url, action_name, options)
|
13
|
-
fabricate_on_enter(url, action_name, options) { return fabricate_without_security(url, action_name, options) }
|
14
|
-
end
|
15
|
-
|
16
10
|
alias_method(:headers_equals_without_security, :headers=)
|
17
11
|
|
18
12
|
def headers=(headers)
|