newrelic_security 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c07c9ef96a8884449bdd0d5be44eccf3fee5ae6dde565863fe1eb24449a902f6
-  data.tar.gz: 56f813a8034eed9fd717a0c549275cd1c1e1386c50acdb2142379fbcc980838d
+  metadata.gz: 82cddb6a77c7bac2c46e31b5e0577dd346aa0e78b919fc8fb3c2765a7192c0a6
+  data.tar.gz: cb8f47120816909cb30ae0d16f9c6c4da3f684a8497747ccab3bbf12aa36e501
 SHA512:
-  metadata.gz: 0a3b7c2e86f9ea2c3a6c4c28da88676f904e78c5cce82fb152897aade9acda32ca9ea485d26dfa7ebb6ef1356c14df9d85407971d05a0a0f69e1ec43824227a1
-  data.tar.gz: d3c6dcd24fe5ed0b54537b503e6994e6ed6701d5bab53f90220cce4b300e14a619db19cab4befec08560b72c941898a082a1b3c9738ed29e3fb8feae714edeb2
+  metadata.gz: 96abdbedd151b564c30d2ccafc1e98b46185130d6028634877ef0a42c1dc8ff775cf5affbcb7fdd7eee305bbb5ca33a266dfd82ebd5b7f0d2f6e396687af7897
+  data.tar.gz: a69f0fcaf8a427a4ec169ed7dd33952ee88e723a89281a2e9b46841029898fdea6a21748290e2058e861629e368da84b0b6d317181248b33f88a3a33bc5049e2

data/.github/workflows/pr_ci.yml

@@ -72,6 +72,6 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           resultPath: lib/coverage_results/.last_run.json
-          failedThreshold: 70
-          failedThresholdBranch: 33
+          failedThreshold: 67
+          failedThresholdBranch: 25
 

data/CHANGELOG.md

@@ -1,5 +1,87 @@
 # New Relic Ruby Security Agent Release Notes
 
+## v0.3.0
+
+Version 0.3.0 introduces more control on IAST scanning through new configs(exclude_from_iast_scan, scan_schedule & scan_controllers) and 
+features like API inventory for gRPC server and IAST scan start related timestamps.
+
+Updated json_version: **1.2.8**
+
+- Feature: IAST scan exclusion for apis, http request parameters(header, query & body) & IAST detection categories and scan scheduling through delay, duration & cron schedule. [PR#131](https://github.com/newrelic/csec-ruby-agent/pull/131)
+
+- Feature: IAST scan request rate limit to control IAST scan request firing. [PR#132](https://github.com/newrelic/csec-ruby-agent/pull/132)
+
+- Feature: API endpoints support for gRPC server applications. [PR#143](https://github.com/newrelic/csec-ruby-agent/pull/143)
+
+- Feature: Reporting of IAST scanning application procStartTime, trafficStartedTime & scanStartTime. [PR#136](https://github.com/newrelic/csec-ruby-agent/pull/136)
+
+- Misc Chore: Optimised SSRF events parameters to send only URL in parameters. [PR#129](https://github.com/newrelic/csec-ruby-agent/pull/129)
+
+##### New security configs
+
+```yaml
+security:
+  exclude_from_iast_scan:
+    api: []
+    http_request_parameters:
+      header: []
+      query: []
+      body: []
+    iast_detection_category:
+      insecure_settings: false
+      invalid_file_access: false
+      sql_injection: false
+      nosql_injection: false
+      ldap_injection: false
+      javascript_injection: false
+      command_injection: false
+      xpath_injection: false
+      ssrf: false
+      rxss: false
+  scan_schedule:
+    delay: 0
+    duration: 0
+    schedule: ""
+    always_sample_traces: false
+  scan_controllers:
+    iast_scan_request_rate_limit: 3600
+```
+
+##### Deprecated security configs (will be removed in next major release v1.0.0)
+```yaml
+security:
+  request:
+    body_limit: 300
+  detection:
+    rci:
+      enabled: true
+    rxss:
+      enabled: true
+    deserialization:
+      enabled: true
+```
+
+## v0.2.0
+
+Version 0.2.0 introuduces Error reporting as part of security. Any unhandled or 5xx errors in application runtime will now be visible in IAST capability UI. Updated json_version: **1.2.4**
+
+- Feature: Unhandled and 5xx error reporting [PR#134](https://github.com/newrelic/csec-ruby-agent/pull/134)
+
+- Bugfix: Fix for API route not present in rails7 [PR#127](https://github.com/newrelic/csec-ruby-agent/pull/127)
+
+- Bugfix: Fix for Sqlite3 parameters sent in wrong fromat [PR#130](https://github.com/newrelic/csec-ruby-agent/pull/130)
+
+- Bugfix: Fix for multiple events have same id [PR#135](https://github.com/newrelic/csec-ruby-agent/pull/135)
+
+- Bugfix: Fix for NR_CSEC_VALIDATOR_HOME_TMP placeholder value not replaced during File Access fuzzing [PR#138](https://github.com/newrelic/csec-ruby-agent/pull/138)
+
+- Bugfix: Fix for appServerInfo fields are not present in File Operation events [PR#139](https://github.com/newrelic/csec-ruby-agent/pull/139)
+
+- Sending security agent critical errors to APM error inbox [PR#137](https://github.com/newrelic/csec-ruby-agent/pull/137)
+
+- Added key identifiers in entityGuid and acccountId in all json reporting [PR#101](https://github.com/newrelic/csec-ruby-agent/pull/101)
+
+
 ## v0.1.0
 
 Version 0.1.0 introduces `newrelic_security` agent for public preview under Newrelic pre-release software notice.

data/THIRD_PARTY_NOTICES.md

@@ -34,3 +34,11 @@ Distributed under the following license(s):
 
   * [The MIT License](http://opensource.org/licenses/MIT)
 
+
+## [parse-cron](https://github.com/siebertm/parse-cron)
+
+Copyright (C) 2013 Michael Siebert
+
+Distributed under the following license(s):
+
+  * [The MIT License](http://opensource.org/licenses/MIT)

data/lib/newrelic_security/agent/agent.rb

@@ -17,13 +17,16 @@ require 'newrelic_security/agent/control/critical_message'
 require 'newrelic_security/agent/control/event_counter'
 require 'newrelic_security/agent/control/event_stats'
 require 'newrelic_security/agent/control/exit_event'
+require 'newrelic_security/agent/control/application_runtime_error'
+require 'newrelic_security/agent/control/error_reporting'
+require 'newrelic_security/agent/control/scan_scheduler'
 require 'newrelic_security/instrumentation-security/instrumentation_loader'
 
 module NewRelic::Security
   module Agent
     class Agent
 
-      attr_accessor :websocket_client, :event_processor, :iast_client, :http_request_count, :event_processed_count, :event_sent_count, :event_drop_count, :route_map, :iast_event_stats, :rasp_event_stats, :exit_event_stats
+      attr_accessor :websocket_client, :event_processor, :iast_client, :http_request_count, :event_processed_count, :event_sent_count, :event_drop_count, :route_map, :iast_event_stats, :rasp_event_stats, :exit_event_stats, :error_reporting, :scan_scheduler
 
       def initialize
         NewRelic::Security::Agent.config
@@ -41,6 +44,8 @@ module NewRelic::Security
         @iast_event_stats = NewRelic::Security::Agent::Control::EventStats.new
         @rasp_event_stats = NewRelic::Security::Agent::Control::EventStats.new
         @exit_event_stats = NewRelic::Security::Agent::Control::EventStats.new
+        @error_reporting = NewRelic::Security::Agent::Control::ErrorReporting.new
+        @scan_scheduler = NewRelic::Security::Agent::Control::ScanScheduler.new
       end
 
       def init
@@ -56,11 +61,24 @@ module NewRelic::Security
         NewRelic::Security::Agent.logger.error "Exception in security agent init: #{exception.inspect} #{exception.backtrace}\n"
       end
 
+      def shutdown_security_agent
+        @iast_client&.fuzzQ&.clear
+        @iast_client&.completed_requests&.clear
+        @iast_client&.pending_request_ids&.clear
+        @iast_client&.iast_data_transfer_request_processor_thread&.kill
+        NewRelic::Security::Agent.config.disable_security
+        stop_websocket_client_if_open
+      end
+
       def start_websocket_client
-        NewRelic::Security::Agent::Control::WebsocketClient.instance.close(false) if NewRelic::Security::Agent::Control::WebsocketClient.instance.is_open?
+        stop_websocket_client_if_open
         @websocket_client = NewRelic::Security::Agent::Control::WebsocketClient.instance.connect
       end
 
+      def stop_websocket_client_if_open
+        NewRelic::Security::Agent::Control::WebsocketClient.instance.close(false) if NewRelic::Security::Agent::Control::WebsocketClient.instance.is_open?
+      end
+
       def start_event_processor
         @event_processor&.event_dequeue_thread&.kill 
         @event_processor&.healthcheck_thread&.kill
@@ -69,9 +87,10 @@ module NewRelic::Security
       end
 
       def start_iast_client
-        @iast_client&.iast_dequeue_threads&.each { |t| t.kill if t }
+        @iast_client&.iast_dequeue_threads&.each { |t| t&.kill }
         @iast_client&.iast_data_transfer_request_processor_thread&.kill
         @iast_client = nil
+        NewRelic::Security::Agent.logger.info "Starting IAST client now at current time: #{Time.now}"
         @iast_client = NewRelic::Security::Agent::Control::IASTClient.new
       end
 
@@ -93,7 +112,8 @@ module NewRelic::Security
       def find_or_create_file_path(path)
         ::FileUtils.mkdir_p(path) unless ::File.directory?(path)
         ::File.directory?(path)
-      rescue
+      rescue => e
+        ::NewRelic::Agent.notice_error(e)
         return false
       end
 

data/lib/newrelic_security/agent/configuration/manager.rb

@@ -27,19 +27,42 @@ module NewRelic::Security
           @cache[:log_level] = ::NewRelic::Agent.config[:log_level]
           @cache[:high_security] = ::NewRelic::Agent.config[:high_security]
           @cache[:'agent.enabled'] = ::NewRelic::Agent.config[:'security.agent.enabled']
-          @cache[:enabled] = ::NewRelic::Agent.config[:'security.enabled']
+          @cache[:'security.enabled'] = ::NewRelic::Agent.config[:'security.enabled']
+          @cache[:enabled] = false
           @cache[:mode] = ::NewRelic::Agent.config[:'security.mode']
           @cache[:validator_service_url] = ::NewRelic::Agent.config[:'security.validator_service_url']
-          @cache[:'security.detection.rci.enabled'] = ::NewRelic::Agent.config[:'security.detection.rci.enabled']
-          @cache[:'security.detection.rxss.enabled'] = ::NewRelic::Agent.config[:'security.detection.rxss.enabled']
-          @cache[:'security.detection.deserialization.enabled'] = ::NewRelic::Agent.config[:'security.detection.deserialization.enabled']
+          # TODO: Remove security.detection.* & security.request.body_limit in next major release
+          @cache[:'security.detection.rci.enabled'] = ::NewRelic::Agent.config[:'security.detection.rci.enabled'].nil? ? true : ::NewRelic::Agent.config[:'security.detection.rci.enabled']
+          @cache[:'security.detection.rxss.enabled'] = ::NewRelic::Agent.config[:'security.detection.rxss.enabled'].nil? ? true : ::NewRelic::Agent.config[:'security.detection.rxss.enabled']
+          @cache[:'security.detection.deserialization.enabled'] = ::NewRelic::Agent.config[:'security.detection.deserialization.enabled'].nil? ? true : ::NewRelic::Agent.config[:'security.detection.deserialization.enabled']
+          @cache[:'security.scan_controllers.iast_scan_request_rate_limit'] = ::NewRelic::Agent.config[:'security.scan_controllers.iast_scan_request_rate_limit'].to_i
           @cache[:framework] = detect_framework
           @cache[:'security.application_info.port'] = ::NewRelic::Agent.config[:'security.application_info.port'].to_i
-          @cache[:'security.request.body_limit'] = ::NewRelic::Agent.config[:'security.request.body_limit'].to_i > 0 ? ::NewRelic::Agent.config[:'security.request.body_limit'].to_i : 300
           @cache[:listen_port] = nil
+          @cache[:process_start_time] = current_time_millis # TODO: Ruby doesn't provide process start time in pure ruby implementation using agent loading time for now.
+          @cache[:traffic_start_time] = nil
+          @cache[:scan_start_time] = nil
           @cache[:app_root] = NewRelic::Security::Agent::Utils.app_root
           @cache[:jruby_objectspace_enabled] = false
-          @cache[:json_version] = :'1.2.0'
+          @cache[:json_version] = :'1.2.8'
+          @cache[:'security.exclude_from_iast_scan.api'] = convert_to_regexp_list(::NewRelic::Agent.config[:'security.exclude_from_iast_scan.api'])
+          @cache[:'security.exclude_from_iast_scan.http_request_parameters.header'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.header']
+          @cache[:'security.exclude_from_iast_scan.http_request_parameters.query'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.query']
+          @cache[:'security.exclude_from_iast_scan.http_request_parameters.body'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.body']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.insecure_settings'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.insecure_settings']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.sql_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.sql_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.nosql_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.nosql_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.ldap_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ldap_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.javascript_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.javascript_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.command_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.command_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.xpath_injection'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.xpath_injection']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.ssrf'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ssrf']
+          @cache[:'security.exclude_from_iast_scan.iast_detection_category.rxss'] = ::NewRelic::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.rxss']
+          @cache[:'security.scan_schedule.delay'] = ::NewRelic::Agent.config[:'security.scan_schedule.delay'].to_i
+          @cache[:'security.scan_schedule.duration'] = ::NewRelic::Agent.config[:'security.scan_schedule.duration'].to_i
+          @cache[:'security.scan_schedule.schedule'] = ::NewRelic::Agent.config[:'security.scan_schedule.schedule']
+          @cache[:'security.scan_schedule.always_sample_traces'] = ::NewRelic::Agent.config[:'security.scan_schedule.always_sample_traces']
 
           @environment_source = NewRelic::Security::Agent::Configuration::EnvironmentSource.new
           @server_source = NewRelic::Security::Agent::Configuration::ServerSource.new
@@ -47,8 +70,7 @@ module NewRelic::Security
           @yaml_source = NewRelic::Security::Agent::Configuration::YamlSource.new
           @default_source = NewRelic::Security::Agent::Configuration::DefaultSource.new
         rescue Exception => exception
-          # TODO: remove this puts once agent stablizes
-          puts "Exception in Configuration::Manager.initialize : #{exception.inspect} #{exception.backtrace}"
+          ::NewRelic::Agent.notice_error(exception)
         end
   
         def [](key)
@@ -94,6 +116,14 @@ module NewRelic::Security
           @cache[:listen_port] = listen_port
         end
 
+        def traffic_start_time=(traffic_start_time)
+          @cache[:traffic_start_time] = traffic_start_time
+        end
+
+        def scan_start_time=(scan_start_time)
+          @cache[:scan_start_time] = scan_start_time
+        end
+
         def app_server=(app_server)
           @cache[:app_server] = app_server
         end
@@ -172,6 +202,19 @@ module NewRelic::Security
         def generate_key(entity_guid)
           ::OpenSSL::PKCS5.pbkdf2_hmac(entity_guid, entity_guid[0..15], 1024, 32, SHA1)
         end
+
+        def current_time_millis
+          (Time.now.to_f * 1000).to_i
+        end
+
+        def convert_to_regexp_list(value_list)
+          value_list.map do |value|
+            next unless value && !value.empty?
+            value = "^#{value}" if value[0] != '^'
+            value = "#{value}$" if value[-1] != '$'
+            /#{value}/
+          end
+        end
       end
     end
   end

data/lib/newrelic_security/agent/control/app_info.rb

@@ -26,6 +26,8 @@ module NewRelic::Security
           @jsonVersion = NewRelic::Security::Agent.config[:json_version]
           @startTime = current_time_millis
           @applicationUUID = NewRelic::Security::Agent.config[:uuid]
+          @appAccountId = NewRelic::Security::Agent.config[:account_id]
+          @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
           @framework = NewRelic::Security::Agent.config[:framework]
           @groupName = NewRelic::Security::Agent.config[:mode]
           @userProvidedApplicationInfo = Hash.new

data/lib/newrelic_security/agent/control/application_runtime_error.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'digest'
+
+module NewRelic::Security
+  module Agent
+    module Control
+      class ApplicationRuntimeError
+        attr_reader :jsonName, :exception
+        attr_accessor :counter
+
+        def initialize(exception, ctxt, response_code, category)
+          @collectorType = RUBY
+          @language = Ruby
+          @jsonName = :'application-runtime-error'
+          @eventType = :'application-runtime-error'
+          @collectorVersion = NewRelic::Security::VERSION
+          @buildNumber = nil
+          @jsonVersion = NewRelic::Security::Agent.config[:json_version]
+          @timestamp = current_time_millis
+          @applicationUUID = NewRelic::Security::Agent.config[:uuid]
+          @appAccountId = NewRelic::Security::Agent.config[:account_id]
+          @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
+          @framework = NewRelic::Security::Agent.config[:framework]
+          @groupName = NewRelic::Security::Agent.config[:mode]
+          @policyVersion = nil
+          @linkingMetadata = add_linking_metadata
+          @httpRequest = get_http_request_data(ctxt)
+          @exception = exception
+          @counter = 1
+          @responseCode = response_code
+          @category = category
+          @traceId = generate_trace_id(ctxt, category)
+        end
+
+        def as_json
+          instance_variables.map! do |ivar|
+            [ivar[1..-1].to_sym, instance_variable_get(ivar)]
+          end.to_h
+        end
+
+        def to_json(*_args)
+          as_json.to_json
+        end
+
+        private 
+
+        def current_time_millis
+          (Time.now.to_f * 1000).to_i
+        end
+
+        def add_linking_metadata
+          linking_metadata = {}
+          linking_metadata[:agentRunId] = NewRelic::Security::Agent.config[:agent_run_id]
+          linking_metadata.merge!(NewRelic::Security::Agent.config[:linking_metadata])
+          # TODO: add other fields as well in linking metadata, for event and heathcheck as well
+        end
+
+        def get_http_request_data(ctxt)
+          return if ctxt.nil?
+          http_request = {}
+          http_request[:parameterMap] = {}
+          http_request[:body] = ctxt.body
+          http_request[:generationTime] = ctxt.time_stamp
+          http_request[:dataTruncated] = false
+          http_request[:method] = ctxt.method
+          http_request[:route] = ctxt.route.split(AT_THE_RATE)[1] if ctxt.route
+          http_request[:url] = URI(ctxt.req[REQUEST_URI]).respond_to?(:request_uri) ? URI(ctxt.req[REQUEST_URI]).request_uri : ctxt.req[REQUEST_URI]
+          http_request[:requestURI] = "#{ctxt.req[RACK_URL_SCHEME]}://#{ctxt.req[HTTP_HOST]}#{ctxt.req[PATH_INFO]}"
+          http_request[:clientIP] = ctxt.headers.key?(X_FORWARDED_FOR) ? ctxt.headers[X_FORWARDED_FOR].split(COMMA)[0].to_s : ctxt.req[REMOTE_ADDR].to_s
+          http_request[:serverPort] = ctxt.req[SERVER_PORT].to_i
+          http_request[:protocol] = ctxt.req[RACK_URL_SCHEME]
+          http_request[:contextPath] = ROOT_PATH
+          http_request[:headers] = ctxt.headers
+          http_request[:contentType] = ctxt.req[CONTENT_TYPE] if ctxt.req.key?(CONTENT_TYPE)
+          http_request[:headers][CONTENT_TYPE1] = ctxt.req[CONTENT_TYPE] if ctxt.req.key?(CONTENT_TYPE)
+          http_request[:dataTruncated] = ctxt.data_truncated
+          http_request
+        end
+
+        def generate_trace_id(ctxt, category)
+          @exception[:stackTrace]
+          method, route = ctxt.route.split(AT_THE_RATE) if ctxt.route
+          if @exception[:stackTrace]
+            ::Digest::SHA256.hexdigest("#{@exception[:stackTrace].join(PIPE)}#{category}#{route}#{method}").to_s
+          else
+            ::Digest::SHA256.hexdigest("#{category}#{route}#{method}").to_s
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/newrelic_security/agent/control/application_url_mappings.rb

@@ -21,6 +21,8 @@ module NewRelic::Security
           @jsonVersion = NewRelic::Security::Agent.config[:json_version]
           @timestamp = current_time_millis
           @applicationUUID = NewRelic::Security::Agent.config[:uuid]
+          @appAccountId = NewRelic::Security::Agent.config[:account_id]
+          @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
           @framework = NewRelic::Security::Agent.config[:framework]
           @groupName = NewRelic::Security::Agent.config[:mode]
           @policyVersion = nil

data/lib/newrelic_security/agent/control/collector.rb

@@ -19,6 +19,8 @@ module NewRelic::Security
         def collect(case_type, args, event_category = nil, **keyword_args)
           return unless NewRelic::Security::Agent.config[:enabled]
           return if NewRelic::Security::Agent::Control::HTTPContext.get_context.nil? && NewRelic::Security::Agent::Control::GRPCContext.get_context.nil?
+          return if check_and_exclude_from_iast_scan_for_api
+          return if check_and_exclude_from_iast_scan_for_detection_category(case_type)
           args.map! { |file| Pathname.new(file).relative? ? File.join(Dir.pwd, file) : file } if [FILE_OPERATION, FILE_INTEGRITY].include?(case_type)
 
           event = NewRelic::Security::Agent::Control::Event.new(case_type, args, event_category)
@@ -44,7 +46,7 @@ module NewRelic::Security
           # Hence, considering only frame absolute_path & lineno for apiId calculation.
           user_frame_index = get_user_frame_index(stk)
           event.apiId = "#{case_type}-#{calculate_api_id(stk[0..user_frame_index].map { |frame| "#{frame.absolute_path}:#{frame.lineno}" }, event.httpRequest[:method], route)}"
-          stk.delete_if {|frame| frame.path.match(/newrelic_security/) || frame.path.match(/new_relic/)}
+          stk.delete_if { |frame| frame.path.match?(/newrelic_security/) || frame.path.match?(/new_relic/) }
           user_frame_index = get_user_frame_index(stk)
           return if case_type != REFLECTED_XSS && user_frame_index == -1 # TODO: Add log message here: "Filtered because User Stk frame NOT FOUND   \r\n"
           if user_frame_index != -1
@@ -59,8 +61,8 @@ module NewRelic::Security
           end
           event.stacktrace = stk[0..user_frame_index].map(&:to_s)
           NewRelic::Security::Agent.agent.event_processor.send_event(event)
-          if event.httpRequest[:headers].key?(NR_CSEC_FUZZ_REQUEST_ID) && event.apiId == event.httpRequest[:headers][NR_CSEC_FUZZ_REQUEST_ID].split(COLON_IAST_COLON)[0]
-            NewRelic::Security::Agent.agent.iast_client.completed_requests[event.parentId] << event.id if NewRelic::Security::Agent.agent.iast_client.completed_requests[event.parentId]
+          if event.httpRequest[:headers].key?(NR_CSEC_FUZZ_REQUEST_ID) && event.apiId == event.httpRequest[:headers][NR_CSEC_FUZZ_REQUEST_ID].split(COLON_IAST_COLON)[0] && NewRelic::Security::Agent.agent.iast_client.completed_requests[event.parentId]
+            NewRelic::Security::Agent.agent.iast_client.completed_requests[event.parentId] << event.id
           end
           event
         rescue Exception => exception
@@ -111,6 +113,35 @@ module NewRelic::Security
           }
         end
 
+        def check_and_exclude_from_iast_scan_for_api
+          NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.api'].each do |api|
+            return true if api&.match?(NewRelic::Security::Agent::Control::HTTPContext.get_context.url)
+          end
+          return false
+        end
+
+        def check_and_exclude_from_iast_scan_for_detection_category(case_type)
+          case case_type
+          when FILE_OPERATION, FILE_INTEGRITY
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access']
+          when SQL_DB_COMMAND
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.sql_injection']
+          when NOSQL_DB_COMMAND
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.nosql_injection']
+          when LDAP
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ldap_injection']
+          when SYSTEM_COMMAND
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.command_injection']
+          when XPATH
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.xpath_injection']
+          when HTTP_REQUEST
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.ssrf']
+          when REFLECTED_XSS
+            NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.iast_detection_category.rxss']
+          else
+            false
+          end
+        end
       end
     end
   end

data/lib/newrelic_security/agent/control/control_command.rb

@@ -32,7 +32,7 @@ module NewRelic::Security
               fuzz_request = NewRelic::Security::Agent::Control::FuzzRequest.new(message_object[:id])
               fuzz_request.request = prepare_fuzz_request(message_object)
               fuzz_request.case_type = message_object[:arguments][1]
-              fuzz_request.reflected_metadata =  message_object[:reflectedMetaData]
+              fuzz_request.reflected_metadata = message_object[:reflectedMetaData]
               NewRelic::Security::Agent.agent.iast_client.pending_request_ids << message_object[:id]
               NewRelic::Security::Agent.agent.iast_client.enqueue(fuzz_request)
               fuzz_request = nil
@@ -49,8 +49,6 @@ module NewRelic::Security
               message_object[:arguments].each { |processed_id| NewRelic::Security::Agent.agent.iast_client.completed_requests.delete(processed_id) }
             when 100
               NewRelic::Security::Agent.logger.debug "Control command : '100', #{message_object.to_json}"
-              ::NewRelic::Agent.instance.events.notify(:security_policy_received, message_object[:data])
-              # TODO: Update policy from file here, if enabled.
             when 101
 
             when 102
@@ -104,6 +102,7 @@ module NewRelic::Security
 
         def prepare_fuzz_request(message_object)
           message_object[:arguments][0].gsub!(NR_CSEC_VALIDATOR_HOME_TMP, NewRelic::Security::Agent.config[:fuzz_dir_path])
+          message_object[:arguments][0].gsub!(NR_CSEC_VALIDATOR_HOME_TMP_URL_ENCODED, NewRelic::Security::Agent.config[:fuzz_dir_path])
           message_object[:arguments][0].gsub!(NR_CSEC_VALIDATOR_FILE_SEPARATOR, ::File::SEPARATOR)
           prepared_fuzz_request = ::JSON.parse(message_object[:arguments][0])
           prepared_fuzz_request[HEADERS][NR_CSEC_PARENT_ID] = message_object[:id]

data/lib/newrelic_security/agent/control/critical_message.rb

@@ -21,6 +21,8 @@ module NewRelic::Security
           @buildNumber = nil
           @jsonVersion = NewRelic::Security::Agent.config[:json_version]
           @applicationUUID = NewRelic::Security::Agent.config[:uuid]
+          @appAccountId = NewRelic::Security::Agent.config[:account_id]
+          @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
           @linkingMetadata = add_linking_metadata
           @timestamp = current_time_millis
           @message = message

data/lib/newrelic_security/agent/control/error_reporting.rb

@@ -0,0 +1,74 @@
+module NewRelic::Security
+  module Agent
+    module Control
+      class ErrorReporting
+
+        STATUS_CODES_5XX = {
+          500 => "Internal Server Error",
+          501 => "Not Implemented",
+          502 => "Bad Gateway",
+          503 => "Service Unavailable",
+          504 => "Gateway Timeout",
+          505 => "HTTP Version Not Supported",
+          506 => "Variant Also Negotiates",
+          507 => "Insufficient Storage",
+          508 => "Loop Detected",
+          509 => "Bandwidth Limit Exceeded",
+          510 => "Not Extended",
+          511 => "Network Authentication Required"
+        }.freeze
+
+        attr_accessor :exceptions_map
+        
+        def initialize
+          @exceptions_map = {}
+        end
+
+        def generate_unhandled_exception(noticed_error, ctxt, response_code)
+          unhandled_exception = {}
+          category = nil
+          if noticed_error
+            unhandled_exception[:message] = noticed_error.message
+            unhandled_exception[:cause] = nil
+            unhandled_exception[:type] = noticed_error.exception_class_name
+            unhandled_exception[:stackTrace] = noticed_error.stack_trace
+            category = noticed_error.exception_class_name
+          end
+          category = STATUS_CODES_5XX[response_code] if response_code
+          application_runtime_error = NewRelic::Security::Agent::Control::ApplicationRuntimeError.new(unhandled_exception, ctxt, response_code, category)
+          key = if response_code
+            # TODO: when do refactoring of ctxt.route, use both route and method to generate key
+            ctxt.route&.+ response_code.to_s
+                else
+            application_runtime_error.exception[:type] + application_runtime_error.exception[:stackTrace][0]
+                end
+          application_runtime_error.counter = @exceptions_map[key].counter + 1 if @exceptions_map.key?(key)
+          @exceptions_map[key] = application_runtime_error
+        rescue Exception => exception
+          NewRelic::Security::Agent.logger.error "Exception in generating unhandled exception: #{exception.inspect} #{exception.backtrace}\n"
+        end
+
+        def extract_noticed_error(current_transaction, ctxt, response_code)
+          # TODO: Below operation is expensive, talk to APM to get optimized way to do this
+          current_transaction.exceptions.each do |_, span|
+            current_transaction.segments.each do |segment|
+              generate_unhandled_exception(segment.noticed_error, ctxt, response_code) if span[:span_id] == segment.guid
+            end
+          end
+        rescue Exception => exception
+          NewRelic::Security::Agent.logger.error "Exception in extract_noticed_error: #{exception.inspect} #{exception.backtrace}\n"
+        end
+
+        def report_unhandled_or_5xx_exceptions(current_transaction, ctxt, response_code = nil)
+          http_response_code = response_code || current_transaction&.http_response_code
+          if current_transaction.exceptions.empty? && http_response_code&.between?(500, 599)
+            generate_unhandled_exception(nil, ctxt, response_code)
+          else
+            extract_noticed_error(current_transaction, ctxt, response_code) unless current_transaction.exceptions.empty?
+          end
+        end
+
+      end 
+    end
+  end
+end

data/lib/newrelic_security/agent/control/event.rb

@@ -26,9 +26,24 @@ module NewRelic::Security
           @buildNumber = nil
           @jsonVersion = NewRelic::Security::Agent.config[:json_version]
           @applicationUUID = NewRelic::Security::Agent.config[:uuid]
+          @appAccountId = NewRelic::Security::Agent.config[:account_id]
+          @appEntityGuid = NewRelic::Security::Agent.config[:entity_guid]
           @httpRequest = Hash.new
           @httpResponse = Hash.new
-          @metaData = { :reflectedMetaData => { :listen_port => NewRelic::Security::Agent.config[:listen_port].to_s } }
+          @metaData = {
+            :reflectedMetaData => {
+              :listen_port => NewRelic::Security::Agent.config[:listen_port].to_s
+            },
+            :appServerInfo => { 
+              :applicationDirectory => NewRelic::Security::Agent.config[:app_root], 
+              :serverBaseDirectory => NewRelic::Security::Agent.config[:app_root] 
+            },
+            :skipScanParameters => {
+              :header => NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.header'],
+              :query => NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.query'],
+              :body => NewRelic::Security::Agent.config[:'security.exclude_from_iast_scan.http_request_parameters.body']
+            }
+          }
           @linkingMetadata = add_linking_metadata
           @pid = pid
           @parameters = args
@@ -113,7 +128,7 @@ module NewRelic::Security
           http_request[:contentType] = "TODO: "
           http_request[:isGrpc] = ctxt.is_grpc
           @httpRequest = http_request
-          @metaData = ctxt.metadata
+          @metaData.merge!(ctxt.metadata)
         end
 
         private
@@ -128,13 +143,20 @@ module NewRelic::Security
         end
 
         def event_id
-          "#{Process.pid}:#{::Thread.current.object_id}:#{thread_monotonic_ctr}"
+          "#{Process.pid}:#{current_transaction&.guid}:#{thread_monotonic_ctr}"
+        end
+
+        def current_transaction
+          ::NewRelic::Agent::Tracer.current_transaction if defined?(::NewRelic::Agent::Tracer)
         end
 
         def thread_monotonic_ctr
           ctxt = NewRelic::Security::Agent::Control::HTTPContext.get_context if NewRelic::Security::Agent::Control::HTTPContext.get_context
           ctxt = NewRelic::Security::Agent::Control::GRPCContext.get_context if NewRelic::Security::Agent::Control::GRPCContext.get_context
-          ctxt.event_counter = ctxt.event_counter + 1 if ctxt
+          return unless ctxt
+          ctxt.mutex.synchronize do
+            ctxt.event_counter = ctxt.event_counter + 1
+          end
         end
 
         def add_linking_metadata